From 00e235d30d99adf75b531c29ec28a8d793690a0d Mon Sep 17 00:00:00 2001 From: zixi0825 Date: Sun, 29 Mar 2026 07:31:33 +0800 Subject: [PATCH] [Refactor][SPI] Refactor spi module --- .../connector/api/ConnectorFactory.java | 10 +- .../plugin/ClickHouseConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/DatabendConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../connector/plugin/DmConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/DorisConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/FileConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/FlinkConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/HiveConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/ImpalaConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/MaxComputeConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/MongodbConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/MysqlConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/OracleConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/PostgreSqlConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/PrestoConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/SparkConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/SqlServerConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/StarRocksConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../plugin/TrinoConnectorFactory.java | 5 + ...o.datavines.connector.api.ConnectorFactory | 1 - ...o.datavines.connector.api.ConnectorFactory | 1 + .../engine/api/component/Component.java | 6 +- .../engine/api/engine/EngineExecutor.java | 11 +- .../engine/api/env/RuntimeEnvironment.java | 6 +- .../config/BaseJobConfigurationBuilder.java | 18 +- .../config/DataVinesConfigurationManager.java | 14 +- .../config/JobConfigurationBuilder.java | 5 +- .../engine/config/MetricParserUtils.java | 4 +- .../engine/core/config/ConfigParser.java | 18 +- .../flink/api/FlinkRuntimeEnvironment.java | 5 + ...atavines.engine.api.env.RuntimeEnvironment | 1 - ...atavines.engine.api.env.RuntimeEnvironment | 1 + .../config/BaseFlinkConfigurationBuilder.java | 25 +- .../config/FlinkDataProfileMetricBuilder.java | 5 + .../FlinkMultiTableAccuracyMetricBuilder.java | 9 +- ...ultiTableValueComparisonMetricBuilder.java | 11 +- .../FlinkSingleTableConfigurationBuilder.java | 11 +- ...ines.engine.config.JobConfigurationBuilder | 4 - ...ines.engine.config.JobConfigurationBuilder | 4 + .../engine/flink/jdbc/sink/JdbcSink.java | 5 + .../engine/flink/jdbc/sink/MySQLSink.java | 5 + .../engine/flink/jdbc/source/JdbcSource.java | 5 + ...o.datavines.engine.api.component.Component | 3 - ...o.datavines.engine.api.component.Component | 3 + .../flink/executor/FlinkEngineExecutor.java | 5 + ...datavines.engine.api.engine.EngineExecutor | 1 - ...datavines.engine.api.engine.EngineExecutor | 1 + .../flink/transform/sql/SqlTransform.java | 5 + ...o.datavines.engine.api.component.Component | 1 - ...o.datavines.engine.api.component.Component | 1 + .../livy/executor/LivyEngineExecutor.java | 5 + ...datavines.engine.api.engine.EngineExecutor | 1 - ...datavines.engine.api.engine.EngineExecutor | 1 + .../local/api/LocalRuntimeEnvironment.java | 5 + .../local/api/entity/ConnectionHolder.java | 6 +- ...atavines.engine.api.env.RuntimeEnvironment | 1 - ...atavines.engine.api.env.RuntimeEnvironment | 1 + .../config/BaseLocalConfigurationBuilder.java | 30 +- .../config/LocalDataProfileMetricBuilder.java | 5 + .../LocalMultiTableAccuracyMetricBuilder.java | 17 +- ...ultiTableValueComparisonMetricBuilder.java | 13 +- .../config/LocalSingleTableMetricBuilder.java | 17 +- ...ines.engine.config.JobConfigurationBuilder | 4 - ...ines.engine.config.JobConfigurationBuilder | 4 + .../engine/local/connector/LocalFileSink.java | 9 +- .../local/connector/LocalFileSource.java | 5 + ...o.datavines.engine.api.component.Component | 2 - ...o.datavines.engine.api.component.Component | 2 + .../engine/local/connector/BaseJdbcSink.java | 5 + .../local/connector/BaseJdbcSource.java | 9 +- .../engine/local/connector/MySQLSink.java | 5 + .../executor/ErrorDataSinkExecutor.java | 10 +- ...o.datavines.engine.api.component.Component | 3 - ...o.datavines.engine.api.component.Component | 3 + .../local/executor/LocalEngineExecutor.java | 5 + ...datavines.engine.api.engine.EngineExecutor | 1 - ...datavines.engine.api.engine.EngineExecutor | 1 + .../local/transform/sql/SqlTransform.java | 5 + ...o.datavines.engine.api.component.Component | 1 - ...o.datavines.engine.api.component.Component | 1 + .../spark/api/SparkRuntimeEnvironment.java | 5 + ...atavines.engine.api.env.RuntimeEnvironment | 2 - ...atavines.engine.api.env.RuntimeEnvironment | 1 + .../config/BaseSparkConfigurationBuilder.java | 712 +++++++++--------- .../config/SparkDataProfileMetricBuilder.java | 5 + .../SparkMultiTableAccuracyMetricBuilder.java | 9 +- ...ultiTableValueComparisonMetricBuilder.java | 11 +- .../config/SparkSingleTableMetricBuilder.java | 11 +- ...ines.engine.config.JobConfigurationBuilder | 8 - ...ines.engine.config.JobConfigurationBuilder | 4 + .../engine/spark/jdbc/sink/JdbcSink.java | 5 + .../engine/spark/jdbc/sink/MySQLSink.java | 5 + .../engine/spark/jdbc/source/JdbcSource.java | 5 + ...o.datavines.engine.api.component.Component | 6 - ...o.datavines.engine.api.component.Component | 3 + .../engine/spark/jdbc/sink/MongodbSink.java | 5 + .../spark/jdbc/source/MongodbSource.java | 5 + ...o.datavines.engine.api.component.Component | 4 - ...o.datavines.engine.api.component.Component | 2 + .../spark/executor/SparkEngineExecutor.java | 5 + ...datavines.engine.api.engine.EngineExecutor | 1 - ...datavines.engine.api.engine.EngineExecutor | 1 + .../spark/transform/sql/SqlTransform.java | 5 + ...o.datavines.engine.api.component.Component | 2 - ...o.datavines.engine.api.component.Component | 1 + .../datavines/metric/api/ExpectedValue.java | 6 +- .../datavines/metric/api/MetricValidator.java | 8 +- .../datavines/metric/api/ResultFormula.java | 11 +- .../io/datavines/metric/api/SqlMetric.java | 11 +- .../plugin/AbstractExpectedValue.java | 6 +- .../metric/expected/plugin/DailyAvg.java | 5 + .../metric/expected/plugin/SparkDailyAvg.java | 66 -- .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../metric/expected/plugin/FixValue.java | 5 + .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../metric/expected/plugin/Last30DayAvg.java | 5 + .../expected/plugin/SparkLast30DayAvg.java | 66 -- .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../metric/expected/plugin/Last7DayAvg.java | 5 + .../expected/plugin/SparkLast7DayAvg.java | 66 -- .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../metric/expected/plugin/MonthlyAvg.java | 5 + .../expected/plugin/SparkMonthlyAvg.java | 66 -- .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../metric/expected/plugin/None.java | 5 + .../metric/expected/plugin/SparkNone.java | 5 + .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 2 + .../expected/plugin/TableTotalRows.java | 5 + .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../expected/plugin/TargetTableTotalRows.java | 5 + .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../expected/plugin/SparkWeeklyAvg.java | 66 -- .../metric/expected/plugin/WeeklyAvg.java | 5 + .../io.datavines.metric.api.ExpectedValue | 4 - .../io.datavines.metric.api.ExpectedValue | 1 + .../metric/plugin/base/BaseSingleTable.java | 4 +- .../metric/plugin/ColumnAvgLength.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../io/datavines/metric/plugin/ColumnAvg.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../datavines/metric/plugin/ColumnBlank.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnDistinct.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnDuplicate.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnHistogram.java | 9 +- .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnInEnums.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../datavines/metric/plugin/ColumnLength.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnMatchNotRegex.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnMatchRegex.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnMaxLength.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../io/datavines/metric/plugin/ColumnMax.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnMinLength.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../io/datavines/metric/plugin/ColumnMin.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnNotInEnums.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnNotNull.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../datavines/metric/plugin/ColumnNull.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../datavines/metric/plugin/ColumnStdDev.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../io/datavines/metric/plugin/ColumnSum.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../datavines/metric/plugin/ColumnUnique.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnValueBetween.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/ColumnVariance.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/CustomAggregateSql.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/MultiTableAccuracy.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../plugin/MultiTableValueComparison.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../io/datavines/metric/plugin/Freshness.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/plugin/TableRowCount.java | 5 + .../plugins/io.datavines.metric.api.SqlMetric | 1 - .../io.datavines.metric.api.SqlMetric | 1 + .../metric/result/formula/Count.java | 5 + .../io.datavines.metric.api.ResultFormula | 1 - .../io.datavines.metric.api.ResultFormula | 1 + .../formula/ActualMinusExpectedDiff.java | 5 + .../io.datavines.metric.api.ResultFormula | 1 - .../io.datavines.metric.api.ResultFormula | 1 + .../metric/result/formula/DiffPercentage.java | 5 + .../io.datavines.metric.api.ResultFormula | 1 - .../io.datavines.metric.api.ResultFormula | 1 + .../datavines/metric/result/formula/Diff.java | 5 + .../io.datavines.metric.api.ResultFormula | 1 - .../io.datavines.metric.api.ResultFormula | 1 + .../metric/result/formula/Percentage.java | 5 + .../io.datavines.metric.api.ResultFormula | 1 - .../io.datavines.metric.api.ResultFormula | 1 + .../api/spi/SlasHandlerPlugin.java | 10 +- .../core/NotificationManager.java | 10 +- .../dingtalk/DingTalkSlasHandlerPlugin.java | 5 + ...nes.notification.api.spi.SlasHandlerPlugin | 1 - ...nes.notification.api.spi.SlasHandlerPlugin | 1 + .../plugin/email/EmailSlasHandlerPlugin.java | 5 + ...nes.notification.api.spi.SlasHandlerPlugin | 1 - ...nes.notification.api.spi.SlasHandlerPlugin | 1 + .../plugin/lark/LarkSlasHandlerPlugin.java | 5 + ...nes.notification.api.spi.SlasHandlerPlugin | 1 - ...nes.notification.api.spi.SlasHandlerPlugin | 1 + .../wecombot/WecomBotSlasHandlerPlugin.java | 5 + ...nes.notification.api.spi.SlasHandlerPlugin | 1 - ...nes.notification.api.spi.SlasHandlerPlugin | 1 + .../io/datavines/registry/api/Registry.java | 11 +- .../registry/plugin/MysqlRegistry.java | 5 + .../io.datavines.registry.api.Registry | 1 - .../io.datavines.registry.api.Registry | 1 + .../registry/plugin/ZooKeeperRegistry.java | 5 + .../io.datavines.registry.api.Registry | 1 - .../io.datavines.registry.api.Registry | 1 + .../java/io/datavines/runner/JobRunner.java | 16 +- .../io/datavines/server/DataVinesServer.java | 6 +- .../api/controller/DataSourceController.java | 6 +- .../ErrorDataStorageController.java | 6 +- .../api/controller/MetricController.java | 32 +- .../validator/JobResultValidator.java | 10 +- .../server/dqc/executor/runner/JobRunner.java | 6 +- .../impl/CatalogEntityRelServiceImpl.java | 4 +- .../service/impl/DataSourceServiceImpl.java | 20 +- .../impl/ErrorDataStorageServiceImpl.java | 6 +- .../JobExecutionErrorDataServiceImpl.java | 4 +- .../impl/JobExecutionResultServiceImpl.java | 10 +- .../service/impl/JobExecutionServiceImpl.java | 18 +- .../impl/JobQualityReportServiceImpl.java | 8 +- .../service/impl/JobServiceImpl.java | 10 +- .../impl/SlaNotificationServiceImpl.java | 6 +- .../service/impl/SlaServiceImpl.java | 10 +- .../CatalogMetaDataFetchExecutorImpl.java | 6 +- .../server/utils/JobParameterUtils.java | 4 +- datavines-spi/pom.xml | 10 - .../java/io/datavines/spi/CachingFactory.java | 103 +++ ...r.java => DuplicateProviderException.java} | 27 +- .../java/io/datavines/spi/KeyedRegistry.java | 232 ++++++ .../io/datavines/spi/OrderedRegistry.java | 97 +++ .../io/datavines/spi/PluginDiscovery.java | 163 ++++ .../java/io/datavines/spi/PluginLoader.java | 486 ------------ .../java/io/datavines/spi/PluginManager.java | 142 ++++ .../java/io/datavines/spi/Prioritized.java | 70 -- ...PI.java => ProviderNotFoundException.java} | 30 +- .../io/datavines/spi/ServiceLoaderUtils.java | 86 +++ .../io/datavines/spi/utils/ClassUtils.java | 50 -- .../spi/utils/UnsafeStringWriter.java | 108 --- 327 files changed, 2092 insertions(+), 1802 deletions(-) delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory create mode 100644 datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor delete mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component create mode 100644 datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/SparkDailyAvg.java delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast30DayAvg.java delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast7DayAvg.java delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkMonthlyAvg.java delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkWeeklyAvg.java delete mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue create mode 100644 datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric create mode 100644 datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric delete mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula create mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula delete mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula create mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula delete mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula create mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula delete mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula create mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula delete mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula create mode 100644 datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula delete mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin create mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin delete mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin create mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin delete mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin create mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin delete mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin create mode 100644 datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin delete mode 100644 datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry create mode 100644 datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/services/io.datavines.registry.api.Registry delete mode 100644 datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry create mode 100644 datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/services/io.datavines.registry.api.Registry create mode 100644 datavines-spi/src/main/java/io/datavines/spi/CachingFactory.java rename datavines-spi/src/main/java/io/datavines/spi/{utils/Holder.java => DuplicateProviderException.java} (51%) create mode 100644 datavines-spi/src/main/java/io/datavines/spi/KeyedRegistry.java create mode 100644 datavines-spi/src/main/java/io/datavines/spi/OrderedRegistry.java create mode 100644 datavines-spi/src/main/java/io/datavines/spi/PluginDiscovery.java delete mode 100644 datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java create mode 100644 datavines-spi/src/main/java/io/datavines/spi/PluginManager.java delete mode 100644 datavines-spi/src/main/java/io/datavines/spi/Prioritized.java rename datavines-spi/src/main/java/io/datavines/spi/{SPI.java => ProviderNotFoundException.java} (53%) create mode 100644 datavines-spi/src/main/java/io/datavines/spi/ServiceLoaderUtils.java delete mode 100644 datavines-spi/src/main/java/io/datavines/spi/utils/ClassUtils.java delete mode 100644 datavines-spi/src/main/java/io/datavines/spi/utils/UnsafeStringWriter.java diff --git a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java index c54aaf733..c239e1d32 100644 --- a/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java +++ b/datavines-connector/datavines-connector-api/src/main/java/io/datavines/connector/api/ConnectorFactory.java @@ -16,11 +16,17 @@ */ package io.datavines.connector.api; -import io.datavines.spi.SPI;; +import java.util.Collection; +import java.util.Collections; -@SPI public interface ConnectorFactory { + String getPluginName(); + + default Collection getPluginNames() { + return Collections.singletonList(getPluginName()); + } + String getCategory(); Connector getConnector(); diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java index ae3e5e6d7..310478a7c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/java/io/datavines/connector/plugin/ClickHouseConnectorFactory.java @@ -54,4 +54,9 @@ public ConfigBuilder getConfigBuilder() { public TypeConverter getTypeConverter() { return new ClickHouseTypeConverter(); } + + @Override + public String getPluginName() { + return "clickhouse"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index a3dbe3520..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -clickhouse=io.datavines.connector.plugin.ClickHouseConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..8cadb3e16 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-clickhouse/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.ClickHouseConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java index a79e12243..ff6234819 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/java/io/datavines/connector/plugin/DatabendConnectorFactory.java @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new DatabendConfigBuilder(); } + + @Override + public String getPluginName() { + return "databend"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 9a740e927..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -databend=io.datavines.connector.plugin.DatabendConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..661cf10ad --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-databend/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.DatabendConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java index 579c46f46..e4fd5c5eb 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/java/io/datavines/connector/plugin/DmConnectorFactory.java @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new DmConfigBuilder(); } + + @Override + public String getPluginName() { + return "dm"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 8ae1a6816..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -dm=io.datavines.connector.plugin.DmConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..6ec6733da --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-dm/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.DmConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java index cce578989..181e8f53f 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/java/io/datavines/connector/plugin/DorisConnectorFactory.java @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new DorisConfigBuilder(); } + + @Override + public String getPluginName() { + return "doris"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 0b0cff57d..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -doris=io.datavines.connector.plugin.DorisConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..2f879c4d0 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-doris/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.DorisConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java index f0a782cee..b6f939635 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/java/io/datavines/connector/plugin/FileConnectorFactory.java @@ -84,4 +84,9 @@ public MetricScript getMetricScript() { public Boolean showInFrontend() { return false; } + + @Override + public String getPluginName() { + return "file"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index ae4dc0e82..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -file=io.datavines.connector.plugin.FileConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..70cb521fe --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-file/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.FileConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/java/io/datavines/connector/plugin/FlinkConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/java/io/datavines/connector/plugin/FlinkConnectorFactory.java index c5bcd05ef..4e68cb053 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/java/io/datavines/connector/plugin/FlinkConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/java/io/datavines/connector/plugin/FlinkConnectorFactory.java @@ -49,4 +49,9 @@ public MetricScript getMetricScript() { public Boolean showInFrontend() { return false; } + + @Override + public String getPluginName() { + return "flink"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 35d33dca2..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -flink=io.datavines.connector.plugin.FlinkConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..ec1444e63 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-flink/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.FlinkConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java index 4f29dc145..de96357a9 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/java/io/datavines/connector/plugin/HiveConnectorFactory.java @@ -54,4 +54,9 @@ public MetricScript getMetricScript() { public DataSourceClient getDataSourceClient() { return new HiveDataSourceClient(); } + + @Override + public String getPluginName() { + return "hive"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index fb4feb721..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -hive=io.datavines.connector.plugin.HiveConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..05f7e14d9 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-hive/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.HiveConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java index ac2515d74..a39381f6c 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/java/io/datavines/connector/plugin/ImpalaConnectorFactory.java @@ -42,4 +42,9 @@ public Connector getConnector() { public Executor getExecutor() { return new ImpalaExecutor(getDataSourceClient()); } + + @Override + public String getPluginName() { + return "impala"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 8d0b75ab7..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -impala=io.datavines.connector.plugin.ImpalaConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..870a1b9d7 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-impala/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.ImpalaConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/java/io/datavines/connector/plugin/MaxComputeConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/java/io/datavines/connector/plugin/MaxComputeConnectorFactory.java index 63c17e9c4..ba5450a55 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/java/io/datavines/connector/plugin/MaxComputeConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/java/io/datavines/connector/plugin/MaxComputeConnectorFactory.java @@ -54,4 +54,9 @@ public ConfigBuilder getConfigBuilder() { public MetricScript getMetricScript() { return new MaxComputeMetricScript(); } + + @Override + public String getPluginName() { + return "maxcompute"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 57b69141e..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -maxcompute=io.datavines.connector.plugin.MaxComputeConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..e9dbd601e --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-maxcompute/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.MaxComputeConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java index 78591eda8..ef8119dfb 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/java/io/datavines/connector/plugin/MongodbConnectorFactory.java @@ -49,4 +49,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new MongodbConfigBuilder(); } + + @Override + public String getPluginName() { + return "mongodb"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 614dfdca3..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -mongodb=io.datavines.connector.plugin.MongodbConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..5cddbf43a --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mongodb/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.MongodbConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java index 303853ef2..c5547245b 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/java/io/datavines/connector/plugin/MysqlConnectorFactory.java @@ -49,4 +49,9 @@ public ConfigBuilder getConfigBuilder() { public MetricScript getMetricScript() { return new MysqlMetricScript(); } + + @Override + public String getPluginName() { + return "mysql"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 4fb335f51..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -mysql=io.datavines.connector.plugin.MysqlConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..fc9fb3737 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-mysql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.MysqlConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java index cfc6ffdb0..fa691fca8 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/java/io/datavines/connector/plugin/OracleConnectorFactory.java @@ -54,4 +54,9 @@ public TypeConverter getTypeConverter() { public MetricScript getMetricScript() { return new OracleMetricScript(); } + + @Override + public String getPluginName() { + return "oracle"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index c21ba1cec..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -oracle=io.datavines.connector.plugin.OracleConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..2bf9b9f26 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-oracle/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.OracleConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java index 45d8fcd27..75686b965 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/java/io/datavines/connector/plugin/PostgreSqlConnectorFactory.java @@ -54,4 +54,9 @@ public TypeConverter getTypeConverter() { public MetricScript getMetricScript() { return new PostgreSqlMetricScript(); } + + @Override + public String getPluginName() { + return "postgresql"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 8a3f6a1a2..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -postgresql=io.datavines.connector.plugin.PostgreSqlConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..951b8ca0f --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-postgresql/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.PostgreSqlConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java index f28fba1a0..7d4bdd45d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/java/io/datavines/connector/plugin/PrestoConnectorFactory.java @@ -49,4 +49,9 @@ public ConfigBuilder getConfigBuilder() { public DataSourceClient getDataSourceClient() { return new PrestoDataSourceClient(); } + + @Override + public String getPluginName() { + return "presto"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index de2d08444..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -presto=io.datavines.connector.plugin.PrestoConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..bd13c5c41 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-presto/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.PrestoConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java index f13d2ef43..731172431 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/java/io/datavines/connector/plugin/SparkConnectorFactory.java @@ -49,4 +49,9 @@ public MetricScript getMetricScript() { public Boolean showInFrontend() { return false; } + + @Override + public String getPluginName() { + return "spark"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index e46f3bf58..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -spark=io.datavines.connector.plugin.SparkConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..9f3f980ae --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-spark/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.SparkConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java index 1efee656d..f05001f48 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/java/io/datavines/connector/plugin/SqlServerConnectorFactory.java @@ -49,4 +49,9 @@ public ConfigBuilder getConfigBuilder() { public MetricScript getMetricScript() { return new SqlServerMetricScript(); } + + @Override + public String getPluginName() { + return "sqlserver"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 8bca161bd..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -sqlserver=io.datavines.connector.plugin.SqlServerConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..4c2abe57a --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-sqlserver/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.SqlServerConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java index 600d205a6..3b8dc9b3d 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/java/io/datavines/connector/plugin/StarRocksConnectorFactory.java @@ -44,4 +44,9 @@ public Executor getExecutor() { public ConfigBuilder getConfigBuilder() { return new StarRocksConfigBuilder(); } + + @Override + public String getPluginName() { + return "starrocks"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index 2167b3cc1..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -starrocks=io.datavines.connector.plugin.StarRocksConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..f3bcb4819 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-starrocks/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.StarRocksConnectorFactory diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java index 4d71bf825..3d6b66fa6 100644 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/java/io/datavines/connector/plugin/TrinoConnectorFactory.java @@ -49,4 +49,9 @@ public ConfigBuilder getConfigBuilder() { public DataSourceClient getDataSourceClient() { return new TrinoDataSourceClient(); } + + @Override + public String getPluginName() { + return "trino"; + } } diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory deleted file mode 100644 index a88b775ff..000000000 --- a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/plugins/io.datavines.connector.api.ConnectorFactory +++ /dev/null @@ -1 +0,0 @@ -trino=io.datavines.connector.plugin.TrinoConnectorFactory \ No newline at end of file diff --git a/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory new file mode 100644 index 000000000..0bb608730 --- /dev/null +++ b/datavines-connector/datavines-connector-plugins/datavines-connector-trino/src/main/resources/META-INF/services/io.datavines.connector.api.ConnectorFactory @@ -0,0 +1 @@ +io.datavines.connector.plugin.TrinoConnectorFactory diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/component/Component.java b/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/component/Component.java index c2291c27d..f9d38f42a 100644 --- a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/component/Component.java +++ b/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/component/Component.java @@ -16,11 +16,13 @@ */ package io.datavines.engine.api.component; -import io.datavines.spi.SPI;; import io.datavines.engine.api.env.RuntimeEnvironment; import io.datavines.engine.api.plugin.Plugin; -@SPI +import java.util.Collection; + public interface Component extends Plugin { + + Collection getPluginNames(); void prepare(RuntimeEnvironment env) throws Exception; } diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java b/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java index f4613c047..e4a226564 100644 --- a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java +++ b/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java @@ -21,11 +21,18 @@ import io.datavines.common.config.Configurations; import io.datavines.common.entity.ProcessResult; -import io.datavines.spi.SPI; -@SPI +import java.util.Collection; +import java.util.Collections; + public interface EngineExecutor { + String getPluginName(); + + default Collection getPluginNames() { + return Collections.singletonList(getPluginName()); + } + void init(JobExecutionRequest jobExecutionRequest, Logger logger, Configurations configurations) throws Exception; void execute() throws Exception; diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java b/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java index ccca77cff..bca302c9b 100644 --- a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java +++ b/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java @@ -16,12 +16,14 @@ */ package io.datavines.engine.api.env; -import io.datavines.spi.SPI;; import io.datavines.engine.api.plugin.Plugin; -@SPI +import java.util.Collection; + public interface RuntimeEnvironment extends Plugin { + Collection getPluginNames(); + void prepare(); Execution getExecution(); diff --git a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/BaseJobConfigurationBuilder.java b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/BaseJobConfigurationBuilder.java index cc2aa83a3..785545f8a 100644 --- a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/BaseJobConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/BaseJobConfigurationBuilder.java @@ -26,7 +26,7 @@ import io.datavines.connector.api.ConnectorFactory; import io.datavines.metric.api.ExpectedValue; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.CollectionUtils; @@ -153,8 +153,8 @@ public void buildTransformConfigs() { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); boolean invalidateItemCanOutput = Boolean.parseBoolean(metricInputParameter.get(INVALIDATE_ITEM_CAN_OUTPUT)); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); @@ -187,8 +187,8 @@ public void buildTransformConfigs() { // generate expected value transform sql String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) + .getNewPlugin(expectedType); ExecuteSql expectedValueExecuteSql = @@ -227,7 +227,7 @@ protected SourceConfig getValidateResultDataSourceConfig() throws DataVinesExcep SourceConfig actualValueSourceConfig = new SourceConfig(); ConnectorFactory storageFactory = - PluginLoader.getPluginLoader(ConnectorFactory.class) + PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) .getOrCreatePlugin(jobExecutionInfo.getValidateResultDataStorageType()); actualValueSourceConfig.setPlugin(storageFactory.getCategory()); @@ -240,7 +240,7 @@ protected SourceConfig getValidateResultDataSourceConfig(String outputTable) thr SourceConfig actualValueSourceConfig = new SourceConfig(); ConnectorFactory storageFactory = - PluginLoader.getPluginLoader(ConnectorFactory.class) + PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) .getOrCreatePlugin(jobExecutionInfo.getValidateResultDataStorageType()); actualValueSourceConfig.setPlugin(storageFactory.getCategory()); @@ -271,7 +271,7 @@ protected SinkConfig getValidateResultDataSinkConfig(ExpectedValue expectedValue private Map getValidateResultSourceConfigMap(String sql, String dbTable) { Map configMap = new HashMap<>(); ConnectorFactory storageFactory = - PluginLoader.getPluginLoader(ConnectorFactory.class) + PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) .getOrCreatePlugin(jobExecutionInfo.getValidateResultDataStorageType()); if (storageFactory != null) { if (StringUtils.isNotEmpty(jobExecutionInfo.getValidateResultDataStorageParameter())) { @@ -292,7 +292,7 @@ private Map getValidateResultSourceConfigMap(String sql, String d private Map getValidateResultSourceConfigMap(String sql, String dbTable,String outputTable) { Map configMap = new HashMap<>(); ConnectorFactory storageFactory = - PluginLoader.getPluginLoader(ConnectorFactory.class) + PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) .getOrCreatePlugin(jobExecutionInfo.getValidateResultDataStorageType()); if (storageFactory != null) { if (StringUtils.isNotEmpty(jobExecutionInfo.getValidateResultDataStorageParameter())) { diff --git a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/DataVinesConfigurationManager.java b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/DataVinesConfigurationManager.java index 5d84afb9d..dd5743194 100644 --- a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/DataVinesConfigurationManager.java +++ b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/DataVinesConfigurationManager.java @@ -26,7 +26,7 @@ import io.datavines.common.exception.DataVinesException; import io.datavines.common.utils.StringUtils; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.List; @@ -68,8 +68,8 @@ public static DataVinesJobConfig generateConfiguration(JobType jobType, throw new DataVinesException("metric type can not be null"); } - sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); if (sqlMetric == null) { @@ -81,13 +81,13 @@ public static DataVinesJobConfig generateConfiguration(JobType jobType, throw new DataVinesException("can not find the metric"); } - JobConfigurationBuilder builder = PluginLoader - .getPluginLoader(JobConfigurationBuilder.class) + JobConfigurationBuilder builder = PluginDiscovery.getMultiKeyPluginDiscovery(JobConfigurationBuilder.class, JobConfigurationBuilder::getPluginNames) + .getNewPlugin(jobExecutionInfo.getEngineType() + "_" + sqlMetric.getType().getDescription()); if (jobType == JobType.DATA_PROFILE) { - builder = PluginLoader - .getPluginLoader(JobConfigurationBuilder.class) + builder = PluginDiscovery.getMultiKeyPluginDiscovery(JobConfigurationBuilder.class, JobConfigurationBuilder::getPluginNames) + .getNewPlugin(jobExecutionInfo.getEngineType() + "_data_profile"); } diff --git a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/JobConfigurationBuilder.java b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/JobConfigurationBuilder.java index 572e5ea4a..3bc3d965d 100644 --- a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/JobConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/JobConfigurationBuilder.java @@ -19,13 +19,14 @@ import io.datavines.common.config.DataVinesJobConfig; import io.datavines.common.entity.JobExecutionInfo; import io.datavines.common.exception.DataVinesException; -import io.datavines.spi.SPI; +import java.util.Collection; import java.util.Map; -@SPI public interface JobConfigurationBuilder { + Collection getPluginNames(); + void init(Map inputParameter, JobExecutionInfo jobExecutionInfo); void buildName(); diff --git a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/MetricParserUtils.java b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/MetricParserUtils.java index eb4a4f7d3..e6bb20762 100644 --- a/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/MetricParserUtils.java +++ b/datavines-engine/datavines-engine-config/src/main/java/io/datavines/engine/config/MetricParserUtils.java @@ -24,7 +24,7 @@ import io.datavines.engine.common.utils.QuoteIdentifier; import io.datavines.metric.api.ConfigItem; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.MapUtils; import java.time.LocalDateTime; @@ -125,7 +125,7 @@ public static String generateUniqueCode(Map inputParameterValue) Map newInputParameterValue = new HashMap<>(); newInputParameterValue.put(METRIC_NAME, inputParameterValue.get(METRIC_NAME)); newInputParameterValue.put(DATABASE, inputParameterValue.get(DATABASE)); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(StringUtils.removeSingeQuotes(inputParameterValue.get(METRIC_NAME))); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(StringUtils.removeSingeQuotes(inputParameterValue.get(METRIC_NAME))); Map configMap = sqlMetric.getConfigMap(); if (MapUtils.isNotEmpty(configMap)) { for(ConfigItem configItem : configMap.values()) { diff --git a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java index 81c02f6ad..5ec6dc7b4 100644 --- a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java +++ b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java @@ -23,7 +23,7 @@ import io.datavines.engine.api.component.Component; import io.datavines.engine.api.env.RuntimeEnvironment; import io.datavines.engine.core.utils.JsonUtils; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,8 +68,8 @@ private DataVinesJobConfig load() { } private RuntimeEnvironment createRuntimeEnvironment() { - RuntimeEnvironment env = PluginLoader - .getPluginLoader(RuntimeEnvironment.class) + RuntimeEnvironment env = PluginDiscovery.getMultiKeyPluginDiscovery(RuntimeEnvironment.class, RuntimeEnvironment::getPluginNames) + .getNewPlugin(envConfig.getEngine()); Config config = new Config(envConfig.getConfig()); config.put(TYPE, envConfig.getType()); @@ -86,8 +86,8 @@ public List getSourcePlugins() { List sourcePluginList = new ArrayList<>(); config.getSourceParameters().forEach(sourceConfig -> { String pluginName = String.format("%s-%s-%s-source", envConfig.getEngine(), envConfig.getType(), sourceConfig.getPlugin()); - Component component = PluginLoader - .getPluginLoader(Component.class) + Component component = PluginDiscovery.getMultiKeyPluginDiscovery(Component.class, Component::getPluginNames) + .getNewPlugin(pluginName); sourceConfig.getConfig().put(PLUGIN_TYPE, sourceConfig.getType()); component.setConfig(new Config(sourceConfig.getConfig())); @@ -100,8 +100,8 @@ public List getSinkPlugins() { List sinkPluginList = new ArrayList<>(); config.getSinkParameters().forEach(sinkConfig -> { String pluginName = String.format("%s-%s-%s-sink", envConfig.getEngine(), envConfig.getType(), sinkConfig.getPlugin()); - Component component = PluginLoader - .getPluginLoader(Component.class) + Component component = PluginDiscovery.getMultiKeyPluginDiscovery(Component.class, Component::getPluginNames) + .getNewPlugin(pluginName); sinkConfig.getConfig().put(PLUGIN_TYPE, sinkConfig.getType()); component.setConfig(new Config(sinkConfig.getConfig())); @@ -114,8 +114,8 @@ public List getTransformPlugins() { List transformPluginList = new ArrayList<>(); config.getTransformParameters().forEach(transformConfig -> { String pluginName = String.format("%s-%s-%s-transform", envConfig.getEngine(), envConfig.getType(), transformConfig.getPlugin()); - Component component = PluginLoader - .getPluginLoader(Component.class) + Component component = PluginDiscovery.getMultiKeyPluginDiscovery(Component.class, Component::getPluginNames) + .getNewPlugin(pluginName); transformConfig.getConfig().put(PLUGIN_TYPE, transformConfig.getType()); component.setConfig(new Config(transformConfig.getConfig())); diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java index a5b5aa392..fe7892436 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java @@ -90,4 +90,9 @@ public void stop() { throw new DataVinesException("Failed to stop Flink environment", e); } } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment deleted file mode 100644 index 298054c2f..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment +++ /dev/null @@ -1 +0,0 @@ -flink=io.datavines.engine.flink.api.FlinkRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment new file mode 100644 index 000000000..3ae30b0ad --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment @@ -0,0 +1 @@ +io.datavines.engine.flink.api.FlinkRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java index 7937463b5..5e747510b 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/BaseFlinkConfigurationBuilder.java @@ -32,7 +32,7 @@ import io.datavines.engine.config.BaseJobConfigurationBuilder; import io.datavines.metric.api.ExpectedValue; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -72,8 +72,8 @@ protected List getSourceConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); if (jobExecutionParameter.getConnectorParameter() != null) { String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); if (sqlMetric.isCustomSql()) { List tables = SqlUtils.extractTablesFromSelect(metricInputParameter.get(ACTUAL_AGGREGATE_SQL)); @@ -83,8 +83,8 @@ protected List getSourceConfigs() throws DataVinesException { Map table2OutputTable = new HashMap<>(); for (String table : tables) { ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(connectorParameter.getType()); table = table.replaceAll(connectorFactory.getDialect().getQuoteIdentifier(), ""); String[] tableArray = table.split("\\."); @@ -148,8 +148,8 @@ protected List getSourceConfigs() throws DataVinesException { } } else { ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(connectorParameter.getType()); SourceConfig sourceConfig = new SourceConfig(); @@ -207,8 +207,8 @@ protected List getSourceConfigs() throws DataVinesException { metricInputParameter.put(SCHEMA2, (String)connectorParameter2.getParameters().get(SCHEMA)); } - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(connectorParameter2.getType()); connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE2)); @@ -243,8 +243,7 @@ protected List getSourceConfigs() throws DataVinesException { String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) .getNewPlugin(expectedType); if (expectedValue.isNeedDefaultDatasource() && !isAddValidateResultDataSource) { @@ -293,8 +292,8 @@ protected SinkConfig getErrorSinkConfig(Map inputParameter) { errorDataSinkConfig.setType(SinkType.ERROR_DATA.getDescription()); Map connectorParameterMap = new HashMap<>(JSONUtils.toMap(jobExecutionInfo.getErrorDataStorageParameter(),String.class, Object.class)); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(jobExecutionInfo.getErrorDataStorageType()); if (connectorFactory == null) { diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkDataProfileMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkDataProfileMetricBuilder.java index 9a39c7702..0b3ea7137 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkDataProfileMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkDataProfileMetricBuilder.java @@ -54,4 +54,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink_data_profile"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableAccuracyMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableAccuracyMetricBuilder.java index ff3a1f233..ce3ce5172 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableAccuracyMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableAccuracyMetricBuilder.java @@ -24,7 +24,7 @@ import io.datavines.common.utils.StringUtils; import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -72,7 +72,7 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) .getNewPlugin(expectedType); //get the actual value storage parameter @@ -107,4 +107,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink_multi_table_accuracy"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableValueComparisonMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableValueComparisonMetricBuilder.java index 087ca60c0..7aa03c070 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableValueComparisonMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkMultiTableValueComparisonMetricBuilder.java @@ -21,7 +21,7 @@ import io.datavines.common.exception.DataVinesException; import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -39,8 +39,8 @@ public void buildTransformConfigs() { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); @@ -72,4 +72,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink_multi_table_value_comparison"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java index d9f844a39..e992ef566 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/java/io/datavines/engine/flink/config/FlinkSingleTableConfigurationBuilder.java @@ -25,7 +25,7 @@ import io.datavines.common.utils.StringUtils; import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -48,8 +48,8 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); metricInputParameter.put(METRIC_UNIQUE_KEY, metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) + .getNewPlugin(expectedType); metricInputParameter.put(UNIQUE_CODE, StringUtils.wrapperSingleQuotes(generateUniqueCode(metricInputParameter))); @@ -85,4 +85,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink_single_table"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder deleted file mode 100644 index 3273d66bb..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder +++ /dev/null @@ -1,4 +0,0 @@ -flink_single_table=io.datavines.engine.flink.config.FlinkSingleTableConfigurationBuilder -flink_data_profile=io.datavines.engine.flink.config.FlinkDataProfileMetricBuilder -flink_multi_table_accuracy=io.datavines.engine.flink.config.FlinkMultiTableAccuracyMetricBuilder -flink_multi_table_value_comparison=io.datavines.engine.flink.config.FlinkMultiTableValueComparisonMetricBuilder diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder new file mode 100644 index 000000000..da2ad9a6a --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder @@ -0,0 +1,4 @@ +io.datavines.engine.flink.config.FlinkDataProfileMetricBuilder +io.datavines.engine.flink.config.FlinkMultiTableAccuracyMetricBuilder +io.datavines.engine.flink.config.FlinkMultiTableValueComparisonMetricBuilder +io.datavines.engine.flink.config.FlinkSingleTableConfigurationBuilder diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java index 7ba1d1fb4..457a8b46c 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java @@ -164,4 +164,9 @@ private void checkTableNotExistAndCreate() throws Exception { conn.prepareStatement(createTableSql).execute(); } } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink-batch-jdbc-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java index a38649459..b7d744415 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java @@ -17,4 +17,9 @@ package io.datavines.engine.flink.jdbc.sink; public class MySQLSink extends JdbcSink { + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink-batch-mysql-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java index ceb04e998..28bc8bfb3 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java @@ -176,4 +176,9 @@ public CheckResult checkConfig() { @Override public void prepare(RuntimeEnvironment env) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink-batch-jdbc-source"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index 93ec94a30..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,3 +0,0 @@ -flink-batch-jdbc-sink=io.datavines.engine.flink.jdbc.sink.JdbcSink -flink-batch-jdbc-source=io.datavines.engine.flink.jdbc.source.JdbcSource -flink-batch-mysql-sink=io.datavines.engine.flink.jdbc.sink.MySQLSink \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..05b69bc51 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1,3 @@ +io.datavines.engine.flink.jdbc.sink.JdbcSink +io.datavines.engine.flink.jdbc.sink.MySQLSink +io.datavines.engine.flink.jdbc.source.JdbcSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/java/io/datavines/engine/flink/executor/FlinkEngineExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/java/io/datavines/engine/flink/executor/FlinkEngineExecutor.java index 83a1925ce..42d9fd25a 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/java/io/datavines/engine/flink/executor/FlinkEngineExecutor.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/java/io/datavines/engine/flink/executor/FlinkEngineExecutor.java @@ -179,4 +179,9 @@ public void logHandle(List logs) { public boolean isCancel() { return this.cancel; } + + @Override + public String getPluginName() { + return "flink"; + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor deleted file mode 100644 index 931d94a8b..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor +++ /dev/null @@ -1 +0,0 @@ -flink=io.datavines.engine.flink.executor.FlinkEngineExecutor diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor new file mode 100644 index 000000000..63412a656 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor @@ -0,0 +1 @@ +io.datavines.engine.flink.executor.FlinkEngineExecutor diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java index bd292ca19..282b3c08d 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java @@ -85,4 +85,9 @@ public void prepare(RuntimeEnvironment env) throws Exception { public List getOutputColumns() { return columns; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("flink-batch-sql-transform"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index c4acad434..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1 +0,0 @@ -flink-batch-sql-transform=io.datavines.engine.flink.transform.sql.SqlTransform \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..ad7df8f28 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1 @@ +io.datavines.engine.flink.transform.sql.SqlTransform diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/java/io/datavines/engine/livy/executor/LivyEngineExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/java/io/datavines/engine/livy/executor/LivyEngineExecutor.java index de366cb76..7833e5708 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/java/io/datavines/engine/livy/executor/LivyEngineExecutor.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/java/io/datavines/engine/livy/executor/LivyEngineExecutor.java @@ -233,4 +233,9 @@ private void checkSet(SparkParameters param, LivySparkParameters parameters) { parameters.setProxyUser(proxyUser); } } + + @Override + public String getPluginName() { + return "livy"; + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor deleted file mode 100644 index 3b3b3d16c..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor +++ /dev/null @@ -1 +0,0 @@ -livy=io.datavines.engine.livy.executor.LivyEngineExecutor \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor new file mode 100644 index 000000000..9a0d545ab --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor @@ -0,0 +1 @@ +io.datavines.engine.livy.executor.LivyEngineExecutor diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java index d157868e4..8de944759 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java @@ -95,4 +95,9 @@ public void close() throws Exception { stop = true; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java index 860819ed6..be7a8401f 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java @@ -19,7 +19,7 @@ import io.datavines.common.config.Config; import io.datavines.connector.api.ConnectorFactory; import io.datavines.engine.local.api.utils.LoggerFactory; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import java.sql.Connection; @@ -45,8 +45,8 @@ public ConnectionHolder(Connection connection, Config config) { public Connection getConnection() throws SQLException { if (connection == null || connection.isClosed() || !connection.isValid(10)) { - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(config.getString(SRC_CONNECTOR_TYPE)); connection = connectorFactory.getDataSourceClient().getConnection(config.configMap(), logger); } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment deleted file mode 100644 index 328dbd411..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment +++ /dev/null @@ -1 +0,0 @@ -local=io.datavines.engine.local.api.LocalRuntimeEnvironment \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment new file mode 100644 index 000000000..187696ce3 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment @@ -0,0 +1 @@ +io.datavines.engine.local.api.LocalRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java index 07c9b8a4d..adeeb2f5f 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/BaseLocalConfigurationBuilder.java @@ -30,7 +30,7 @@ import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -64,13 +64,13 @@ protected List getSourceConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); if (jobExecutionParameter.getConnectorParameter() != null) { String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); if (sqlMetric.isCustomSql()) { ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(connectorParameter.getType()); List tables = SqlUtils.extractTablesFromSelect(metricInputParameter.get(ACTUAL_AGGREGATE_SQL)); @@ -136,8 +136,8 @@ protected List getSourceConfigs() throws DataVinesException { sourceConnectorSet.add(connectorUuid); } else { ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(connectorParameter.getType()); Map connectorParameterMap = new HashMap<>(connectorParameter.getParameters()); @@ -197,8 +197,8 @@ protected List getSourceConfigs() throws DataVinesException { connectorParameterMap.putAll(metricInputParameter); connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE2)); connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE2)); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(connectorParameter2.getType()); if (connectorParameter2.getParameters().get(SCHEMA) != null) { @@ -241,8 +241,8 @@ protected List getSourceConfigs() throws DataVinesException { String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) + .getNewPlugin(expectedType); if (expectedValue.isNeedDefaultDatasource() && !isAddValidateResultDataSource) { @@ -267,8 +267,8 @@ public void buildTransformConfigs() { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); @@ -304,8 +304,8 @@ public void buildTransformConfigs() { // generate expected value transform sql String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) + .getNewPlugin(expectedType); expectedValue.prepare(metricInputParameter); diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalDataProfileMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalDataProfileMetricBuilder.java index 3ee63618b..db0c7020e 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalDataProfileMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalDataProfileMetricBuilder.java @@ -51,4 +51,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local_data_profile"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableAccuracyMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableAccuracyMetricBuilder.java index c37357e55..5d649dd6c 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableAccuracyMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableAccuracyMetricBuilder.java @@ -29,7 +29,7 @@ import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -79,7 +79,7 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) .getNewPlugin(expectedType); String validateResultSinkSql = SinkSqlBuilder.getJobExecutionResultSql() @@ -107,8 +107,8 @@ public void buildSinkConfigs() throws DataVinesException { errorDataSinkConfig.setType(SinkType.ERROR_DATA.getDescription()); Map connectorParameterMap = new HashMap<>(JSONUtils.toMap(jobExecutionInfo.getErrorDataStorageParameter(),String.class, Object.class)); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(jobExecutionInfo.getErrorDataStorageType()); if (connectorFactory == null) { @@ -127,8 +127,8 @@ public void buildSinkConfigs() throws DataVinesException { connectorParameterMap.put(METRIC_NAME, metricInputParameter.get(METRIC_NAME)); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); if (sqlMetric.getInvalidateItems(metricInputParameter) != null) { @@ -151,4 +151,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local_multi_table_accuracy"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableValueComparisonMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableValueComparisonMetricBuilder.java index d29fdf8bc..d50e8be89 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableValueComparisonMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalMultiTableValueComparisonMetricBuilder.java @@ -29,7 +29,7 @@ import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; @@ -53,8 +53,8 @@ public void buildTransformConfigs() { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); metricInputParameter.put(METRIC_UNIQUE_KEY, metricUniqueKey); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); @@ -112,7 +112,7 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); metricInputParameter.put(METRIC_UNIQUE_KEY, metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) .getNewPlugin(expectedType); String validateResultSinkSql = SinkSqlBuilder.getJobExecutionResultSql() @@ -137,4 +137,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local_multi_table_value_comparison"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalSingleTableMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalSingleTableMetricBuilder.java index 3933e13bb..4b6ed954f 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalSingleTableMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/java/io/datavines/engine/local/config/LocalSingleTableMetricBuilder.java @@ -28,7 +28,7 @@ import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -52,7 +52,7 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); metricInputParameter.put(METRIC_UNIQUE_KEY, metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) .getNewPlugin(expectedType); String validateResultSinkSql = SinkSqlBuilder.getJobExecutionResultSql() @@ -79,8 +79,8 @@ public void buildSinkConfigs() throws DataVinesException { errorDataSinkConfig.setType(SinkType.ERROR_DATA.getDescription()); Map connectorParameterMap = new HashMap<>(JSONUtils.toMap(jobExecutionInfo.getErrorDataStorageParameter(),String.class, Object.class)); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getNewPlugin(jobExecutionInfo.getErrorDataStorageType()); if (connectorFactory == null) { @@ -99,8 +99,8 @@ public void buildSinkConfigs() throws DataVinesException { connectorParameterMap.put(METRIC_NAME, metricInputParameter.get(METRIC_NAME)); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); if (sqlMetric.getInvalidateItems(metricInputParameter) != null) { @@ -123,4 +123,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local_single_table"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder deleted file mode 100644 index b41116c0e..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder +++ /dev/null @@ -1,4 +0,0 @@ -local_single_table=io.datavines.engine.local.config.LocalSingleTableMetricBuilder -local_data_profile=io.datavines.engine.local.config.LocalDataProfileMetricBuilder -local_multi_table_value_comparison=io.datavines.engine.local.config.LocalMultiTableValueComparisonMetricBuilder -local_multi_table_accuracy=io.datavines.engine.local.config.LocalMultiTableAccuracyMetricBuilder \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder new file mode 100644 index 000000000..1188f3086 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder @@ -0,0 +1,4 @@ +io.datavines.engine.local.config.LocalDataProfileMetricBuilder +io.datavines.engine.local.config.LocalMultiTableAccuracyMetricBuilder +io.datavines.engine.local.config.LocalMultiTableValueComparisonMetricBuilder +io.datavines.engine.local.config.LocalSingleTableMetricBuilder diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java index 2a7a987ba..18db31061 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java @@ -31,7 +31,7 @@ import io.datavines.engine.local.api.utils.FileUtils; import io.datavines.engine.local.api.utils.LoggerFactory; import io.datavines.connector.api.utils.SqlUtils; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; @@ -130,7 +130,7 @@ private void sinkErrorData(LocalRuntimeEnvironment env) throws SQLException{ } String srcConnectorType = config.getString(SRC_CONNECTOR_TYPE); - TypeConverter typeConverter = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType).getTypeConverter(); + TypeConverter typeConverter = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(srcConnectorType).getTypeConverter(); if (count > 0) { count = Math.min(count, 10000); //根据行数进行分页查询。分批写到文件里面 @@ -157,4 +157,9 @@ private void sinkErrorData(LocalRuntimeEnvironment env) throws SQLException{ } } } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local-file-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java index d9e322db3..0333c500d 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java @@ -140,4 +140,9 @@ private String buildCreateTableSql(String tableName, String schema, String fileN public boolean checkTableExist() { return true; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local-file-source"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index 2aa349162..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,2 +0,0 @@ -local-file-sink=io.datavines.engine.local.connector.LocalFileSink -local-file-source=io.datavines.engine.local.connector.LocalFileSource \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..57cfb3d7a --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1,2 @@ +io.datavines.engine.local.connector.LocalFileSink +io.datavines.engine.local.connector.LocalFileSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java index 4bb2c128c..c23af40e8 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java @@ -132,4 +132,9 @@ protected String getProfileValueTableSql() { return ""; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local-batch-jdbc-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java index 58daf4aed..840fd7ffc 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java @@ -27,7 +27,7 @@ import io.datavines.engine.local.api.LocalSource; import io.datavines.engine.local.api.entity.ConnectionHolder; import io.datavines.engine.local.api.utils.LoggerFactory; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import java.util.ArrayList; @@ -98,7 +98,7 @@ public boolean checkTableExist() { int retryTimes = 3; while (retryTimes > 0) { if (connectionHolder != null) { - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) .getOrCreatePlugin(config.getString(SRC_CONNECTOR_TYPE)); JdbcOptions jdbcOptions = new JdbcOptions(); jdbcOptions.setDatabaseName(config.getString(DATABASE)); @@ -117,4 +117,9 @@ public boolean checkTableExist() { return false; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local-batch-jdbc-source"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/MySQLSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/MySQLSink.java index 46230be58..310ed5104 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/MySQLSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/MySQLSink.java @@ -75,4 +75,9 @@ protected String getProfileValueTableSql() { " UNIQUE KEY `dv_entity_definition_un` (`entity_uuid`)\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n"; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local-batch-mysql-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java index 11e65caa0..5bf1d81fa 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java @@ -31,7 +31,7 @@ import io.datavines.connector.api.entity.ResultList; import io.datavines.engine.local.api.utils.LoggerFactory; import io.datavines.connector.api.utils.SqlUtils; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import java.math.BigDecimal; @@ -106,7 +106,7 @@ private void sinkErrorDataToDataSource() { String srcConnectorType = config.getString(SRC_CONNECTOR_TYPE); - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(srcConnectorType); Dialect dialect = connectorFactory.getDialect(); if (!checkTableExist(getConnectionHolder().getConnection(), dialect.quoteIdentifier(targetDatabase)+"."+dialect.quoteIdentifier(targetTable), dialect)) { @@ -139,8 +139,8 @@ private void sinkErrorData() { sourceConnectionStatement = env.getSourceConnection().getConnection().createStatement(); env.setCurrentStatement(sourceConnectionStatement); String srcConnectorType = config.getString(SRC_CONNECTOR_TYPE); - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType); - ConnectorFactory errorDataConnectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(config.getString(ERROR_DATA_CONNECTOR_TYPE)); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(srcConnectorType); + ConnectorFactory errorDataConnectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(config.getString(ERROR_DATA_CONNECTOR_TYPE)); int count = 0; //执行统计行数语句 countResultSet = sourceConnectionStatement.executeQuery(connectorFactory.getDialect().getCountQuery(sourceTable)); @@ -340,7 +340,7 @@ private void createTable(TypeConverter typeConverter, Dialect dialect, private List getTableSchema(Statement statement, Config config, TypeConverter typeConverter) { if (statement != null) { - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class) + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) .getOrCreatePlugin(config.getString(SRC_CONNECTOR_TYPE)); String tableName = config.getString(INVALIDATE_ITEMS_TABLE); diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index 8a07c7adf..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,3 +0,0 @@ -local-batch-mysql-sink=io.datavines.engine.local.connector.MySQLSink -local-batch-jdbc-sink=io.datavines.engine.local.connector.BaseJdbcSink -local-batch-jdbc-source=io.datavines.engine.local.connector.BaseJdbcSource \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..86a5e2a07 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1,3 @@ +io.datavines.engine.local.connector.BaseJdbcSink +io.datavines.engine.local.connector.BaseJdbcSource +io.datavines.engine.local.connector.MySQLSink diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/executor/LocalEngineExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/executor/LocalEngineExecutor.java index 63555c866..dee1b02f4 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/executor/LocalEngineExecutor.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/executor/LocalEngineExecutor.java @@ -74,4 +74,9 @@ public JobExecutionRequest getTaskRequest() { protected String buildCommand() { return null; } + + @Override + public String getPluginName() { + return "local"; + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor deleted file mode 100644 index 263f85786..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor +++ /dev/null @@ -1 +0,0 @@ -local=io.datavines.engine.local.executor.LocalEngineExecutor \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor new file mode 100644 index 000000000..dc46a7e4c --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor @@ -0,0 +1 @@ +io.datavines.engine.local.executor.LocalEngineExecutor diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java index 91d5260f2..fa017e527 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java @@ -122,4 +122,9 @@ public ResultList process(LocalRuntimeEnvironment env) throws DataVinesException return resultList; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Collections.singletonList("local-batch-sql-transform"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index ef9c2abd1..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1 +0,0 @@ -local-batch-sql-transform=io.datavines.engine.local.transform.sql.SqlTransform \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..70dc5c0a3 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1 @@ +io.datavines.engine.local.transform.sql.SqlTransform diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java index f5171cb15..5df7f362f 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java @@ -114,4 +114,9 @@ public Execution getExecution() { } return execution; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy", "spark"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment deleted file mode 100644 index 3355a6606..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/plugins/io.datavines.engine.api.env.RuntimeEnvironment +++ /dev/null @@ -1,2 +0,0 @@ -spark=io.datavines.engine.spark.api.SparkRuntimeEnvironment -livy=io.datavines.engine.spark.api.SparkRuntimeEnvironment \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment new file mode 100644 index 000000000..d4969b187 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment @@ -0,0 +1 @@ +io.datavines.engine.spark.api.SparkRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/BaseSparkConfigurationBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/BaseSparkConfigurationBuilder.java index f812b3c38..80df939ad 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/BaseSparkConfigurationBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/BaseSparkConfigurationBuilder.java @@ -1,356 +1,356 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.engine.spark.config; - -import io.datavines.common.config.EnvConfig; -import io.datavines.common.config.SinkConfig; -import io.datavines.common.config.SourceConfig; -import io.datavines.common.config.enums.SinkType; -import io.datavines.common.config.enums.SourceType; -import io.datavines.common.entity.ConnectorParameter; -import io.datavines.common.entity.job.BaseJobParameter; -import io.datavines.common.exception.DataVinesException; -import io.datavines.common.utils.JSONUtils; -import io.datavines.common.utils.StringUtils; -import io.datavines.connector.api.ConnectorFactory; -import io.datavines.connector.api.utils.SqlUtils; -import io.datavines.engine.common.utils.ParserUtils; -import io.datavines.engine.config.BaseJobConfigurationBuilder; -import io.datavines.metric.api.ExpectedValue; -import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.*; -import java.util.stream.Collectors; - -import static io.datavines.common.CommonConstants.*; -import static io.datavines.common.ConfigConstants.*; -import static io.datavines.common.ConfigConstants.TABLE; - -@Slf4j -public abstract class BaseSparkConfigurationBuilder extends BaseJobConfigurationBuilder { - - @Override - protected EnvConfig getEnvConfig() { - EnvConfig envConfig = new EnvConfig(); - envConfig.setEngine(jobExecutionInfo.getEngineType()); - Map configMap = envConfig.getConfig(); - if (configMap == null) { - configMap = new HashMap<>(); - } - - ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - String srcConnectorType = ""; - boolean isEnableSparkHiveSupport = false; - if (connectorParameter != null) { - srcConnectorType = connectorParameter.getType(); - if (connectorParameter.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT) != null) { - isEnableSparkHiveSupport = Boolean.parseBoolean((String) connectorParameter.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT)); - } - } - - ConnectorParameter connectorParameter2 = jobExecutionParameter.getConnectorParameter2(); - - String srcConnectorType2 = ""; - boolean isEnableSparkHiveSupport2 = false; - if (connectorParameter2 != null) { - srcConnectorType2 = connectorParameter2.getType(); - if (connectorParameter2.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT) != null) { - isEnableSparkHiveSupport2 = Boolean.parseBoolean((String) connectorParameter2.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT)); - } - } - - if ((HIVE.equalsIgnoreCase(srcConnectorType) && isEnableSparkHiveSupport) || - (HIVE.equalsIgnoreCase(srcConnectorType2) && isEnableSparkHiveSupport2)) { - configMap.put(ENABLE_SPARK_HIVE_SUPPORT, Boolean.TRUE); - } - - envConfig.setConfig(configMap); - return envConfig; - } - - @Override - protected List getSourceConfigs() throws DataVinesException { - List sourceConfigs = new ArrayList<>(); - List metricJobParameterList = jobExecutionParameter.getMetricParameterList(); - boolean isAddValidateResultDataSource = false; - if (CollectionUtils.isNotEmpty(metricJobParameterList)) { - Set sourceConnectorSet = new HashSet<>(); - Set targetConnectorSet = new HashSet<>(); - for (BaseJobParameter parameter : metricJobParameterList) { - String metricUniqueKey = getMetricUniqueKey(parameter); - Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); - if (jobExecutionParameter.getConnectorParameter() != null) { - String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) - .getNewPlugin(metricType); - if (sqlMetric.isCustomSql()) { - List tables = SqlUtils.extractTablesFromSelect(metricInputParameter.get(ACTUAL_AGGREGATE_SQL)); - if (CollectionUtils.isEmpty(tables)) { - throw new DataVinesException("custom sql must have table"); - } - Map table2OutputTable = new HashMap<>(); - for (String table : tables) { - ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) - .getNewPlugin(connectorParameter.getType()); - table = table.replaceAll(connectorFactory.getDialect().getQuoteIdentifier(), ""); - String[] tableArray = table.split("\\."); - if (tableArray.length == 1) { - metricInputParameter.put(TABLE, tableArray[0]); - } else { - metricInputParameter.put(DATABASE, tableArray[0]); - metricInputParameter.put(TABLE, tableArray[1]); - } - - SourceConfig sourceConfig = new SourceConfig(); - - Map connectorParameterMap = new HashMap<>(connectorParameter.getParameters()); - connectorParameterMap.putAll(metricInputParameter); - - if (connectorParameter.getParameters().get(SCHEMA) != null) { - metricInputParameter.put(SCHEMA, (String)connectorParameter.getParameters().get(SCHEMA)); - } - - metricInputParameter.put(DATABASE_NAME, metricInputParameter.get(DATABASE)); - metricInputParameter.put(TABLE_NAME, metricInputParameter.get(TABLE)); - metricInputParameter.put(COLUMN_NAME, StringUtils.isEmpty(metricInputParameter.get(COLUMN)) ? "--" : metricInputParameter.get(COLUMN)); - - connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE)); - connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE)); - connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); - connectorParameterMap.put(PASSWORD, ParserUtils.encode((String)connectorParameterMap.get(PASSWORD))); - - String outputTable = getOutputTable(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE)); - String tableAlias = getTableAlias(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE), "1"); - connectorParameterMap.put(OUTPUT_TABLE, outputTable); - connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); - connectorParameterMap.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); - table2OutputTable.put(table, outputTable); - - metricInputParameter.put(TABLE, outputTable); - metricInputParameter.put(TABLE_ALIAS, tableAlias); - metricInputParameter.put(COLUMN, metricInputParameter.get(COLUMN)); - metricInputParameter.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); - - String connectorUUID = connectorFactory.getConnectorParameterConverter().getConnectorUUID(connectorParameterMap); - - if (sourceConnectorSet.contains(connectorUUID)) { - continue; - } - - sourceConfig.setPlugin(connectorFactory.getCategory()); - sourceConfig.setConfig(connectorParameterMap); - sourceConfig.setType(SourceType.SOURCE.getDescription()); - sourceConfigs.add(sourceConfig); - sourceConnectorSet.add(connectorUUID); - } - - if (StringUtils.isNotEmpty(metricInputParameter.get(ACTUAL_AGGREGATE_SQL))) { - String sql = metricInputParameter.get(ACTUAL_AGGREGATE_SQL); - for (Map.Entry entry : table2OutputTable.entrySet()) { - sql = sql.replaceAll(entry.getKey(), entry.getValue()); - } - - metricInputParameter.put(ACTUAL_AGGREGATE_SQL, sql); - } - } else { - ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); - SourceConfig sourceConfig = new SourceConfig(); - - Map connectorParameterMap = new HashMap<>(connectorParameter.getParameters()); - connectorParameterMap.putAll(metricInputParameter); - - if (connectorParameter.getParameters().get(SCHEMA) != null) { - metricInputParameter.put(SCHEMA, (String)connectorParameter.getParameters().get(SCHEMA)); - } - - metricInputParameter.put(DATABASE_NAME,metricInputParameter.get(DATABASE)); - metricInputParameter.put(TABLE_NAME,metricInputParameter.get(TABLE)); - metricInputParameter.put(COLUMN_NAME,metricInputParameter.get(COLUMN)); - - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) - .getNewPlugin(connectorParameter.getType()); - - connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE)); - connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE)); - connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); - connectorParameterMap.put(PASSWORD, ParserUtils.encode((String)connectorParameterMap.get(PASSWORD))); - - String outputTable = getOutputTable(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE)); - String tableAlias = getTableAlias(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE), "1"); - connectorParameterMap.put(OUTPUT_TABLE, outputTable); - connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); - connectorParameterMap.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); - - metricInputParameter.put(TABLE, outputTable); - metricInputParameter.put(TABLE_ALIAS, tableAlias); - metricInputParameter.put(COLUMN, metricInputParameter.get(COLUMN)); - metricInputParameter.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); - metricInputParameter.put(ENGINE_TYPE, SPARK); - - String connectorUUID = connectorFactory.getConnectorParameterConverter().getConnectorUUID(connectorParameterMap); - - if (sourceConnectorSet.contains(connectorUUID)) { - continue; - } - - sourceConfig.setPlugin(connectorFactory.getCategory()); - sourceConfig.setConfig(connectorParameterMap); - sourceConfig.setType(SourceType.SOURCE.getDescription()); - sourceConfigs.add(sourceConfig); - sourceConnectorSet.add(connectorUUID); - } - } - - if (jobExecutionParameter.getConnectorParameter2() != null - && jobExecutionParameter.getConnectorParameter2().getParameters() !=null) { - ConnectorParameter connectorParameter2 = jobExecutionParameter.getConnectorParameter2(); - SourceConfig sourceConfig = new SourceConfig(); - - Map connectorParameterMap = new HashMap<>(connectorParameter2.getParameters()); - connectorParameterMap.putAll(metricInputParameter); - - if (connectorParameter2.getParameters().get(SCHEMA) != null) { - metricInputParameter.put(SCHEMA2, (String)connectorParameter2.getParameters().get(SCHEMA)); - } - - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) - .getNewPlugin(connectorParameter2.getType()); - - connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE2)); - connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE2)); - connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); - connectorParameterMap.put(PASSWORD, ParserUtils.encode((String)connectorParameterMap.get(PASSWORD))); - - String outputTable = getOutputTable(metricInputParameter.get(DATABASE2), - metricInputParameter.get(SCHEMA2), - metricInputParameter.get(TABLE2)) + "_2"; - String tableAlias2 = getTableAlias(metricInputParameter.get(DATABASE2), - metricInputParameter.get(SCHEMA2), - metricInputParameter.get(TABLE2) ,"2"); - connectorParameterMap.put(OUTPUT_TABLE, outputTable); - connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); - metricInputParameter.put(TABLE2, outputTable); - metricInputParameter.put(TABLE2_ALIAS, tableAlias2); - - String connectorUUID = connectorFactory.getConnectorParameterConverter().getConnectorUUID(connectorParameterMap); - if (targetConnectorSet.contains(connectorUUID)) { - continue; - } - - sourceConfig.setPlugin(connectorFactory.getCategory()); - sourceConfig.setConfig(connectorParameterMap); - sourceConfig.setType(SourceType.TARGET.getDescription()); - sourceConfigs.add(sourceConfig); - targetConnectorSet.add(connectorUUID); - } - - metricInputParameter.put("actual_value", "actual_value_" + metricUniqueKey); - - String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) - .getNewPlugin(expectedType); - - if (expectedValue.isNeedDefaultDatasource() && !isAddValidateResultDataSource) { - sourceConfigs.add(getValidateResultDataSourceConfig()); - isAddValidateResultDataSource = true; - } - - metric2InputParameter.put(metricUniqueKey, metricInputParameter); - } - } - - return sourceConfigs; - } - - protected SinkConfig getErrorSinkConfig(Map inputParameter) { - SinkConfig errorDataSinkConfig = null; - if (StringUtils.isNotEmpty(jobExecutionInfo.getErrorDataStorageType()) - && StringUtils.isNotEmpty(jobExecutionInfo.getErrorDataStorageParameter())) { - errorDataSinkConfig = new SinkConfig(); - errorDataSinkConfig.setType(SinkType.ERROR_DATA.getDescription()); - - Map connectorParameterMap = new HashMap<>(JSONUtils.toMap(jobExecutionInfo.getErrorDataStorageParameter(),String.class, Object.class)); - ConnectorFactory connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) - .getNewPlugin(jobExecutionInfo.getErrorDataStorageType()); - - if (connectorFactory == null) { - return null; - } - - String errorDataOutputToDataSourceDatabase = String.valueOf(connectorParameterMap.get(ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE)); - - if (connectorParameterMap.get(ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE) == null) { - connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); - } else { - List sourceConfigs = getSourceConfigs() - .stream().filter(x->SourceType.SOURCE.getDescription().equalsIgnoreCase(x.getType())).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(sourceConfigs)) { - return null; - } - - connectorParameterMap = sourceConfigs.get(0).getConfig(); - connectorParameterMap.put(DATABASE, errorDataOutputToDataSourceDatabase); - } - - errorDataSinkConfig.setPlugin(connectorFactory.getCategory()); - connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); - connectorParameterMap.put(ERROR_DATA_FILE_NAME, jobExecutionInfo.getErrorDataFileName()); - connectorParameterMap.put(TABLE, jobExecutionInfo.getErrorDataFileName()); - connectorParameterMap.put(SQL, "SELECT * FROM "+ inputParameter.get(INVALIDATE_ITEMS_TABLE)); - errorDataSinkConfig.setConfig(connectorParameterMap); - } - - return errorDataSinkConfig; - } - - public String getOutputTable(String database, String schema, String table) { - - if (!StringUtils.isEmptyOrNullStr(schema)) { - table = schema + "_" + table; - } - - if (!StringUtils.isEmptyOrNullStr(database)) { - table = database + "_" + table; - } - - return table; - } - - public String getTableAlias(String database, String schema, String table, String order) { - - if (!StringUtils.isEmptyOrNullStr(schema)) { - table = schema + "_" + table; - } - - if (!StringUtils.isEmptyOrNullStr(database)) { - table = database + "_" + table; - } - - return table+ "_" + order; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.engine.spark.config; + +import io.datavines.common.config.EnvConfig; +import io.datavines.common.config.SinkConfig; +import io.datavines.common.config.SourceConfig; +import io.datavines.common.config.enums.SinkType; +import io.datavines.common.config.enums.SourceType; +import io.datavines.common.entity.ConnectorParameter; +import io.datavines.common.entity.job.BaseJobParameter; +import io.datavines.common.exception.DataVinesException; +import io.datavines.common.utils.JSONUtils; +import io.datavines.common.utils.StringUtils; +import io.datavines.connector.api.ConnectorFactory; +import io.datavines.connector.api.utils.SqlUtils; +import io.datavines.engine.common.utils.ParserUtils; +import io.datavines.engine.config.BaseJobConfigurationBuilder; +import io.datavines.metric.api.ExpectedValue; +import io.datavines.metric.api.SqlMetric; +import io.datavines.spi.PluginDiscovery; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.*; +import java.util.stream.Collectors; + +import static io.datavines.common.CommonConstants.*; +import static io.datavines.common.ConfigConstants.*; +import static io.datavines.common.ConfigConstants.TABLE; + +@Slf4j +public abstract class BaseSparkConfigurationBuilder extends BaseJobConfigurationBuilder { + + @Override + protected EnvConfig getEnvConfig() { + EnvConfig envConfig = new EnvConfig(); + envConfig.setEngine(jobExecutionInfo.getEngineType()); + Map configMap = envConfig.getConfig(); + if (configMap == null) { + configMap = new HashMap<>(); + } + + ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); + String srcConnectorType = ""; + boolean isEnableSparkHiveSupport = false; + if (connectorParameter != null) { + srcConnectorType = connectorParameter.getType(); + if (connectorParameter.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT) != null) { + isEnableSparkHiveSupport = Boolean.parseBoolean((String) connectorParameter.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT)); + } + } + + ConnectorParameter connectorParameter2 = jobExecutionParameter.getConnectorParameter2(); + + String srcConnectorType2 = ""; + boolean isEnableSparkHiveSupport2 = false; + if (connectorParameter2 != null) { + srcConnectorType2 = connectorParameter2.getType(); + if (connectorParameter2.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT) != null) { + isEnableSparkHiveSupport2 = Boolean.parseBoolean((String) connectorParameter2.getParameters().get(ENABLE_SPARK_HIVE_SUPPORT)); + } + } + + if ((HIVE.equalsIgnoreCase(srcConnectorType) && isEnableSparkHiveSupport) || + (HIVE.equalsIgnoreCase(srcConnectorType2) && isEnableSparkHiveSupport2)) { + configMap.put(ENABLE_SPARK_HIVE_SUPPORT, Boolean.TRUE); + } + + envConfig.setConfig(configMap); + return envConfig; + } + + @Override + protected List getSourceConfigs() throws DataVinesException { + List sourceConfigs = new ArrayList<>(); + List metricJobParameterList = jobExecutionParameter.getMetricParameterList(); + boolean isAddValidateResultDataSource = false; + if (CollectionUtils.isNotEmpty(metricJobParameterList)) { + Set sourceConnectorSet = new HashSet<>(); + Set targetConnectorSet = new HashSet<>(); + for (BaseJobParameter parameter : metricJobParameterList) { + String metricUniqueKey = getMetricUniqueKey(parameter); + Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); + if (jobExecutionParameter.getConnectorParameter() != null) { + String metricType = parameter.getMetricType(); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + + .getNewPlugin(metricType); + if (sqlMetric.isCustomSql()) { + List tables = SqlUtils.extractTablesFromSelect(metricInputParameter.get(ACTUAL_AGGREGATE_SQL)); + if (CollectionUtils.isEmpty(tables)) { + throw new DataVinesException("custom sql must have table"); + } + Map table2OutputTable = new HashMap<>(); + for (String table : tables) { + ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + + .getNewPlugin(connectorParameter.getType()); + table = table.replaceAll(connectorFactory.getDialect().getQuoteIdentifier(), ""); + String[] tableArray = table.split("\\."); + if (tableArray.length == 1) { + metricInputParameter.put(TABLE, tableArray[0]); + } else { + metricInputParameter.put(DATABASE, tableArray[0]); + metricInputParameter.put(TABLE, tableArray[1]); + } + + SourceConfig sourceConfig = new SourceConfig(); + + Map connectorParameterMap = new HashMap<>(connectorParameter.getParameters()); + connectorParameterMap.putAll(metricInputParameter); + + if (connectorParameter.getParameters().get(SCHEMA) != null) { + metricInputParameter.put(SCHEMA, (String)connectorParameter.getParameters().get(SCHEMA)); + } + + metricInputParameter.put(DATABASE_NAME, metricInputParameter.get(DATABASE)); + metricInputParameter.put(TABLE_NAME, metricInputParameter.get(TABLE)); + metricInputParameter.put(COLUMN_NAME, StringUtils.isEmpty(metricInputParameter.get(COLUMN)) ? "--" : metricInputParameter.get(COLUMN)); + + connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE)); + connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE)); + connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); + connectorParameterMap.put(PASSWORD, ParserUtils.encode((String)connectorParameterMap.get(PASSWORD))); + + String outputTable = getOutputTable(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE)); + String tableAlias = getTableAlias(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE), "1"); + connectorParameterMap.put(OUTPUT_TABLE, outputTable); + connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); + connectorParameterMap.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); + table2OutputTable.put(table, outputTable); + + metricInputParameter.put(TABLE, outputTable); + metricInputParameter.put(TABLE_ALIAS, tableAlias); + metricInputParameter.put(COLUMN, metricInputParameter.get(COLUMN)); + metricInputParameter.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); + + String connectorUUID = connectorFactory.getConnectorParameterConverter().getConnectorUUID(connectorParameterMap); + + if (sourceConnectorSet.contains(connectorUUID)) { + continue; + } + + sourceConfig.setPlugin(connectorFactory.getCategory()); + sourceConfig.setConfig(connectorParameterMap); + sourceConfig.setType(SourceType.SOURCE.getDescription()); + sourceConfigs.add(sourceConfig); + sourceConnectorSet.add(connectorUUID); + } + + if (StringUtils.isNotEmpty(metricInputParameter.get(ACTUAL_AGGREGATE_SQL))) { + String sql = metricInputParameter.get(ACTUAL_AGGREGATE_SQL); + for (Map.Entry entry : table2OutputTable.entrySet()) { + sql = sql.replaceAll(entry.getKey(), entry.getValue()); + } + + metricInputParameter.put(ACTUAL_AGGREGATE_SQL, sql); + } + } else { + ConnectorParameter connectorParameter = jobExecutionParameter.getConnectorParameter(); + SourceConfig sourceConfig = new SourceConfig(); + + Map connectorParameterMap = new HashMap<>(connectorParameter.getParameters()); + connectorParameterMap.putAll(metricInputParameter); + + if (connectorParameter.getParameters().get(SCHEMA) != null) { + metricInputParameter.put(SCHEMA, (String)connectorParameter.getParameters().get(SCHEMA)); + } + + metricInputParameter.put(DATABASE_NAME,metricInputParameter.get(DATABASE)); + metricInputParameter.put(TABLE_NAME,metricInputParameter.get(TABLE)); + metricInputParameter.put(COLUMN_NAME,metricInputParameter.get(COLUMN)); + + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + + .getNewPlugin(connectorParameter.getType()); + + connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE)); + connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE)); + connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); + connectorParameterMap.put(PASSWORD, ParserUtils.encode((String)connectorParameterMap.get(PASSWORD))); + + String outputTable = getOutputTable(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE)); + String tableAlias = getTableAlias(metricInputParameter.get(DATABASE), metricInputParameter.get(SCHEMA), metricInputParameter.get(TABLE), "1"); + connectorParameterMap.put(OUTPUT_TABLE, outputTable); + connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); + connectorParameterMap.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); + + metricInputParameter.put(TABLE, outputTable); + metricInputParameter.put(TABLE_ALIAS, tableAlias); + metricInputParameter.put(COLUMN, metricInputParameter.get(COLUMN)); + metricInputParameter.put(SRC_CONNECTOR_TYPE, connectorParameter.getType()); + metricInputParameter.put(ENGINE_TYPE, SPARK); + + String connectorUUID = connectorFactory.getConnectorParameterConverter().getConnectorUUID(connectorParameterMap); + + if (sourceConnectorSet.contains(connectorUUID)) { + continue; + } + + sourceConfig.setPlugin(connectorFactory.getCategory()); + sourceConfig.setConfig(connectorParameterMap); + sourceConfig.setType(SourceType.SOURCE.getDescription()); + sourceConfigs.add(sourceConfig); + sourceConnectorSet.add(connectorUUID); + } + } + + if (jobExecutionParameter.getConnectorParameter2() != null + && jobExecutionParameter.getConnectorParameter2().getParameters() !=null) { + ConnectorParameter connectorParameter2 = jobExecutionParameter.getConnectorParameter2(); + SourceConfig sourceConfig = new SourceConfig(); + + Map connectorParameterMap = new HashMap<>(connectorParameter2.getParameters()); + connectorParameterMap.putAll(metricInputParameter); + + if (connectorParameter2.getParameters().get(SCHEMA) != null) { + metricInputParameter.put(SCHEMA2, (String)connectorParameter2.getParameters().get(SCHEMA)); + } + + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + + .getNewPlugin(connectorParameter2.getType()); + + connectorParameterMap.put(TABLE, metricInputParameter.get(TABLE2)); + connectorParameterMap.put(DATABASE, metricInputParameter.get(DATABASE2)); + connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); + connectorParameterMap.put(PASSWORD, ParserUtils.encode((String)connectorParameterMap.get(PASSWORD))); + + String outputTable = getOutputTable(metricInputParameter.get(DATABASE2), + metricInputParameter.get(SCHEMA2), + metricInputParameter.get(TABLE2)) + "_2"; + String tableAlias2 = getTableAlias(metricInputParameter.get(DATABASE2), + metricInputParameter.get(SCHEMA2), + metricInputParameter.get(TABLE2) ,"2"); + connectorParameterMap.put(OUTPUT_TABLE, outputTable); + connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); + metricInputParameter.put(TABLE2, outputTable); + metricInputParameter.put(TABLE2_ALIAS, tableAlias2); + + String connectorUUID = connectorFactory.getConnectorParameterConverter().getConnectorUUID(connectorParameterMap); + if (targetConnectorSet.contains(connectorUUID)) { + continue; + } + + sourceConfig.setPlugin(connectorFactory.getCategory()); + sourceConfig.setConfig(connectorParameterMap); + sourceConfig.setType(SourceType.TARGET.getDescription()); + sourceConfigs.add(sourceConfig); + targetConnectorSet.add(connectorUUID); + } + + metricInputParameter.put("actual_value", "actual_value_" + metricUniqueKey); + + String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); + + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) + + .getNewPlugin(expectedType); + + if (expectedValue.isNeedDefaultDatasource() && !isAddValidateResultDataSource) { + sourceConfigs.add(getValidateResultDataSourceConfig()); + isAddValidateResultDataSource = true; + } + + metric2InputParameter.put(metricUniqueKey, metricInputParameter); + } + } + + return sourceConfigs; + } + + protected SinkConfig getErrorSinkConfig(Map inputParameter) { + SinkConfig errorDataSinkConfig = null; + if (StringUtils.isNotEmpty(jobExecutionInfo.getErrorDataStorageType()) + && StringUtils.isNotEmpty(jobExecutionInfo.getErrorDataStorageParameter())) { + errorDataSinkConfig = new SinkConfig(); + errorDataSinkConfig.setType(SinkType.ERROR_DATA.getDescription()); + + Map connectorParameterMap = new HashMap<>(JSONUtils.toMap(jobExecutionInfo.getErrorDataStorageParameter(),String.class, Object.class)); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + + .getNewPlugin(jobExecutionInfo.getErrorDataStorageType()); + + if (connectorFactory == null) { + return null; + } + + String errorDataOutputToDataSourceDatabase = String.valueOf(connectorParameterMap.get(ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE)); + + if (connectorParameterMap.get(ERROR_DATA_OUTPUT_TO_DATASOURCE_DATABASE) == null) { + connectorParameterMap = connectorFactory.getConnectorParameterConverter().converter(connectorParameterMap); + } else { + List sourceConfigs = getSourceConfigs() + .stream().filter(x->SourceType.SOURCE.getDescription().equalsIgnoreCase(x.getType())).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(sourceConfigs)) { + return null; + } + + connectorParameterMap = sourceConfigs.get(0).getConfig(); + connectorParameterMap.put(DATABASE, errorDataOutputToDataSourceDatabase); + } + + errorDataSinkConfig.setPlugin(connectorFactory.getCategory()); + connectorParameterMap.put(DRIVER, connectorFactory.getDialect().getDriver()); + connectorParameterMap.put(ERROR_DATA_FILE_NAME, jobExecutionInfo.getErrorDataFileName()); + connectorParameterMap.put(TABLE, jobExecutionInfo.getErrorDataFileName()); + connectorParameterMap.put(SQL, "SELECT * FROM "+ inputParameter.get(INVALIDATE_ITEMS_TABLE)); + errorDataSinkConfig.setConfig(connectorParameterMap); + } + + return errorDataSinkConfig; + } + + public String getOutputTable(String database, String schema, String table) { + + if (!StringUtils.isEmptyOrNullStr(schema)) { + table = schema + "_" + table; + } + + if (!StringUtils.isEmptyOrNullStr(database)) { + table = database + "_" + table; + } + + return table; + } + + public String getTableAlias(String database, String schema, String table, String order) { + + if (!StringUtils.isEmptyOrNullStr(schema)) { + table = schema + "_" + table; + } + + if (!StringUtils.isEmptyOrNullStr(database)) { + table = database + "_" + table; + } + + return table+ "_" + order; + } +} diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkDataProfileMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkDataProfileMetricBuilder.java index fed62f981..06d268001 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkDataProfileMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkDataProfileMetricBuilder.java @@ -54,4 +54,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy_data_profile", "spark_data_profile"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableAccuracyMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableAccuracyMetricBuilder.java index de0f7ada0..e1e1712b9 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableAccuracyMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableAccuracyMetricBuilder.java @@ -24,7 +24,7 @@ import io.datavines.common.entity.MappingColumn; import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.ExpectedValue; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -74,7 +74,7 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) .getNewPlugin(expectedType); //get the actual value storage parameter @@ -104,4 +104,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy_multi_table_accuracy", "spark_multi_table_accuracy"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableValueComparisonMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableValueComparisonMetricBuilder.java index 5c56f54c7..1345b2f39 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableValueComparisonMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkMultiTableValueComparisonMetricBuilder.java @@ -21,7 +21,7 @@ import io.datavines.common.exception.DataVinesException; import io.datavines.engine.config.MetricParserUtils; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -39,8 +39,8 @@ public void buildTransformConfigs() { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); String metricType = parameter.getMetricType(); - SqlMetric sqlMetric = PluginLoader - .getPluginLoader(SqlMetric.class) + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) + .getNewPlugin(metricType); MetricParserUtils.operateInputParameter(metricInputParameter, sqlMetric, jobExecutionInfo); @@ -72,4 +72,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy_multi_table_value_comparison", "spark_multi_table_value_comparison"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkSingleTableMetricBuilder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkSingleTableMetricBuilder.java index c581e3043..db3e18410 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkSingleTableMetricBuilder.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/java/io/datavines/engine/spark/config/SparkSingleTableMetricBuilder.java @@ -21,7 +21,7 @@ import io.datavines.common.exception.DataVinesException; import io.datavines.common.utils.StringUtils; import io.datavines.metric.api.ExpectedValue; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -45,8 +45,8 @@ public void buildSinkConfigs() throws DataVinesException { Map metricInputParameter = metric2InputParameter.get(metricUniqueKey); metricInputParameter.put(METRIC_UNIQUE_KEY, metricUniqueKey); String expectedType = jobExecutionInfo.getEngineType() + "_" + parameter.getExpectedType(); - ExpectedValue expectedValue = PluginLoader - .getPluginLoader(ExpectedValue.class) + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames) + .getNewPlugin(expectedType); metricInputParameter.put(UNIQUE_CODE, StringUtils.wrapperSingleQuotes(generateUniqueCode(metricInputParameter))); @@ -78,4 +78,9 @@ public void buildSinkConfigs() throws DataVinesException { configuration.setSinkParameters(sinkConfigs); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy_single_table", "spark_single_table"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder deleted file mode 100644 index ebd3b0f5e..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/plugins/io.datavines.engine.config.JobConfigurationBuilder +++ /dev/null @@ -1,8 +0,0 @@ -spark_single_table=io.datavines.engine.spark.config.SparkSingleTableMetricBuilder -spark_data_profile=io.datavines.engine.spark.config.SparkDataProfileMetricBuilder -spark_multi_table_accuracy=io.datavines.engine.spark.config.SparkMultiTableAccuracyMetricBuilder -spark_multi_table_value_comparison=io.datavines.engine.spark.config.SparkMultiTableValueComparisonMetricBuilder -livy_single_table=io.datavines.engine.spark.config.SparkSingleTableMetricBuilder -livy_data_profile=io.datavines.engine.spark.config.SparkDataProfileMetricBuilder -livy_multi_table_accuracy=io.datavines.engine.spark.config.SparkMultiTableAccuracyMetricBuilder -livy_multi_table_value_comparison=io.datavines.engine.spark.config.SparkMultiTableValueComparisonMetricBuilder \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder new file mode 100644 index 000000000..4c5d34bae --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/src/main/resources/META-INF/services/io.datavines.engine.config.JobConfigurationBuilder @@ -0,0 +1,4 @@ +io.datavines.engine.spark.config.SparkDataProfileMetricBuilder +io.datavines.engine.spark.config.SparkMultiTableAccuracyMetricBuilder +io.datavines.engine.spark.config.SparkMultiTableValueComparisonMetricBuilder +io.datavines.engine.spark.config.SparkSingleTableMetricBuilder diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java index 77cd6892f..02832af26 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java @@ -147,4 +147,9 @@ public Void output(Dataset data, SparkRuntimeEnvironment environment) { return null; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy-batch-jdbc-sink", "spark-batch-jdbc-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java index 9975daca2..3b94d6f93 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java @@ -17,4 +17,9 @@ package io.datavines.engine.spark.jdbc.sink; public class MySQLSink extends JdbcSink { + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy-batch-mysql-sink", "spark-batch-mysql-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java index abf1ef5c8..926ef5f23 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java @@ -114,4 +114,9 @@ private Dataset hiveSourceData(SparkRuntimeEnvironment env) { return sparkSession.table(config.getString(TABLE)); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy-batch-jdbc-source", "spark-batch-jdbc-source"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index b9433e91e..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,6 +0,0 @@ -spark-batch-jdbc-sink=io.datavines.engine.spark.jdbc.sink.JdbcSink -spark-batch-jdbc-source=io.datavines.engine.spark.jdbc.source.JdbcSource -spark-batch-mysql-sink=io.datavines.engine.spark.jdbc.sink.MySQLSink -livy-batch-jdbc-sink=io.datavines.engine.spark.jdbc.sink.JdbcSink -livy-batch-jdbc-source=io.datavines.engine.spark.jdbc.source.JdbcSource -livy-batch-mysql-sink=io.datavines.engine.spark.jdbc.sink.MySQLSink \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..ddb4119a6 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1,3 @@ +io.datavines.engine.spark.jdbc.sink.JdbcSink +io.datavines.engine.spark.jdbc.sink.MySQLSink +io.datavines.engine.spark.jdbc.source.JdbcSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java index d481948f9..e1d279913 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java @@ -95,4 +95,9 @@ public Void output(Dataset data, SparkRuntimeEnvironment environment) { return null; } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy-batch-mongodb-sink", "spark-batch-mongodb-sink"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java index 1969aeb4b..6be0cbff0 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java @@ -84,4 +84,9 @@ public Dataset getData(SparkRuntimeEnvironment env) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy-batch-mongodb-source", "spark-batch-mongodb-source"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index d7def4073..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,4 +0,0 @@ -spark-batch-mongodb-sink=io.datavines.engine.spark.jdbc.sink.MongodbSink -spark-batch-mongodb-source=io.datavines.engine.spark.jdbc.source.MongodbSource -livy-batch-mongodb-sink=io.datavines.engine.spark.jdbc.sink.MongodbSink -livy-batch-mongodb-source=io.datavines.engine.spark.jdbc.source.MongodbSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..10483d987 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1,2 @@ +io.datavines.engine.spark.jdbc.sink.MongodbSink +io.datavines.engine.spark.jdbc.source.MongodbSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/java/io/datavines/engine/spark/executor/SparkEngineExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/java/io/datavines/engine/spark/executor/SparkEngineExecutor.java index ba96b6a35..5f7fa00c0 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/java/io/datavines/engine/spark/executor/SparkEngineExecutor.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/java/io/datavines/engine/spark/executor/SparkEngineExecutor.java @@ -149,4 +149,9 @@ protected String buildCommand() { return command; } + + @Override + public String getPluginName() { + return "spark"; + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor deleted file mode 100644 index 15db5e1f1..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/plugins/io.datavines.engine.api.engine.EngineExecutor +++ /dev/null @@ -1 +0,0 @@ -spark=io.datavines.engine.spark.executor.SparkEngineExecutor \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor new file mode 100644 index 000000000..0f6744e6a --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/src/main/resources/META-INF/services/io.datavines.engine.api.engine.EngineExecutor @@ -0,0 +1 @@ +io.datavines.engine.spark.executor.SparkEngineExecutor diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java index a69fffdb1..cc230a4b3 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java @@ -61,4 +61,9 @@ public void prepare(RuntimeEnvironment prepareEnv) { public Dataset process(Dataset data, SparkRuntimeEnvironment env) { return env.sparkSession().sql(config.getString(SQL)); } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy-batch-sql-transform", "spark-batch-sql-transform"); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component deleted file mode 100644 index 3dab29e5b..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/plugins/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,2 +0,0 @@ -spark-batch-sql-transform=io.datavines.engine.spark.transform.sql.SqlTransform -livy-batch-sql-transform=io.datavines.engine.spark.transform.sql.SqlTransform \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component new file mode 100644 index 000000000..820017a22 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component @@ -0,0 +1 @@ +io.datavines.engine.spark.transform.sql.SqlTransform diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ExpectedValue.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ExpectedValue.java index c1cfbbb5a..f9bb5f97a 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ExpectedValue.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ExpectedValue.java @@ -16,13 +16,13 @@ */ package io.datavines.metric.api; -import io.datavines.spi.SPI; - +import java.util.Collection; import java.util.Map; -@SPI public interface ExpectedValue { + Collection getPluginNames(); + /** * get value name * @return String diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java index 0acd5008a..7b308cc53 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/MetricValidator.java @@ -17,7 +17,7 @@ package io.datavines.metric.api; import io.datavines.common.enums.OperatorType; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.lang3.StringUtils; import java.math.BigDecimal; @@ -36,7 +36,7 @@ public static boolean isSuccess(MetricExecutionResult executionResult) { BigDecimal expectedValue = executionResult.getExpectedValue(); OperatorType operatorType = OperatorType.of(StringUtils.trim(executionResult.getOperator())); - ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class) + ResultFormula resultFormula = PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames) .getOrCreatePlugin(executionResult.getResultFormula()); return getCompareResult(operatorType, @@ -48,9 +48,9 @@ public static BigDecimal getQualityScore(MetricExecutionResult executionResult, BigDecimal actualValue = executionResult.getActualValue(); BigDecimal expectedValue = executionResult.getExpectedValue(); - ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class) + ResultFormula resultFormula = PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames) .getOrCreatePlugin(executionResult.getResultFormula()); - SqlMetric metric = PluginLoader.getPluginLoader(SqlMetric.class) + SqlMetric metric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames) .getOrCreatePlugin(executionResult.getMetricName()); return resultFormula.getScore(actualValue, expectedValue, isSuccess, metric.getDirectionType()); diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java index 9faa566cf..f87ad9f72 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/ResultFormula.java @@ -16,15 +16,20 @@ */ package io.datavines.metric.api; -import io.datavines.spi.SPI; - import java.math.BigDecimal; +import java.util.Collection; +import java.util.Collections; -@SPI public interface ResultFormula { String getName(); + String getPluginName(); + + default Collection getPluginNames() { + return Collections.singletonList(getPluginName()); + } + String getZhName(); default String getNameByLanguage(boolean isEn) { diff --git a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java index 1eff89697..d7376091e 100644 --- a/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java +++ b/datavines-metric/datavines-metric-api/src/main/java/io/datavines/metric/api/SqlMetric.java @@ -16,6 +16,7 @@ */ package io.datavines.metric.api; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -23,13 +24,19 @@ import io.datavines.common.config.CheckResult; import io.datavines.common.entity.ExecuteSql; import io.datavines.common.enums.DataVinesDataType; -import io.datavines.spi.SPI;; -@SPI public interface SqlMetric { String getName(); + default String getPluginName() { + return getName(); + } + + default Collection getPluginNames() { + return Collections.singletonList(getPluginName()); + } + String getZhName(); default String getNameByLanguage(boolean isEn) { diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java index fe1fa0db1..77267ddd1 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-base/src/main/java/io/datavines/metric/expected/plugin/AbstractExpectedValue.java @@ -18,7 +18,7 @@ import io.datavines.connector.api.ConnectorFactory; import io.datavines.metric.api.ExpectedValue; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import java.util.Map; @@ -26,10 +26,10 @@ public abstract class AbstractExpectedValue implements ExpectedValue { protected ConnectorFactory getConnectorFactory(Map inputParameter) { String srcConnectorType = inputParameter.get("src_connector_type"); - return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType); + return PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(srcConnectorType); } protected ConnectorFactory getConnectorFactory(String connectorType) { - return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(connectorType); + return PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(connectorType); } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java index e620f4120..252dcc928 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/DailyAvg.java @@ -72,4 +72,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_daily_avg", "livy_daily_avg", "local_daily_avg", "spark_daily_avg"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/SparkDailyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/SparkDailyAvg.java deleted file mode 100644 index 8056570e8..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/java/io/datavines/metric/expected/plugin/SparkDailyAvg.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.metric.expected.plugin; - -import io.datavines.metric.api.ExpectedValue; - -import java.util.Map; - -import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; - -public class SparkDailyAvg implements ExpectedValue { - - @Override - public String getName() { - return "daily_avg"; - } - - @Override - public String getZhName() { - return "日均值"; - } - - @Override - public String getKey(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return String.format("daily_range_%s.expected_value_%s",uniqueKey,uniqueKey); - } - - @Override - public String getExecuteSql(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >=date_format(${data_time}, 'yyyy-MM-dd')" + - " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; - } - - @Override - public String getOutputTable(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "daily_range_" + uniqueKey; - } - - @Override - public boolean isNeedDefaultDatasource() { - return true; - } - - @Override - public void prepare(Map config) { - - } -} diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index c3217560a..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_daily_avg=io.datavines.metric.expected.plugin.DailyAvg -spark_daily_avg=io.datavines.metric.expected.plugin.DailyAvg -livy_daily_avg=io.datavines.metric.expected.plugin.DailyAvg -flink_daily_avg=io.datavines.metric.expected.plugin.DailyAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..4924f6e93 --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-daily-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.DailyAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/java/io/datavines/metric/expected/plugin/FixValue.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/java/io/datavines/metric/expected/plugin/FixValue.java index ad338b47c..bb8738297 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/java/io/datavines/metric/expected/plugin/FixValue.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/java/io/datavines/metric/expected/plugin/FixValue.java @@ -58,4 +58,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_fix_value", "livy_fix_value", "local_fix_value", "spark_fix_value"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index bce9667bb..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_fix_value=io.datavines.metric.expected.plugin.FixValue -spark_fix_value=io.datavines.metric.expected.plugin.FixValue -livy_fix_value=io.datavines.metric.expected.plugin.FixValue -flink_fix_value=io.datavines.metric.expected.plugin.FixValue \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..fcb909fe3 --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-fix/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.FixValue diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java index 2de39a76b..f847ad243 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/Last30DayAvg.java @@ -71,4 +71,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_last_30d_avg", "livy_last_30d_avg", "local_last_30d_avg", "spark_last_30d_avg"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast30DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast30DayAvg.java deleted file mode 100644 index 5a8c903e7..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast30DayAvg.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.metric.expected.plugin; - -import io.datavines.metric.api.ExpectedValue; - -import java.util.Map; - -import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; - -public class SparkLast30DayAvg implements ExpectedValue { - - @Override - public String getName() { - return "last_30d_avg"; - } - - @Override - public String getZhName() { - return "最近30天均值"; - } - - @Override - public String getKey(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return String.format("last_30d_%s.expected_value_%s", uniqueKey, uniqueKey); - } - - @Override - public String getExecuteSql(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_add(date_format(${data_time}, 'yyyy-MM-dd'),-30)" + - " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; - } - - @Override - public String getOutputTable(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "last_30d_" + uniqueKey; - } - - @Override - public boolean isNeedDefaultDatasource() { - return true; - } - - @Override - public void prepare(Map config) { - - } -} diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index 0a9716fd9..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg -spark_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg -livy_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg -flink_last_30d_avg=io.datavines.metric.expected.plugin.Last30DayAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..1656378ee --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last30day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.Last30DayAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java index d9d444c09..8ac8914c1 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/Last7DayAvg.java @@ -71,4 +71,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_last_7d_avg", "livy_last_7d_avg", "local_last_7d_avg", "spark_last_7d_avg"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast7DayAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast7DayAvg.java deleted file mode 100644 index 443f1812f..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/java/io/datavines/metric/expected/plugin/SparkLast7DayAvg.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.metric.expected.plugin; - -import io.datavines.metric.api.ExpectedValue; - -import java.util.Map; - -import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; - -public class SparkLast7DayAvg implements ExpectedValue { - - @Override - public String getName() { - return "last_7d_avg"; - } - - @Override - public String getZhName() { - return "最近7天均值"; - } - - @Override - public String getKey(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return String.format("last_7d_%s.expected_value_%s",uniqueKey,uniqueKey); - } - - @Override - public String getExecuteSql(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_add(date_format(${data_time}, 'yyyy-MM-dd'),-7)" + - " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; - } - - @Override - public String getOutputTable(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "last_7d_" + uniqueKey; - } - - @Override - public boolean isNeedDefaultDatasource() { - return true; - } - - @Override - public void prepare(Map config) { - - } -} diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index 4c4a340ae..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg -spark_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg -livy_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg -flink_last_7d_avg=io.datavines.metric.expected.plugin.Last7DayAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..7732af90b --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-last7day-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.Last7DayAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java index 7de3f7264..5c05dfa77 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/MonthlyAvg.java @@ -71,4 +71,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_monthly_avg", "livy_monthly_avg", "local_monthly_avg", "spark_monthly_avg"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkMonthlyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkMonthlyAvg.java deleted file mode 100644 index 48699f5a6..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkMonthlyAvg.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.metric.expected.plugin; - -import io.datavines.metric.api.ExpectedValue; - -import java.util.Map; - -import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; - -public class SparkMonthlyAvg implements ExpectedValue { - - @Override - public String getName() { - return "monthly_avg"; - } - - @Override - public String getZhName() { - return "月均值"; - } - - @Override - public String getKey(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return String.format("monthly_range_%s.expected_value_%s",uniqueKey,uniqueKey); - } - - @Override - public String getExecuteSql(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_format(${data_time}, 'yyyy-MM-01')" + - " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; - } - - @Override - public String getOutputTable(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "monthly_range_" + uniqueKey; - } - - @Override - public boolean isNeedDefaultDatasource() { - return true; - } - - @Override - public void prepare(Map config) { - - } -} diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index ee2481e4c..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg -spark_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg -livy_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg -flink_monthly_avg=io.datavines.metric.expected.plugin.MonthlyAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..0b8f56ffc --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-monthly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.MonthlyAvg diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/None.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/None.java index e01cf77a1..b510ceccc 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/None.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/None.java @@ -59,4 +59,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_none", "local_none"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/SparkNone.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/SparkNone.java index 17430e3b9..dda0d4b29 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/SparkNone.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/java/io/datavines/metric/expected/plugin/SparkNone.java @@ -56,4 +56,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("livy_none", "spark_none"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index 4134bd927..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_none=io.datavines.metric.expected.plugin.None -spark_none=io.datavines.metric.expected.plugin.SparkNone -livy_none=io.datavines.metric.expected.plugin.SparkNone -flink_none=io.datavines.metric.expected.plugin.None diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..688956005 --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-none/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1,2 @@ +io.datavines.metric.expected.plugin.None +io.datavines.metric.expected.plugin.SparkNone diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java index dd0e2b50f..8899ce9bd 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/java/io/datavines/metric/expected/plugin/TableTotalRows.java @@ -72,4 +72,9 @@ public void prepare(Map config) { } } } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_table_total_rows", "livy_table_total_rows", "local_table_total_rows", "spark_table_total_rows"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index f84be9653..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows -spark_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows -livy_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows -flink_table_total_rows=io.datavines.metric.expected.plugin.TableTotalRows \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..1f121d64a --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.TableTotalRows diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/java/io/datavines/metric/expected/plugin/TargetTableTotalRows.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/java/io/datavines/metric/expected/plugin/TargetTableTotalRows.java index 86a3a2ad5..707c946fe 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/java/io/datavines/metric/expected/plugin/TargetTableTotalRows.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/java/io/datavines/metric/expected/plugin/TargetTableTotalRows.java @@ -72,4 +72,9 @@ public void prepare(Map config) { } } } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_target_table_total_rows", "livy_target_table_total_rows", "local_target_table_total_rows", "spark_target_table_total_rows"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index b105fbcda..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows -spark_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows -livy_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows -flink_target_table_total_rows=io.datavines.metric.expected.plugin.TargetTableTotalRows \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..960d75fa7 --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-target-table-rows/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.TargetTableTotalRows diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkWeeklyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkWeeklyAvg.java deleted file mode 100644 index 9bc7481e0..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/SparkWeeklyAvg.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.metric.expected.plugin; - -import io.datavines.metric.api.ExpectedValue; - -import java.util.Map; - -import static io.datavines.common.ConfigConstants.METRIC_UNIQUE_KEY; - -public class SparkWeeklyAvg implements ExpectedValue { - - @Override - public String getName() { - return "weekly_avg"; - } - - @Override - public String getKey(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return String.format("weekly_range_%s.expected_value_%s",uniqueKey,uniqueKey); - } - - @Override - public String getZhName() { - return "周均值"; - } - - @Override - public String getExecuteSql(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "select round(avg(actual_value),2) as expected_value_" + uniqueKey + - " from dv_actual_values where data_time >= date_sub(date_format(${data_time},'yyyy-MM-dd'), (7- datediff(next_day(date_format(${data_time}, 'yyyy-MM-dd'),'Sunday'),date_format(${data_time}, 'yyyy-MM-dd')))-1)" + - " and data_time < date_add(date_format(${data_time}, 'yyyy-MM-dd'),1) and unique_code = ${unique_code}"; - } - - @Override - public String getOutputTable(Map inputParameter) { - String uniqueKey = inputParameter.get(METRIC_UNIQUE_KEY); - return "weekly_range_" + uniqueKey; - } - - @Override - public boolean isNeedDefaultDatasource() { - return true; - } - - @Override - public void prepare(Map config) { - - } -} diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java index 901c66031..c8af4fd80 100644 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/java/io/datavines/metric/expected/plugin/WeeklyAvg.java @@ -72,4 +72,9 @@ public boolean isNeedDefaultDatasource() { public void prepare(Map config) { } + + @Override + public java.util.Collection getPluginNames() { + return java.util.Arrays.asList("flink_weekly_avg", "livy_weekly_avg", "local_weekly_avg", "spark_weekly_avg"); + } } diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue deleted file mode 100644 index eef2d6152..000000000 --- a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.ExpectedValue +++ /dev/null @@ -1,4 +0,0 @@ -local_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg -spark_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg -livy_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg -flink_weekly_avg=io.datavines.metric.expected.plugin.WeeklyAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue new file mode 100644 index 000000000..262ab1728 --- /dev/null +++ b/datavines-metric/datavines-metric-expected-plugins/datavines-metric-expected-weekly-avg/src/main/resources/META-INF/services/io.datavines.metric.api.ExpectedValue @@ -0,0 +1 @@ +io.datavines.metric.expected.plugin.WeeklyAvg diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java b/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java index 13d959036..784dc0fda 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-base/src/main/java/io/datavines/metric/plugin/base/BaseSingleTable.java @@ -23,7 +23,7 @@ import io.datavines.metric.api.ConfigItem; import io.datavines.metric.api.MetricLevel; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.lang3.StringUtils; import java.util.*; @@ -95,6 +95,6 @@ public MetricLevel getLevel() { protected ConnectorFactory getConnectorFactory(Map inputParameter) { String srcConnectorType = inputParameter.get("src_connector_type"); - return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(srcConnectorType); + return PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(srcConnectorType); } } \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java index 1d26ab6fc..4adfaae7e 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/java/io/datavines/metric/plugin/ColumnAvgLength.java @@ -84,4 +84,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_avg_length"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 6f2b004d8..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_avg_length=io.datavines.metric.plugin.ColumnAvgLength \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..ad5b282f6 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnAvgLength diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java index 06b21303b..f0cf07976 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/java/io/datavines/metric/plugin/ColumnAvg.java @@ -88,4 +88,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Collections.singletonList(DataVinesDataType.NUMERIC_TYPE); } + + @Override + public String getPluginName() { + return "column_avg"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 3d01d7242..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_avg=io.datavines.metric.plugin.ColumnAvg \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..75318ecfc --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-avg/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnAvg diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java index 12768db55..cc085ef0e 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/java/io/datavines/metric/plugin/ColumnBlank.java @@ -74,4 +74,9 @@ public List suitableType() { public MetricDirectionType getDirectionType() { return MetricDirectionType.NEGATIVE; } + + @Override + public String getPluginName() { + return "column_blank"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 9eb8fedb7..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_blank=io.datavines.metric.plugin.ColumnBlank \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..6d2c770c1 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-blank/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnBlank diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java index d86a49104..c8a20d3ab 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/java/io/datavines/metric/plugin/ColumnDistinct.java @@ -79,4 +79,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_distinct"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 7c2adb4a5..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_distinct=io.datavines.metric.plugin.ColumnDistinct \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..a96a25be7 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-distinct/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnDistinct diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java index 812abe3a8..2f6f4ec77 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/java/io/datavines/metric/plugin/ColumnDuplicate.java @@ -80,4 +80,9 @@ public List suitableType() { public MetricDirectionType getDirectionType() { return MetricDirectionType.NEGATIVE; } + + @Override + public String getPluginName() { + return "column_duplicate"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 381157c2b..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_duplicate=io.datavines.metric.plugin.ColumnDuplicate \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..d1e57c7c0 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-duplicate/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnDuplicate diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java index 179384325..57ac8dfb7 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/java/io/datavines/metric/plugin/ColumnHistogram.java @@ -23,7 +23,7 @@ import io.datavines.metric.api.MetricDimension; import io.datavines.metric.api.MetricType; import io.datavines.metric.plugin.base.BaseSingleTableColumn; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import java.util.Arrays; import java.util.List; @@ -109,6 +109,11 @@ public List suitableType() { } private ConnectorFactory getConnectorFactory() { - return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin("spark"); + return PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin("spark"); + } + + @Override + public String getPluginName() { + return "column_histogram"; } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 8b418430f..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_histogram=io.datavines.metric.plugin.ColumnHistogram \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..8a5d75ed1 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-histogram/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnHistogram diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java index 14b31691d..35135da5c 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/java/io/datavines/metric/plugin/ColumnInEnums.java @@ -71,4 +71,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_in_enums"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 3dff59416..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_in_enums=io.datavines.metric.plugin.ColumnInEnums \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..c6e236fdd --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnInEnums diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java index a09a6c66d..b5aff558e 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/java/io/datavines/metric/plugin/ColumnLength.java @@ -76,4 +76,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_length"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 8720def6f..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_length=io.datavines.metric.plugin.ColumnLength \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..f2546eeb7 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnLength diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java index 7742c56ef..b17021bb6 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchNotRegex.java @@ -80,4 +80,9 @@ public List suitableType() { public MetricDirectionType getDirectionType() { return MetricDirectionType.NEGATIVE; } + + @Override + public String getPluginName() { + return "column_match_not_regex"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 9e65d81aa..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_match_not_regex=io.datavines.metric.plugin.ColumnMatchNotRegex \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..86e290735 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-not-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnMatchNotRegex diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java index 584248907..92a4bfa40 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/java/io/datavines/metric/plugin/ColumnMatchRegex.java @@ -73,4 +73,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_match_regex"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 6b99ae8ef..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_match_regex=io.datavines.metric.plugin.ColumnMatchRegex \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..7eae13206 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-match-regex/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnMatchRegex diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java index 7f62c6cee..c432bfb09 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/java/io/datavines/metric/plugin/ColumnMaxLength.java @@ -84,4 +84,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_max_length"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 35e779a7c..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_max_length=io.datavines.metric.plugin.ColumnMaxLength \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..e7c5f64b7 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnMaxLength diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java index 02c67d544..6469abf5c 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/java/io/datavines/metric/plugin/ColumnMax.java @@ -85,4 +85,9 @@ public List suitableType() { return Collections.singletonList(DataVinesDataType.NUMERIC_TYPE); } + + @Override + public String getPluginName() { + return "column_max"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index a3cde2640..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_max=io.datavines.metric.plugin.ColumnMax \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..2d545ef8a --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-max/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnMax diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java index 99a0260f4..902a14425 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/java/io/datavines/metric/plugin/ColumnMinLength.java @@ -87,4 +87,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Arrays.asList(DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_min_length"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index cb5626933..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_min_length=io.datavines.metric.plugin.ColumnMinLength \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..a43361a84 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min-length/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnMinLength diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java index 4cc47d171..f14314fb7 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/java/io/datavines/metric/plugin/ColumnMin.java @@ -84,4 +84,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Collections.singletonList(DataVinesDataType.NUMERIC_TYPE); } + + @Override + public String getPluginName() { + return "column_min"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 2b3660c11..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_min=io.datavines.metric.plugin.ColumnMin \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..5adb78d2f --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-min/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnMin diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java index bdd1168dd..36d631ec7 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/java/io/datavines/metric/plugin/ColumnNotInEnums.java @@ -79,4 +79,9 @@ public MetricDirectionType getDirectionType() { return MetricDirectionType.NEGATIVE; } + + @Override + public String getPluginName() { + return "column_not_in_enums"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 257795afb..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_not_in_enums=io.datavines.metric.plugin.ColumnNotInEnums \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..7d48f019f --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-in-enums/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnNotInEnums diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java index 897e0a778..8e679b2c1 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/java/io/datavines/metric/plugin/ColumnNotNull.java @@ -68,4 +68,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_not_null"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 47ab322b0..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_not_null=io.datavines.metric.plugin.ColumnNotNull \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..b21f123ac --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-not-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnNotNull diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java index a8ea50dcf..aad34a4e9 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/java/io/datavines/metric/plugin/ColumnNull.java @@ -74,4 +74,9 @@ public List suitableType() { public MetricDirectionType getDirectionType() { return MetricDirectionType.NEGATIVE; } + + @Override + public String getPluginName() { + return "column_null"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index a78524534..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_null=io.datavines.metric.plugin.ColumnNull \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..5bbda1725 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-null/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnNull diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java index 66e52f6e0..abaf3e18a 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/java/io/datavines/metric/plugin/ColumnStdDev.java @@ -84,4 +84,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Collections.singletonList(DataVinesDataType.NUMERIC_TYPE); } + + @Override + public String getPluginName() { + return "column_std_dev"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index ced510750..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_std_dev=io.datavines.metric.plugin.ColumnStdDev \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..a52dbecd0 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-std-dev/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnStdDev diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java index 0c7f47337..a19d56ca0 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/java/io/datavines/metric/plugin/ColumnSum.java @@ -84,4 +84,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Collections.singletonList(DataVinesDataType.NUMERIC_TYPE); } + + @Override + public String getPluginName() { + return "column_sum"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 74a3c1233..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_sum=io.datavines.metric.plugin.ColumnSum \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..bc8c8cc9e --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-sum/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnSum diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java index 3582b1004..dc06763e0 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/java/io/datavines/metric/plugin/ColumnUnique.java @@ -75,4 +75,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_unique"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 0e1871d86..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_unique=io.datavines.metric.plugin.ColumnUnique \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..ec1923e1d --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-unique/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnUnique diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java index 4c4ed85cd..75f3c5079 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/java/io/datavines/metric/plugin/ColumnValueBetween.java @@ -79,4 +79,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "column_value_between"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 8e9b9c7a4..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_value_between=io.datavines.metric.plugin.ColumnValueBetween \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..16cfe4c9e --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-value-between/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnValueBetween diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java index b22a02612..e6d0d0c97 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/java/io/datavines/metric/plugin/ColumnVariance.java @@ -84,4 +84,9 @@ public ExecuteSql getActualValue(Map inputParameter) { public List suitableType() { return Collections.singletonList(DataVinesDataType.NUMERIC_TYPE); } + + @Override + public String getPluginName() { + return "column_variance"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index f73b40bd5..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -column_variance=io.datavines.metric.plugin.ColumnVariance \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..5812735b0 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-column-variance/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.ColumnVariance diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/java/io/datavines/metric/plugin/CustomAggregateSql.java b/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/java/io/datavines/metric/plugin/CustomAggregateSql.java index 71954ec13..0d9be4664 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/java/io/datavines/metric/plugin/CustomAggregateSql.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/java/io/datavines/metric/plugin/CustomAggregateSql.java @@ -115,4 +115,9 @@ public List suitableType() { public boolean isCustomSql() { return true; } + + @Override + public String getPluginName() { + return "custom_aggregate_sql"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 639776080..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -custom_aggregate_sql=io.datavines.metric.plugin.CustomAggregateSql \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..b454898ba --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-custom-aggregate-sql/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.CustomAggregateSql diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/java/io/datavines/metric/plugin/MultiTableAccuracy.java b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/java/io/datavines/metric/plugin/MultiTableAccuracy.java index 88b5d9fc5..dea25a1a1 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/java/io/datavines/metric/plugin/MultiTableAccuracy.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/java/io/datavines/metric/plugin/MultiTableAccuracy.java @@ -133,4 +133,9 @@ public ExecuteSql getDirectActualValue(Map inputParameter) { public List suitableType() { return Collections.emptyList(); } + + @Override + public String getPluginName() { + return "multi_table_accuracy"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 0bf558952..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -multi_table_accuracy=io.datavines.metric.plugin.MultiTableAccuracy \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..2cb899e4a --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-accuracy/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.MultiTableAccuracy diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/java/io/datavines/metric/plugin/MultiTableValueComparison.java b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/java/io/datavines/metric/plugin/MultiTableValueComparison.java index f2cc9899a..af344a2ef 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/java/io/datavines/metric/plugin/MultiTableValueComparison.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/java/io/datavines/metric/plugin/MultiTableValueComparison.java @@ -84,4 +84,9 @@ public void prepare(Map config) { public List suitableType() { return Collections.emptyList(); } + + @Override + public String getPluginName() { + return "multi_table_value_comparison"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index c7c10cae5..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -multi_table_value_comparison=io.datavines.metric.plugin.MultiTableValueComparison \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..100a5ba62 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-multi-table-value-comparison/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.MultiTableValueComparison diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java index a0e1ff285..b78d041db 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/java/io/datavines/metric/plugin/Freshness.java @@ -76,4 +76,9 @@ public void prepare(Map config) { public List suitableType() { return Arrays.asList(DataVinesDataType.NUMERIC_TYPE, DataVinesDataType.STRING_TYPE, DataVinesDataType.DATE_TIME_TYPE); } + + @Override + public String getPluginName() { + return "freshness"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 1f10cbc0e..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -freshness=io.datavines.metric.plugin.Freshness \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..452f39cc3 --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-table-freshness/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.Freshness diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/java/io/datavines/metric/plugin/TableRowCount.java b/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/java/io/datavines/metric/plugin/TableRowCount.java index ec0a3c5d5..d3f052a64 100644 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/java/io/datavines/metric/plugin/TableRowCount.java +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/java/io/datavines/metric/plugin/TableRowCount.java @@ -123,4 +123,9 @@ public List> getMetricParameter(Map metricPa return result; } + + @Override + public String getPluginName() { + return "table_row_count"; + } } diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric deleted file mode 100644 index 4a2f8650c..000000000 --- a/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.SqlMetric +++ /dev/null @@ -1 +0,0 @@ -table_row_count=io.datavines.metric.plugin.TableRowCount \ No newline at end of file diff --git a/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric b/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric new file mode 100644 index 000000000..6d48bc1bf --- /dev/null +++ b/datavines-metric/datavines-metric-plugins/datavines-metric-table-row-count/src/main/resources/META-INF/services/io.datavines.metric.api.SqlMetric @@ -0,0 +1 @@ +io.datavines.metric.plugin.TableRowCount diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java index 0a3f35b1a..a40895ddf 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/java/io/datavines/metric/result/formula/Count.java @@ -58,4 +58,9 @@ public String getSymbol() { public ResultFormulaType getType() { return ResultFormulaType.VALUE; } + + @Override + public String getPluginName() { + return "count"; + } } diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula deleted file mode 100644 index 929ccdf42..000000000 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula +++ /dev/null @@ -1 +0,0 @@ -count=io.datavines.metric.result.formula.Count \ No newline at end of file diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula new file mode 100644 index 000000000..814ed7d5b --- /dev/null +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-count/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula @@ -0,0 +1 @@ +io.datavines.metric.result.formula.Count diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java index 68fdd73c6..ee2ff4b54 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/java/io/datavines/metric/result/formula/ActualMinusExpectedDiff.java @@ -58,4 +58,9 @@ public String getSymbol() { public ResultFormulaType getType() { return ResultFormulaType.VALUE; } + + @Override + public String getPluginName() { + return "diff-actual-expected"; + } } diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula deleted file mode 100644 index f7bc28821..000000000 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula +++ /dev/null @@ -1 +0,0 @@ -diff-actual-expected=io.datavines.metric.result.formula.ActualMinusExpectedDiff \ No newline at end of file diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula new file mode 100644 index 000000000..7f9e8860e --- /dev/null +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-actual-expected/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula @@ -0,0 +1 @@ +io.datavines.metric.result.formula.ActualMinusExpectedDiff diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java index 55b97be69..977e845f6 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/java/io/datavines/metric/result/formula/DiffPercentage.java @@ -66,4 +66,9 @@ public String getSymbol() { public ResultFormulaType getType() { return ResultFormulaType.PERCENTAGE; } + + @Override + public String getPluginName() { + return "diff-percentage"; + } } diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula deleted file mode 100644 index 918335007..000000000 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula +++ /dev/null @@ -1 +0,0 @@ -diff-percentage=io.datavines.metric.result.formula.DiffPercentage \ No newline at end of file diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula new file mode 100644 index 000000000..ae31ac7d4 --- /dev/null +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula @@ -0,0 +1 @@ +io.datavines.metric.result.formula.DiffPercentage diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java index 51e2fc986..3d538baad 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/java/io/datavines/metric/result/formula/Diff.java @@ -58,4 +58,9 @@ public String getSymbol() { public ResultFormulaType getType() { return ResultFormulaType.VALUE; } + + @Override + public String getPluginName() { + return "diff"; + } } diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula deleted file mode 100644 index 78c2fdd5f..000000000 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula +++ /dev/null @@ -1 +0,0 @@ -diff=io.datavines.metric.result.formula.Diff \ No newline at end of file diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula new file mode 100644 index 000000000..f26f42e8e --- /dev/null +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-diff/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula @@ -0,0 +1 @@ +io.datavines.metric.result.formula.Diff diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java index 82f0a113d..b260be3d2 100644 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/java/io/datavines/metric/result/formula/Percentage.java @@ -75,4 +75,9 @@ public ResultFormulaType getType() { return ResultFormulaType.PERCENTAGE; } + + @Override + public String getPluginName() { + return "percentage"; + } } diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula deleted file mode 100644 index 0f7ec547e..000000000 --- a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/plugins/io.datavines.metric.api.ResultFormula +++ /dev/null @@ -1 +0,0 @@ -percentage=io.datavines.metric.result.formula.Percentage \ No newline at end of file diff --git a/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula new file mode 100644 index 000000000..c2a6ee6bf --- /dev/null +++ b/datavines-metric/datavines-metric-result-formula-plugins/datavines-metric-result-formula-percentage/src/main/resources/META-INF/services/io.datavines.metric.api.ResultFormula @@ -0,0 +1 @@ +io.datavines.metric.result.formula.Percentage diff --git a/datavines-notification/datavines-notification-api/src/main/java/io/datavines/notification/api/spi/SlasHandlerPlugin.java b/datavines-notification/datavines-notification-api/src/main/java/io/datavines/notification/api/spi/SlasHandlerPlugin.java index cbd3b8512..67d64dd37 100644 --- a/datavines-notification/datavines-notification-api/src/main/java/io/datavines/notification/api/spi/SlasHandlerPlugin.java +++ b/datavines-notification/datavines-notification-api/src/main/java/io/datavines/notification/api/spi/SlasHandlerPlugin.java @@ -20,14 +20,20 @@ import io.datavines.notification.api.entity.SlaNotificationResult; import io.datavines.notification.api.entity.SlaConfigMessage; import io.datavines.notification.api.entity.SlaSenderMessage; -import io.datavines.spi.SPI; +import java.util.Collection; +import java.util.Collections; import java.util.Map; import java.util.Set; -@SPI public interface SlasHandlerPlugin { + String getPluginName(); + + default Collection getPluginNames() { + return Collections.singletonList(getPluginName()); + } + /** * save message to db then send message to receiver , return the status finally * @param slaNotificationMessage issue message a diff --git a/datavines-notification/datavines-notification-core/src/main/java/io/datavines/notification/core/NotificationManager.java b/datavines-notification/datavines-notification-core/src/main/java/io/datavines/notification/core/NotificationManager.java index 6f72323b3..ce39fd52b 100644 --- a/datavines-notification/datavines-notification-core/src/main/java/io/datavines/notification/core/NotificationManager.java +++ b/datavines-notification/datavines-notification-core/src/main/java/io/datavines/notification/core/NotificationManager.java @@ -22,7 +22,7 @@ import io.datavines.notification.api.entity.SlaConfigMessage; import io.datavines.notification.api.entity.SlaSenderMessage; import io.datavines.notification.api.spi.SlasHandlerPlugin; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -37,8 +37,8 @@ public class NotificationManager { private final Set supportedPlugins; public NotificationManager(){ - supportedPlugins = PluginLoader - .getPluginLoader(SlasHandlerPlugin.class) + supportedPlugins = PluginDiscovery.getMultiKeyPluginDiscovery(SlasHandlerPlugin.class, SlasHandlerPlugin::getPluginNames) + .getSupportedPlugins(); } @@ -54,8 +54,8 @@ public SlaNotificationResult notify(SlaNotificationMessage slaNotificationMessag if (!supportedPlugins.contains(type)) { throw new DataVinesException("sender type not support of "+ type); } - SlasHandlerPlugin handlerPlugin = PluginLoader - .getPluginLoader(SlasHandlerPlugin.class) + SlasHandlerPlugin handlerPlugin = PluginDiscovery.getMultiKeyPluginDiscovery(SlasHandlerPlugin.class, SlasHandlerPlugin::getPluginNames) + .getOrCreatePlugin(type); Map> senderEntity = new HashMap(){ { diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/DingTalkSlasHandlerPlugin.java b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/DingTalkSlasHandlerPlugin.java index 827b3584a..f2f8a52ad 100644 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/DingTalkSlasHandlerPlugin.java +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/java/io/datavines/notification/plugin/dingtalk/DingTalkSlasHandlerPlugin.java @@ -130,4 +130,9 @@ public String getConfigJson() { return result; } + + @Override + public String getPluginName() { + return "dingtalk"; + } } diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin deleted file mode 100644 index 7556810ce..000000000 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin +++ /dev/null @@ -1 +0,0 @@ -dingtalk=io.datavines.notification.plugin.dingtalk.DingTalkSlasHandlerPlugin \ No newline at end of file diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin new file mode 100644 index 000000000..fde06a325 --- /dev/null +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-dingtalk/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin @@ -0,0 +1 @@ +io.datavines.notification.plugin.dingtalk.DingTalkSlasHandlerPlugin diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/java/io/datavines/notification/plugin/email/EmailSlasHandlerPlugin.java b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/java/io/datavines/notification/plugin/email/EmailSlasHandlerPlugin.java index 5f9842654..be9176afe 100644 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/java/io/datavines/notification/plugin/email/EmailSlasHandlerPlugin.java +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/java/io/datavines/notification/plugin/email/EmailSlasHandlerPlugin.java @@ -221,4 +221,9 @@ private InputParam getInputParamNoValidate(String field, String title, String pl .setEmit(null) .build(); } + + @Override + public String getPluginName() { + return "email"; + } } diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin deleted file mode 100644 index 490325b84..000000000 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin +++ /dev/null @@ -1 +0,0 @@ -email=io.datavines.notification.plugin.email.EmailSlasHandlerPlugin \ No newline at end of file diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin new file mode 100644 index 000000000..6c47758cb --- /dev/null +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-email/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin @@ -0,0 +1 @@ +io.datavines.notification.plugin.email.EmailSlasHandlerPlugin diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/java/io/datavines/notification/plugin/lark/LarkSlasHandlerPlugin.java b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/java/io/datavines/notification/plugin/lark/LarkSlasHandlerPlugin.java index 9d10e9d24..15626895b 100644 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/java/io/datavines/notification/plugin/lark/LarkSlasHandlerPlugin.java +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/java/io/datavines/notification/plugin/lark/LarkSlasHandlerPlugin.java @@ -129,4 +129,9 @@ public String getConfigJson() { return result; } + + @Override + public String getPluginName() { + return "lark"; + } } diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin deleted file mode 100644 index 9196b6dad..000000000 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin +++ /dev/null @@ -1 +0,0 @@ -lark=io.datavines.notification.plugin.lark.LarkSlasHandlerPlugin \ No newline at end of file diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin new file mode 100644 index 000000000..8f06d3b27 --- /dev/null +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-lark/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin @@ -0,0 +1 @@ +io.datavines.notification.plugin.lark.LarkSlasHandlerPlugin diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/java/io/datavines/notification/plugin/wecombot/WecomBotSlasHandlerPlugin.java b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/java/io/datavines/notification/plugin/wecombot/WecomBotSlasHandlerPlugin.java index 5baf90bab..6a4651d3d 100644 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/java/io/datavines/notification/plugin/wecombot/WecomBotSlasHandlerPlugin.java +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/java/io/datavines/notification/plugin/wecombot/WecomBotSlasHandlerPlugin.java @@ -94,4 +94,9 @@ public String getConfigJson() { } return result; } + + @Override + public String getPluginName() { + return "wecombot"; + } } diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin deleted file mode 100644 index bdcc030a3..000000000 --- a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/plugins/io.datavines.notification.api.spi.SlasHandlerPlugin +++ /dev/null @@ -1 +0,0 @@ -wecombot=io.datavines.notification.plugin.wecombot.WecomBotSlasHandlerPlugin \ No newline at end of file diff --git a/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin new file mode 100644 index 000000000..dfdf99fe8 --- /dev/null +++ b/datavines-notification/datavines-notification-plugins/datavines-notification-plugin-wecombot/src/main/resources/META-INF/services/io.datavines.notification.api.spi.SlasHandlerPlugin @@ -0,0 +1 @@ +io.datavines.notification.plugin.wecombot.WecomBotSlasHandlerPlugin diff --git a/datavines-registry/datavines-registry-api/src/main/java/io/datavines/registry/api/Registry.java b/datavines-registry/datavines-registry-api/src/main/java/io/datavines/registry/api/Registry.java index a402dbe21..8e8ca0706 100644 --- a/datavines-registry/datavines-registry-api/src/main/java/io/datavines/registry/api/Registry.java +++ b/datavines-registry/datavines-registry-api/src/main/java/io/datavines/registry/api/Registry.java @@ -16,15 +16,20 @@ */ package io.datavines.registry.api; -import io.datavines.spi.SPI; - import java.sql.SQLException; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Properties; -@SPI public interface Registry { + String getPluginName(); + + default Collection getPluginNames() { + return Collections.singletonList(getPluginName()); + } + void init(Properties properties) throws Exception; boolean acquire(String key, long timeout); diff --git a/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlRegistry.java b/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlRegistry.java index 12ad49cea..d8cbaeca0 100644 --- a/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlRegistry.java +++ b/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/java/io/datavines/registry/plugin/MysqlRegistry.java @@ -103,4 +103,9 @@ public void close() throws SQLException { mysqlMutex.close(); mysqlServerStateManager.close(); } + + @Override + public String getPluginName() { + return "mysql"; + } } diff --git a/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry b/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry deleted file mode 100644 index 1d049622c..000000000 --- a/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry +++ /dev/null @@ -1 +0,0 @@ -mysql=io.datavines.registry.plugin.MysqlRegistry \ No newline at end of file diff --git a/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/services/io.datavines.registry.api.Registry b/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/services/io.datavines.registry.api.Registry new file mode 100644 index 000000000..fe31afc87 --- /dev/null +++ b/datavines-registry/datavines-registry-plugins/datavines-registry-mysql/src/main/resources/META-INF/services/io.datavines.registry.api.Registry @@ -0,0 +1 @@ +io.datavines.registry.plugin.MysqlRegistry diff --git a/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/java/io/datavines/registry/plugin/ZooKeeperRegistry.java b/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/java/io/datavines/registry/plugin/ZooKeeperRegistry.java index 94c775f15..c103c0095 100644 --- a/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/java/io/datavines/registry/plugin/ZooKeeperRegistry.java +++ b/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/java/io/datavines/registry/plugin/ZooKeeperRegistry.java @@ -237,4 +237,9 @@ public EventAdaptor(TreeCacheEvent event, String key) { } } } + + @Override + public String getPluginName() { + return "zookeeper"; + } } diff --git a/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry b/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry deleted file mode 100644 index 3f51cf57a..000000000 --- a/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/plugins/io.datavines.registry.api.Registry +++ /dev/null @@ -1 +0,0 @@ -zookeeper=io.datavines.registry.plugin.ZooKeeperRegistry \ No newline at end of file diff --git a/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/services/io.datavines.registry.api.Registry b/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/services/io.datavines.registry.api.Registry new file mode 100644 index 000000000..3266c59f1 --- /dev/null +++ b/datavines-registry/datavines-registry-plugins/datavines-registry-zookeeper/src/main/resources/META-INF/services/io.datavines.registry.api.Registry @@ -0,0 +1 @@ +io.datavines.registry.plugin.ZooKeeperRegistry diff --git a/datavines-runner/src/main/java/io/datavines/runner/JobRunner.java b/datavines-runner/src/main/java/io/datavines/runner/JobRunner.java index 1f4fc696c..785761d7d 100644 --- a/datavines-runner/src/main/java/io/datavines/runner/JobRunner.java +++ b/datavines-runner/src/main/java/io/datavines/runner/JobRunner.java @@ -37,7 +37,7 @@ import io.datavines.notification.api.entity.SlaNotificationMessage; import io.datavines.notification.api.entity.SlaSenderMessage; import io.datavines.notification.core.NotificationManager; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -71,8 +71,8 @@ public void run() { Logger taskLogger = LoggerFactory.getLogger(taskLoggerName); Thread.currentThread().setName(taskLoggerName); - engineExecutor = PluginLoader - .getPluginLoader(EngineExecutor.class) + engineExecutor = PluginDiscovery.getMultiKeyPluginDiscovery(EngineExecutor.class, EngineExecutor::getPluginNames) + .getNewPlugin(jobExecutionRequest.getEngineType()); engineExecutor.init(jobExecutionRequest, taskLogger, configurations); @@ -86,7 +86,7 @@ public void run() { Long jobExecutionId = jobExecutionRequest.getJobExecutionId(); String validateResultStorageType = jobExecutionRequest.getValidateResultDataStorageType(); ConnectorFactory validateResultStorageFactory = - PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(validateResultStorageType); + PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(validateResultStorageType); if (validateResultStorageFactory == null) { log.error("validate result storage type {} is not supported", validateResultStorageType); return; @@ -159,15 +159,15 @@ private String buildAlertMessage(MetricExecutionResult metricExecutionResult, St parameters.put(ConfigConstants.THRESHOLD, metricExecutionResult.getThreshold()+""); parameters.put(ConfigConstants.OPERATOR, OperatorType.of(metricExecutionResult.getOperator()).getSymbol()); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(metricExecutionResult.getMetricName()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(metricExecutionResult.getMetricName()); messages.add((isEn ? "Metric" : "检查规则") + " : " + sqlMetric.getNameByLanguage(isEn)); ResultFormula resultFormula = - PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(metricExecutionResult.getResultFormula()); + PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(metricExecutionResult.getResultFormula()); messages.add((isEn ? "Check Subject" : "检查目标") + " : " + metricExecutionResult.getDatabaseName() + "." + metricExecutionResult.getTableName() + "." + metricExecutionResult.getColumnName()); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin(engineType + "_" + metricExecutionResult.getExpectedType()); + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getOrCreatePlugin(engineType + "_" + metricExecutionResult.getExpectedType()); messages.add((isEn ? "Expected Value Type" : "期望值类型") + " : " + expectedValue.getNameByLanguage(isEn)); String resultFormulaFormat = resultFormula.getResultFormat(isEn)+" ${operator} ${threshold}"; @@ -180,7 +180,7 @@ private String buildAlertMessage(MetricExecutionResult metricExecutionResult, St private String buildAlertSubject(MetricExecutionResult metricExecutionResult, boolean isEn) { String checkSubject = metricExecutionResult.getDatabaseName() + "." + metricExecutionResult.getTableName() + "." + metricExecutionResult.getColumnName(); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(metricExecutionResult.getMetricName()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(metricExecutionResult.getMetricName()); return isEn ? (sqlMetric.getNameByLanguage(true) + " alerting on " + checkSubject) : checkSubject + "在" + sqlMetric.getNameByLanguage(false) + "中异常"; } diff --git a/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java b/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java index c5cd79f48..6f9fa8977 100644 --- a/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java +++ b/datavines-server/src/main/java/io/datavines/server/DataVinesServer.java @@ -33,7 +33,7 @@ import io.datavines.server.dqc.coordinator.runner.JobScheduler; import io.datavines.server.registry.RegistryHolder; import io.datavines.server.utils.SpringApplicationContext; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -89,8 +89,8 @@ private void initializeAndStart() throws Exception { jobExecutionFailover = new JobExecutionFailover(jobExecuteManager); - Registry registry = PluginLoader - .getPluginLoader(Registry.class) + Registry registry = PluginDiscovery.getMultiKeyPluginDiscovery(Registry.class, Registry::getPluginNames) + .getOrCreatePlugin(CommonPropertyUtils .getString(CommonPropertyUtils.REGISTRY_TYPE, CommonPropertyUtils.REGISTRY_TYPE_DEFAULT)); registry.init(CommonPropertyUtils.getProperties()); diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java index 3a38f033f..533c94863 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/DataSourceController.java @@ -31,7 +31,7 @@ import io.datavines.server.repository.entity.catalog.CatalogEntityInstance; import io.datavines.server.repository.service.CatalogEntityInstanceService; import io.datavines.server.repository.service.DataSourceService; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -171,11 +171,11 @@ public Object getConfigJson(@PathVariable String type){ @ApiOperation(value = "get connector type list") @GetMapping(value = "/type/list") public Object getConnectorTypeList() { - Set connectorList = PluginLoader.getPluginLoader(ConnectorFactory.class).getSupportedPlugins(); + Set connectorList = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); connectorList.forEach(it -> { - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(it); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(it); if (connectorFactory.showInFrontend()) { Item item = new Item(it,it); items.add(item); diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/ErrorDataStorageController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/ErrorDataStorageController.java index 7a9c71d57..a1da9437d 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/ErrorDataStorageController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/ErrorDataStorageController.java @@ -29,7 +29,7 @@ import io.datavines.server.api.dto.vo.ErrorDataStorageVO; import io.datavines.server.api.dto.vo.Item; import io.datavines.server.repository.service.ErrorDataStorageService; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; @@ -120,10 +120,10 @@ public Object getConfigJson(@PathVariable String type){ @ApiOperation(value = "get error data storage type list") @GetMapping(value = "/type/list") public Object getErrorDataStorageTypeList() { - Set errorDataStorageList = PluginLoader.getPluginLoader(ConnectorFactory.class).getSupportedPlugins(); + Set errorDataStorageList = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); for (String errorDataStorage : errorDataStorageList) { - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(errorDataStorage); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(errorDataStorage); if (connectorFactory == null) { continue; } diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java index 808cf5139..9c08b88c4 100644 --- a/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java +++ b/datavines-server/src/main/java/io/datavines/server/api/controller/MetricController.java @@ -26,7 +26,7 @@ import io.datavines.server.api.dto.vo.Item; import io.datavines.server.api.dto.vo.MetricConfigItem; import io.datavines.server.api.dto.vo.MetricItem; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.http.MediaType; @@ -46,10 +46,10 @@ public class MetricController { @ApiOperation(value = "get metric list") @GetMapping(value = "/list") public Object getMetricList() { - Set metricList = PluginLoader.getPluginLoader(SqlMetric.class).getSupportedPlugins(); + Set metricList = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); metricList.forEach(it -> { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(it); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(it); if (sqlMetric != null) { Item item = new Item(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext()),it); items.add(item); @@ -62,7 +62,7 @@ public Object getMetricList() { @ApiOperation(value = "get metric list by type") @GetMapping(value = "/list/{type}") public Object getMetricListByType(@PathVariable("type") String type) { - Set metricList = PluginLoader.getPluginLoader(SqlMetric.class).getSupportedPlugins(); + Set metricList = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); JobType jobType = JobType.of(type); if (jobType == null) { @@ -72,7 +72,7 @@ public Object getMetricListByType(@PathVariable("type") String type) { switch (jobType) { case DATA_QUALITY: metricList.forEach(it -> { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(it); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(it); if (sqlMetric != null && sqlMetric.getType().isSingleTable()) { if (!excludeMetricSet.contains(it)) { MetricItem item = new MetricItem(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext()), it, sqlMetric.getLevel().getDescription()); @@ -83,7 +83,7 @@ public Object getMetricListByType(@PathVariable("type") String type) { break; case DATA_RECONCILIATION: metricList.forEach(it -> { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(it); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(it); if (sqlMetric != null && !sqlMetric.getType().isSingleTable()) { MetricItem item = new MetricItem(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext()), it, sqlMetric.getLevel().getDescription()); items.add(item); @@ -99,11 +99,11 @@ public Object getMetricListByType(@PathVariable("type") String type) { @ApiOperation(value = "get reconciliation metric list") @GetMapping(value = "/reconciliation/list") public Object getReconciliationMetricList() { - Set metricList = PluginLoader.getPluginLoader(SqlMetric.class).getSupportedPlugins(); + Set metricList = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); metricList.forEach(it -> { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(it); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(it); if (sqlMetric != null && !sqlMetric.getType().isSingleTable()) { Item item = new Item(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext()),it); items.add(item); @@ -116,11 +116,11 @@ public Object getReconciliationMetricList() { @ApiOperation(value = "get quality metric list") @GetMapping(value = "/quality/list/{level}") public Object getDataQualityMetricList(@NotNull @PathVariable("level") String level) { - Set metricList = PluginLoader.getPluginLoader(SqlMetric.class).getSupportedPlugins(); + Set metricList = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); metricList.forEach(it -> { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(it); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(it); if (sqlMetric != null && sqlMetric.getType().isSingleTable() && level.equals(sqlMetric.getLevel().getDescription())) { if (!excludeMetricSet.contains(it)) { Item item = new Item(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext()),it); @@ -135,7 +135,7 @@ public Object getDataQualityMetricList(@NotNull @PathVariable("level") String le @ApiOperation(value = "get metric info") @GetMapping(value = "/configs/{name}") public Object getMetricConfig(@PathVariable("name") String name) { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(name); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(name); if (sqlMetric != null) { Map resultSet = sqlMetric.getConfigMap(); List items = new ArrayList<>(); @@ -152,7 +152,7 @@ public Object getMetricConfig(@PathVariable("name") String name) { @ApiOperation(value = "get expected value list") @GetMapping(value = "/expectedValue/list/{type}") public Object getExpectedTypeList(@PathVariable("type") String type) { - Set expectedValueList = PluginLoader.getPluginLoader(ExpectedValue.class).getSupportedPlugins(); + Set expectedValueList = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getSupportedPlugins(); Set afterFilterSet = new HashSet<>(); afterFilterSet = expectedValueList.stream() .map(it ->it.replace("local_", "") @@ -179,7 +179,7 @@ public Object getExpectedTypeList(@PathVariable("type") String type) { } afterFilterSet.forEach(it -> { - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin("local_" + it); + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getOrCreatePlugin("local_" + it); if (expectedValue != null) { Item item = new Item(expectedValue.getNameByLanguage(!LanguageUtils.isZhContext()),it); items.add(item); @@ -192,7 +192,7 @@ public Object getExpectedTypeList(@PathVariable("type") String type) { @ApiOperation(value = "get engine type list") @GetMapping(value = "/engine/list") public Object getEngineTypeList() { - Set engineTypeList = PluginLoader.getPluginLoader(EngineExecutor.class).getSupportedPlugins(); + Set engineTypeList = PluginDiscovery.getMultiKeyPluginDiscovery(EngineExecutor.class, EngineExecutor::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); engineTypeList.forEach(it -> { Item item = new Item(it,it); @@ -205,10 +205,10 @@ public Object getEngineTypeList() { @ApiOperation(value = "get result formula list") @GetMapping(value = "/resultFormula/list") public Object getResultFormulaList() { - Set resultFormulaTypeList = PluginLoader.getPluginLoader(ResultFormula.class).getSupportedPlugins(); + Set resultFormulaTypeList = PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getSupportedPlugins(); List items = new ArrayList<>(); resultFormulaTypeList.forEach(it -> { - ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(it); + ResultFormula resultFormula = PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(it); if (resultFormula != null) { Item item = new Item(resultFormula.getNameByLanguage(!LanguageUtils.isZhContext()),it); items.add(item); diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java index 947cf8abf..920b4f329 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/coordinator/validator/JobResultValidator.java @@ -36,7 +36,7 @@ import io.datavines.server.repository.entity.JobExecutionResult; import io.datavines.server.repository.service.*; import io.datavines.server.repository.service.impl.JobExternalService; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -185,15 +185,15 @@ private String buildAlertMessage(List messages, MetricExecutionResult me parameters.put("threshold", String.valueOf(metricExecutionResult.getThreshold())); parameters.put("operator",OperatorType.of(metricExecutionResult.getOperator()).getSymbol()); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(metricExecutionResult.getMetricName()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(metricExecutionResult.getMetricName()); messages.add((isEn ? "Metric" : "检查规则") + " : " + sqlMetric.getNameByLanguage(isEn)); ResultFormula resultFormula = - PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(metricExecutionResult.getResultFormula()); + PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(metricExecutionResult.getResultFormula()); messages.add((isEn ? "Check Subject" : "检查目标") + " : " + metricExecutionResult.getDatabaseName() + "." + metricExecutionResult.getTableName() + "." + metricExecutionResult.getColumnName()); - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin(engineType + "_" + metricExecutionResult.getExpectedType()); + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getOrCreatePlugin(engineType + "_" + metricExecutionResult.getExpectedType()); messages.add((isEn ? "Expected Value Type" : "期望值类型") + " : " + expectedValue.getNameByLanguage(isEn)); String resultFormulaFormat = resultFormula.getResultFormat(isEn)+" ${operator} ${threshold}"; @@ -210,7 +210,7 @@ private String buildAlertMessage(List messages, MetricExecutionResult me private String buildAlertSubject(MetricExecutionResult metricExecutionResult, boolean isEn) { String checkSubject = metricExecutionResult.getDatabaseName() + "." + metricExecutionResult.getTableName() + "." + metricExecutionResult.getColumnName(); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(metricExecutionResult.getMetricName()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(metricExecutionResult.getMetricName()); return isEn ? (sqlMetric.getNameByLanguage(true) + "alerting on " + checkSubject) : checkSubject + "在" + sqlMetric.getNameByLanguage(false) + "中异常了"; } diff --git a/datavines-server/src/main/java/io/datavines/server/dqc/executor/runner/JobRunner.java b/datavines-server/src/main/java/io/datavines/server/dqc/executor/runner/JobRunner.java index bd5b450da..c36c0695c 100644 --- a/datavines-server/src/main/java/io/datavines/server/dqc/executor/runner/JobRunner.java +++ b/datavines-server/src/main/java/io/datavines/server/dqc/executor/runner/JobRunner.java @@ -25,7 +25,7 @@ import io.datavines.engine.api.engine.EngineExecutor; import io.datavines.server.dqc.command.JobExecuteResponseCommand; import io.datavines.server.dqc.coordinator.cache.JobExecutionResponseProcessor; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,8 +59,8 @@ public void run() { Logger taskLogger = LoggerFactory.getLogger(taskLoggerName); Thread.currentThread().setName(taskLoggerName); - engineExecutor = PluginLoader - .getPluginLoader(EngineExecutor.class) + engineExecutor = PluginDiscovery.getMultiKeyPluginDiscovery(EngineExecutor.class, EngineExecutor::getPluginNames) + .getNewPlugin(jobExecutionRequest.getEngineType()); engineExecutor.init(jobExecutionRequest, taskLogger, configurations); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityRelServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityRelServiceImpl.java index 4e9d1b7bb..91e3514d9 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityRelServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/CatalogEntityRelServiceImpl.java @@ -42,7 +42,7 @@ import io.datavines.server.repository.service.CatalogEntityRelService; import io.datavines.server.repository.service.DataSourceService; import io.datavines.server.utils.ContextHolder; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import org.jetbrains.annotations.NotNull; import org.springframework.beans.BeanUtils; @@ -160,7 +160,7 @@ public boolean addLineageByParseSql(SqlWithDataSourceList sqlWithDataSourceList) StringBuilder errors = new StringBuilder(); for (DataSourceInfo dataSourceInfo: dataSourceInfos) { - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(dataSourceInfo.getType()); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(dataSourceInfo.getType()); if (connectorFactory == null) { errors.append("Unsupported datasource type: ").append(dataSourceInfo.getType()).append("; "); continue; diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java index c66809d32..39b08562b 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/DataSourceServiceImpl.java @@ -39,7 +39,7 @@ import io.datavines.server.repository.service.*; import io.datavines.core.exception.DataVinesServerException; import io.datavines.server.utils.ContextHolder; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -74,7 +74,7 @@ public class DataSourceServiceImpl extends ServiceImpl keyProperties = connectorFactory.getConnector().keyProperties(); List keyPropertyValueList = new ArrayList<>(); keyPropertyValueList.add(dataSourceCreate.getType().toLowerCase()); @@ -166,7 +166,7 @@ public int update(DataSourceUpdate dataSourceUpdate) throws DataVinesException { String type = dataSourceUpdate.getType(); - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(type); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(type); List keyProperties = connectorFactory.getConnector().keyProperties(); List keyPropertyValueList = new ArrayList<>(); keyPropertyValueList.add(dataSourceUpdate.getType().toLowerCase()); @@ -271,7 +271,7 @@ public List listByInfo(DataSourceKeyProperties dataSourceKeyProp Map paramMap = dataSourceKeyProperties.getParam(); String type = dataSourceKeyProperties.getType(); String paramCode = ""; - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(type); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(type); List keyProperties = connectorFactory.getConnector().keyProperties(); List keyPropertyValueList = new ArrayList<>(); keyPropertyValueList.add(type.toLowerCase()); @@ -315,7 +315,7 @@ public Object getDatabaseList(Long id) throws DataVinesServerException { param.setDataSourceParam(dataSource.getParam()); Object result = null; - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(param.getType()); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(param.getType()); try { ConnectorResponse response = connectorFactory.getConnector().getDatabases(param); result = response.getResult(); @@ -336,7 +336,7 @@ public Object getTableList(Long id, String database) throws DataVinesServerExcep param.setDatabase(database); Object result = null; - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(param.getType()); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(param.getType()); try { ConnectorResponse response = connectorFactory.getConnector().getTables(param); result = response.getResult(); @@ -358,7 +358,7 @@ public Object getColumnList(Long id, String database, String table) throws DataV param.setTable(table); Object result = null; - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(param.getType()); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(param.getType()); try { ConnectorResponse response = connectorFactory.getConnector().getColumns(param); result = response.getResult(); @@ -378,7 +378,7 @@ public Object executeScript(ExecuteRequest request) throws DataVinesServerExcept param.setDataSourceParam(dataSource.getParam()); param.setScript(request.getScript()); Object result = null; - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(param.getType()); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(param.getType()); try { ConnectorResponse response = connectorFactory.getExecutor().queryForList(param); result = response.getResult(); @@ -392,6 +392,6 @@ public Object executeScript(ExecuteRequest request) throws DataVinesServerExcept @Override public String getConfigJson(String type) { - return PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(type).getConfigBuilder().build(!LanguageUtils.isZhContext()); + return PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(type).getConfigBuilder().build(!LanguageUtils.isZhContext()); } } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java index eef799a12..4d03ffba3 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/ErrorDataStorageServiceImpl.java @@ -35,7 +35,7 @@ import io.datavines.server.repository.mapper.ErrorDataStorageMapper; import io.datavines.server.repository.service.ErrorDataStorageService; import io.datavines.server.utils.ContextHolder; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -52,7 +52,7 @@ public class ErrorDataStorageServiceImpl extends ServiceImpl jobParameterList = JSONUtils.toList(job.getParameter(),BaseJobParameter.class); for (BaseJobParameter jobParameter : jobParameterList) { if (jobParameter != null) { - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(jobParameter.getMetricType()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(jobParameter.getMetricType()); Map configMap = sqlMetric.getConfigMap(); Map paramMap = new HashMap<>(); String uniqueName = jobParameter.getMetricType() + "." @@ -170,7 +170,7 @@ private JobExecutionResultVO generateJobExecutionResultVO(long jobExecutionId, J } ResultFormula resultFormula = - PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(jobExecutionResult.getResultFormula()); + PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(jobExecutionResult.getResultFormula()); String resultFormulaFormat = resultFormula.getResultFormat(!LanguageUtils.isZhContext())+" ${operator} ${threshold}"; String checkSubject = jobExecutionResult.getDatabaseName() + "." + jobExecutionResult.getTableName(); @@ -179,9 +179,9 @@ private JobExecutionResultVO generateJobExecutionResultVO(long jobExecutionId, J } jobExecutionResultVO.setCheckSubject(checkSubject); jobExecutionResultVO.setCheckResult(JobCheckState.of(jobExecutionResult.getState()).getDescription(!LanguageUtils.isZhContext())); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(jobExecutionResult.getMetricName()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(jobExecutionResult.getMetricName()); if (!"multi_table_value_comparison".equalsIgnoreCase(sqlMetric.getName())) { - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin(jobExecution.getEngineType() + "_" + jobExecutionResult.getExpectedType()); + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getOrCreatePlugin(jobExecution.getEngineType() + "_" + jobExecutionResult.getExpectedType()); jobExecutionResultVO.setExpectedType(expectedValue.getNameByLanguage(!LanguageUtils.isZhContext())); } jobExecutionResultVO.setMetricName(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext())); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java index e4f781c2b..37fb35f63 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobExecutionServiceImpl.java @@ -44,7 +44,7 @@ import io.datavines.server.repository.service.*; import io.datavines.server.repository.entity.Command; import io.datavines.server.repository.mapper.JobExecutionMapper; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.springframework.beans.BeanUtils; @@ -241,19 +241,19 @@ public List listJobExecutionNotInServerList(List hostList) private void checkJobExecutionParameter(JobExecutionParameter jobExecutionParameter, String engineType) throws DataVinesServerException { // String metricType = jobExecutionParameter.getMetricType(); -// Set metricPluginSet = PluginLoader.getPluginLoader(SqlMetric.class).getSupportedPlugins(); +// Set metricPluginSet = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getSupportedPlugins(); // if (!metricPluginSet.contains(metricType)) { // throw new DataVinesServerException(String.format("%s metric does not supported", metricType)); // } // -// SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(metricType); +// SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(metricType); // CheckResult checkResult = sqlMetric.validateConfig(jobExecutionParameter.getMetricParameter()); // if (checkResult== null || !checkResult.isSuccess()) { // throw new DataVinesServerException(checkResult== null? "check error": checkResult.getMsg()); // } // // String configBuilder = engineType + "_" + sqlMetric.getType().getDescription(); -// Set configBuilderPluginSet = PluginLoader.getPluginLoader(JobConfigurationBuilder.class).getSupportedPlugins(); +// Set configBuilderPluginSet = PluginDiscovery.getMultiKeyPluginDiscovery(JobConfigurationBuilder.class, JobConfigurationBuilder::getPluginNames).getSupportedPlugins(); // if (!configBuilderPluginSet.contains(configBuilder)) { // throw new DataVinesServerException(String.format("%s engine does not supported %s metric", engineType, metricType)); // } @@ -262,13 +262,13 @@ private void checkJobExecutionParameter(JobExecutionParameter jobExecutionParame // if (connectorParameter != null) { // String connectorType = connectorParameter.getType(); // Set connectorFactoryPluginSet = -// PluginLoader.getPluginLoader(ConnectorFactory.class).getSupportedPlugins(); +// PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getSupportedPlugins(); // if (!connectorFactoryPluginSet.contains(connectorType)) { // throw new DataVinesServerException(String.format("%s connector does not supported", connectorType)); // } // // if (LOCAL.equals(engineType)) { -// ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(connectorType); +// ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(connectorType); // if (!JDBC.equals(connectorFactory.getCategory())) { // throw new DataVinesServerException(String.format("jdbc engine does not supported %s connector", connectorType)); // } @@ -278,13 +278,13 @@ private void checkJobExecutionParameter(JobExecutionParameter jobExecutionParame // } // // String expectedMetric = jobExecutionParameter.getExpectedType(); -// Set expectedValuePluginSet = PluginLoader.getPluginLoader(ExpectedValue.class).getSupportedPlugins(); +// Set expectedValuePluginSet = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getSupportedPlugins(); // if (!expectedValuePluginSet.contains(expectedMetric)) { // throw new DataVinesServerException(String.format("%s expected value does not supported", metricType)); // } // // String resultFormula = jobExecutionParameter.getResultFormula(); -// Set resultFormulaPluginSet = PluginLoader.getPluginLoader(ResultFormula.class).getSupportedPlugins(); +// Set resultFormulaPluginSet = PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getSupportedPlugins(); // if (!resultFormulaPluginSet.contains(resultFormula)) { // throw new DataVinesServerException(String.format("%s result formula does not supported", metricType)); // } @@ -324,7 +324,7 @@ public List getMetricExecutionDashBoard(Long jobId, St executionResults.forEach(result -> { ResultFormula resultFormula = - PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(result.getResultFormula()); + PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(result.getResultFormula()); MetricExecutionDashBoard executionDashBoard = new MetricExecutionDashBoard(); executionDashBoard.setValue(resultFormula.getResult(result.getActualValue(), Objects.isNull(result.getExpectedValue()) ? BigDecimal.valueOf(0) : result.getExpectedValue())); executionDashBoard.setType(resultFormula.getType().getDescription()); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java index 11de2a70d..ec1862bec 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobQualityReportServiceImpl.java @@ -37,7 +37,7 @@ import io.datavines.server.repository.service.JobExecutionService; import io.datavines.server.repository.service.JobQualityReportService; import io.datavines.server.repository.service.JobExecutionResultReportRelService; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.springframework.beans.BeanUtils; @@ -415,14 +415,14 @@ public List listColumnExecution(Long reportId) { JobExecution jobExecution = jobExecutionService.getById(jobExecutionResult.getJobExecutionId()); JobExecutionResultVO jobExecutionResultVO = new JobExecutionResultVO(); ResultFormula resultFormula = - PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(jobExecutionResult.getResultFormula()); + PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(jobExecutionResult.getResultFormula()); String resultFormulaFormat = resultFormula.getResultFormat(!LanguageUtils.isZhContext())+" ${operator} ${threshold}"; jobExecutionResultVO.setCheckSubject(jobExecutionResult.getDatabaseName() + "." + jobExecutionResult.getTableName() + "." + jobExecutionResult.getColumnName()); jobExecutionResultVO.setCheckResult(JobCheckState.of(jobExecutionResult.getState()).getDescription(!LanguageUtils.isZhContext())); - SqlMetric sqlMetric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(jobExecutionResult.getMetricName()); + SqlMetric sqlMetric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(jobExecutionResult.getMetricName()); if (!"multi_table_value_comparison".equalsIgnoreCase(sqlMetric.getName())) { - ExpectedValue expectedValue = PluginLoader.getPluginLoader(ExpectedValue.class).getOrCreatePlugin(jobExecution.getEngineType() + "_" + jobExecutionResult.getExpectedType()); + ExpectedValue expectedValue = PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames).getOrCreatePlugin(jobExecution.getEngineType() + "_" + jobExecutionResult.getExpectedType()); jobExecutionResultVO.setExpectedType(expectedValue.getNameByLanguage(!LanguageUtils.isZhContext())); } jobExecutionResultVO.setMetricName(sqlMetric.getNameByLanguage(!LanguageUtils.isZhContext())); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobServiceImpl.java index be84afcb8..f57a28c0c 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/JobServiceImpl.java @@ -63,7 +63,7 @@ import io.datavines.server.utils.ContextHolder; import io.datavines.server.utils.DefaultDataSourceInfoUtils; import io.datavines.server.utils.JobParameterUtils; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.CollectionUtils; @@ -138,7 +138,7 @@ public long create(JobCreate jobCreate) throws DataVinesServerException { DataSource dataSource = dataSourceService.getDataSourceById(jobCreate.getDataSourceId()); if (dataSource != null) { String errorDataStorageType = dataSource.getType(); - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(errorDataStorageType); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(errorDataStorageType); if (connectorFactory == null || !connectorFactory.getDialect().invalidateItemCanOutputToSelf()) { throw new DataVinesServerException(Status.DATASOURCE_NOT_SUPPORT_ERROR_DATA_OUTPUT_TO_SELF_ERROR, errorDataStorageType); } @@ -210,7 +210,7 @@ public long update(JobUpdate jobUpdate) { DataSource dataSource = dataSourceService.getDataSourceById(job.getDataSourceId()); if (dataSource != null) { String errorDataStorageType = dataSource.getType(); - ConnectorFactory connectorFactory = PluginLoader.getPluginLoader(ConnectorFactory.class).getOrCreatePlugin(errorDataStorageType); + ConnectorFactory connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames).getOrCreatePlugin(errorDataStorageType); if (connectorFactory == null || !connectorFactory.getDialect().invalidateItemCanOutputToSelf()) { throw new DataVinesServerException(Status.DATASOURCE_NOT_SUPPORT_ERROR_DATA_OUTPUT_TO_SELF_ERROR, errorDataStorageType); } @@ -365,7 +365,7 @@ private void isMetricSuitable(long datasourceId, long datasourceId2, String engi throw new DataVinesServerException(Status.ENTITY_TYPE_NOT_EXIST); } - SqlMetric metric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(jobParameter.getMetricType()); + SqlMetric metric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(jobParameter.getMetricType()); if (metric == null) { throw new DataVinesServerException(Status.METRIC_JOB_RELATED_ENTITY_NOT_EXIST, jobParameter.getMetricType().toUpperCase()); } @@ -620,7 +620,7 @@ public String getJobName(String jobType, String parameter) { throw new DataVinesServerException(Status.JOB_PARAMETER_IS_NULL_ERROR); } - ResultFormula resultFormula = PluginLoader.getPluginLoader(ResultFormula.class).getOrCreatePlugin(baseJobParameter.getResultFormula()); + ResultFormula resultFormula = PluginDiscovery.getMultiKeyPluginDiscovery(ResultFormula.class, ResultFormula::getPluginNames).getOrCreatePlugin(baseJobParameter.getResultFormula()); String database = (String)metricParameter.get(DATABASE); String table = (String)metricParameter.get(TABLE); diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaNotificationServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaNotificationServiceImpl.java index 92cea1a87..19fbfa3b8 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaNotificationServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaNotificationServiceImpl.java @@ -35,7 +35,7 @@ import io.datavines.server.repository.service.SlaNotificationService; import io.datavines.server.repository.service.SlaSenderService; import io.datavines.server.utils.ContextHolder; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -122,8 +122,8 @@ public Set listReceiverMessageBySlaId(Long id){ @Override public String getConfigJson(String type) { - return PluginLoader - .getPluginLoader(SlasHandlerPlugin.class) + return PluginDiscovery.getMultiKeyPluginDiscovery(SlasHandlerPlugin.class, SlasHandlerPlugin::getPluginNames) + .getOrCreatePlugin(type) .getConfigJson(); } diff --git a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaServiceImpl.java b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaServiceImpl.java index 5c8ae86b5..e8a5bc432 100644 --- a/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaServiceImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/repository/service/impl/SlaServiceImpl.java @@ -37,7 +37,7 @@ import io.datavines.server.repository.service.SlaJobService; import io.datavines.server.repository.service.SlaService; import io.datavines.server.utils.ContextHolder; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -74,16 +74,16 @@ public boolean deleteById(Long id) { @Override public String getSenderConfigJson(String type) { - return PluginLoader - .getPluginLoader(SlasHandlerPlugin.class) + return PluginDiscovery.getMultiKeyPluginDiscovery(SlasHandlerPlugin.class, SlasHandlerPlugin::getPluginNames) + .getOrCreatePlugin(type) .getConfigSenderJson(); } @Override public Set getSupportPlugin(){ - Set supportedPlugins = PluginLoader - .getPluginLoader(SlasHandlerPlugin.class) + Set supportedPlugins = PluginDiscovery.getMultiKeyPluginDiscovery(SlasHandlerPlugin.class, SlasHandlerPlugin::getPluginNames) + .getSupportedPlugins(); return supportedPlugins; } diff --git a/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java index 5667ca292..606e1e44e 100644 --- a/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java +++ b/datavines-server/src/main/java/io/datavines/server/scheduler/metadata/task/CatalogMetaDataFetchExecutorImpl.java @@ -42,7 +42,7 @@ import io.datavines.server.repository.service.CatalogSchemaChangeService; import io.datavines.server.scheduler.CommonTaskRequest; import io.datavines.server.utils.SpringApplicationContext; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -79,8 +79,8 @@ public CatalogMetaDataFetchExecutorImpl(CommonTaskRequest request) { this.dataSource = request.getDataSource(); - this.connectorFactory = PluginLoader - .getPluginLoader(ConnectorFactory.class) + this.connectorFactory = PluginDiscovery.getMultiKeyPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginNames) + .getOrCreatePlugin(dataSource.getType()); this.instanceService = SpringApplicationContext.getBean(CatalogEntityInstanceService.class); diff --git a/datavines-server/src/main/java/io/datavines/server/utils/JobParameterUtils.java b/datavines-server/src/main/java/io/datavines/server/utils/JobParameterUtils.java index 4bb3790a0..cf4512b3f 100644 --- a/datavines-server/src/main/java/io/datavines/server/utils/JobParameterUtils.java +++ b/datavines-server/src/main/java/io/datavines/server/utils/JobParameterUtils.java @@ -18,7 +18,7 @@ import io.datavines.common.entity.job.BaseJobParameter; import io.datavines.metric.api.SqlMetric; -import io.datavines.spi.PluginLoader; +import io.datavines.spi.PluginDiscovery; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.BeanUtils; @@ -31,7 +31,7 @@ public class JobParameterUtils { public static List regenerateJobParameterList(List jobParameters) { List result = new ArrayList<>(); for (BaseJobParameter jobParameter : jobParameters) { - SqlMetric metric = PluginLoader.getPluginLoader(SqlMetric.class).getOrCreatePlugin(jobParameter.getMetricType()); + SqlMetric metric = PluginDiscovery.getMultiKeyPluginDiscovery(SqlMetric.class, SqlMetric::getPluginNames).getOrCreatePlugin(jobParameter.getMetricType()); if (metric == null) { continue; } diff --git a/datavines-spi/pom.xml b/datavines-spi/pom.xml index 75853fa26..0ed3a36ec 100644 --- a/datavines-spi/pom.xml +++ b/datavines-spi/pom.xml @@ -31,20 +31,10 @@ datavines-spi - - org.apache.commons - commons-lang3 - - org.slf4j slf4j-api - - - org.apache.commons - commons-collections4 - \ No newline at end of file diff --git a/datavines-spi/src/main/java/io/datavines/spi/CachingFactory.java b/datavines-spi/src/main/java/io/datavines/spi/CachingFactory.java new file mode 100644 index 000000000..0103c18b4 --- /dev/null +++ b/datavines-spi/src/main/java/io/datavines/spi/CachingFactory.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.spi; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +/** + * 线程安全的带缓存实例工厂。 + * + *

适用于"无配置实例全局缓存、有配置实例每次新建"的场景。 + * + * @param Key 类型 + * @param 实例类型 + */ +public final class CachingFactory { + + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + private final Function creator; + + public CachingFactory(Function creator) { + this.creator = creator; + } + + /** + * 获取缓存实例。同一个 Key 始终返回同一个实例。 + */ + public T get(K key) { + return cache.computeIfAbsent(key, creator); + } + + /** + * 创建新实例，不走缓存。 + */ + public T create(K key) { + return creator.apply(key); + } + + /** + * 返回已加载（已创建并进入缓存）的 Key 集合。 + */ + public Set loadedKeys() { + return Collections.unmodifiableSet(new TreeSet<>(cache.keySet())); + } + + /** + * 返回已加载实例，不触发新的创建。 + */ + public List loadedInstances() { + return Collections.unmodifiableList(new ArrayList<>(cache.values())); + } + + /** + * 返回指定 Key 的已加载实例；如果尚未创建则返回 null，不触发加载。 + */ + public T getIfLoaded(K key) { + return cache.get(key); + } + + /** + * 判断指定 Key 是否已经创建并进入缓存。 + */ + public boolean isLoaded(K key) { + return cache.containsKey(key); + } + + /** + * 清除所有缓存。 + */ + public void clear() { + cache.clear(); + } + + /** + * 清除指定 Key 的缓存。 + */ + public void evict(K key) { + cache.remove(key); + } + + public int cacheSize() { + return cache.size(); + } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/utils/Holder.java b/datavines-spi/src/main/java/io/datavines/spi/DuplicateProviderException.java similarity index 51% rename from datavines-spi/src/main/java/io/datavines/spi/utils/Holder.java rename to datavines-spi/src/main/java/io/datavines/spi/DuplicateProviderException.java index 787c6350c..61b1cc941 100644 --- a/datavines-spi/src/main/java/io/datavines/spi/utils/Holder.java +++ b/datavines-spi/src/main/java/io/datavines/spi/DuplicateProviderException.java @@ -14,21 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.spi.utils; +package io.datavines.spi; /** - * Helper Class for hold a value. + * 同一个 Key 被多个 Provider 声明时抛出。 */ -public class Holder { +public class DuplicateProviderException extends RuntimeException { - private volatile T value; + private final String registryName; + private final Object key; - public void set(T value) { - this.value = value; + public DuplicateProviderException(String registryName, Object key, + String existingClass, String conflictingClass) { + super(String.format( + "[%s] Duplicate provider for key '%s': [%s] and [%s]. " + + "Check classpath for duplicate JARs.", + registryName, key, existingClass, conflictingClass)); + this.registryName = registryName; + this.key = key; } - public T get() { - return value; - } - -} \ No newline at end of file + public String getRegistryName() { return registryName; } + public Object getKey() { return key; } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/KeyedRegistry.java b/datavines-spi/src/main/java/io/datavines/spi/KeyedRegistry.java new file mode 100644 index 000000000..97c069841 --- /dev/null +++ b/datavines-spi/src/main/java/io/datavines/spi/KeyedRegistry.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.spi; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * 唯一型注册表：一个 Key 有且仅有一个 Provider。 + * + *

初始化完成后不可变，天然线程安全。 + * + * @param Key 类型 + * @param

Provider 类型 + */ +public final class KeyedRegistry { + + private static final Logger log = LoggerFactory.getLogger(KeyedRegistry.class); + + private final Map providers; + private final String registryName; + + private KeyedRegistry(Map providers, String registryName) { + this.providers = Collections.unmodifiableMap(providers); + this.registryName = registryName; + } + + /** + * 从 ServiceLoader 加载所有 Provider 并构建注册表。 + * + * @param providerType Provider 的 SPI 接口类型 + * @param keyExtractor 从 Provider 中提取 Key 的函数 + * @return 不可变的注册表 + * @throws DuplicateProviderException 同一个 Key 被多个 Provider 声明 + */ + public static KeyedRegistry load( + Class

providerType, + Function keyExtractor) { + return load(providerType, keyExtractor, providerType.getSimpleName()); + } + + /** + * 从 ServiceLoader 加载，指定注册表名称（用于日志和异常信息）。 + */ + public static KeyedRegistry load( + Class

providerType, + Function keyExtractor, + String registryName) { + return from(ServiceLoaderUtils.loadAll(providerType), keyExtractor, registryName); + } + + /** + * 从显式 Provider 列表构建注册表（测试和手动组装场景）。 + */ + @SafeVarargs + public static KeyedRegistry of( + Function keyExtractor, + P... providers) { + return from(Arrays.asList(providers), keyExtractor, "test"); + } + + /** + * 从可迭代的 Provider 集合构建注册表。 + * + * @param providers Provider 集合 + * @param keyExtractor Key 提取函数 + * @param registryName 注册表名称（用于日志和异常信息） + * @return 不可变的注册表 + * @throws DuplicateProviderException 同一个 Key 被多个 Provider 声明 + */ + public static KeyedRegistry from( + Iterable

providers, + Function keyExtractor, + String registryName) { + + Map map = new LinkedHashMap<>(); + + for (P provider : providers) { + K key = keyExtractor.apply(provider); + Objects.requireNonNull(key, + () -> "Provider " + provider.getClass().getName() + + " returned null key in registry [" + registryName + "]"); + + P existing = map.put(key, provider); + if (existing != null) { + throw new DuplicateProviderException( + registryName, key, + existing.getClass().getName(), + provider.getClass().getName()); + } + + log.info("[{}] Registered: {} -> {}", + registryName, key, provider.getClass().getName()); + } + + log.info("[{}] Initialized with {} providers: {}", + registryName, map.size(), map.keySet()); + + return new KeyedRegistry<>(map, registryName); + } + + /** + * 从 ServiceLoader 加载，每个 Provider 可以注册多个 Key。 + * 适用于同一个类注册多个 engine-prefixed 名称的场景。 + */ + public static KeyedRegistry loadMultiKey( + Class

providerType, + Function> keysExtractor, + String registryName) { + return fromMultiKey(ServiceLoaderUtils.loadAll(providerType), keysExtractor, registryName); + } + + /** + * 从 Provider 集合构建注册表，每个 Provider 注册多个 Key。 + */ + public static KeyedRegistry fromMultiKey( + Iterable

providers, + Function> keysExtractor, + String registryName) { + + Map map = new LinkedHashMap<>(); + + for (P provider : providers) { + Collection keys = keysExtractor.apply(provider); + if (keys == null || keys.isEmpty()) { + log.warn("[{}] Provider {} returned empty keys, skipping", + registryName, provider.getClass().getName()); + continue; + } + + for (K key : keys) { + Objects.requireNonNull(key, + () -> "Provider " + provider.getClass().getName() + + " returned null key in registry [" + registryName + "]"); + + P existing = map.put(key, provider); + if (existing != null) { + throw new DuplicateProviderException( + registryName, key, + existing.getClass().getName(), + provider.getClass().getName()); + } + + log.info("[{}] Registered: {} -> {}", + registryName, key, provider.getClass().getName()); + } + } + + log.info("[{}] Initialized with {} keys from providers", + registryName, map.size()); + + return new KeyedRegistry<>(map, registryName); + } + + /** + * 获取指定 Key 对应的 Provider。 + * + * @throws ProviderNotFoundException 如果 Key 不存在 + */ + public P get(K key) { + P provider = providers.get(key); + if (provider == null) { + throw new ProviderNotFoundException(registryName, key, providers.keySet()); + } + return provider; + } + + /** + * 查找指定 Key 对应的 Provider，不存在时返回 null。 + */ + public P find(K key) { + return providers.get(key); + } + + public boolean supports(K key) { + return providers.containsKey(key); + } + + public Set keys() { + return providers.keySet(); + } + + /** + * 返回所有已支持的 Key。 + */ + public Set getSupportedKeys() { + return keys(); + } + + /** + * 返回所有已注册 Provider。 + */ + public Collection

providers() { + return providers.values(); + } + + /** + * 返回 Key 到 Provider 的只读映射。 + */ + public Map asMap() { + return providers; + } + + public int size() { + return providers.size(); + } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/OrderedRegistry.java b/datavines-spi/src/main/java/io/datavines/spi/OrderedRegistry.java new file mode 100644 index 000000000..844babd6d --- /dev/null +++ b/datavines-spi/src/main/java/io/datavines/spi/OrderedRegistry.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.spi; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * 列表型注册表：多个 Provider 共存，按优先级排序。 + * + *

初始化完成后不可变，天然线程安全。 + * + * @param

Provider 类型 + */ +public final class OrderedRegistry

{ + + private static final Logger log = LoggerFactory.getLogger(OrderedRegistry.class); + + private final List

providers; + + private OrderedRegistry(List

providers) { + this.providers = Collections.unmodifiableList(providers); + } + + /** + * 从 ServiceLoader 加载并按指定比较器排序。 + */ + public static

OrderedRegistry

load( + Class

providerType, + Comparator

ordering) { + return from(ServiceLoaderUtils.loadAll(providerType), ordering, providerType.getSimpleName()); + } + + /** + * 从显式列表构建（测试场景）。 + */ + @SafeVarargs + public static

OrderedRegistry

of(Comparator

ordering, P... providers) { + return from(Arrays.asList(providers), ordering, "test"); + } + + /** + * 从可迭代集合构建。 + */ + public static

OrderedRegistry

from( + Iterable

providers, + Comparator

ordering, + String registryName) { + List

sorted = StreamSupport.stream(providers.spliterator(), false) + .sorted(ordering) + .collect(Collectors.toList()); + + for (int i = 0; i < sorted.size(); i++) { + log.info("[{}] Registered [order={}]: {}", + registryName, i, sorted.get(i).getClass().getName()); + } + + log.info("[{}] Initialized with {} providers", registryName, sorted.size()); + return new OrderedRegistry<>(sorted); + } + + /** + * 获取所有 Provider（按优先级排序）。 + */ + public List

getAll() { + return providers; + } + + public boolean isEmpty() { + return providers.isEmpty(); + } + + public int size() { + return providers.size(); + } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/PluginDiscovery.java b/datavines-spi/src/main/java/io/datavines/spi/PluginDiscovery.java new file mode 100644 index 000000000..164eb7d23 --- /dev/null +++ b/datavines-spi/src/main/java/io/datavines/spi/PluginDiscovery.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.spi; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +/** + * PluginLoader 的替代品，提供与旧 API 兼容的接口。 + * + *

基于 JDK {@link java.util.ServiceLoader} + {@link KeyedRegistry} + {@link CachingFactory} 实现。 + * 支持单名称插件和多名称插件（同一个类注册多个名称）。 + * + *

使用示例： + * 

{@code
+ * // 单名称插件
+ * PluginDiscovery discovery =
+ *     PluginDiscovery.getPluginDiscovery(ConnectorFactory.class, ConnectorFactory::getPluginName);
+ *
+ * // 多名称插件
+ * PluginDiscovery discovery =
+ *     PluginDiscovery.getMultiKeyPluginDiscovery(ExpectedValue.class, ExpectedValue::getPluginNames);
+ *
+ * // 获取或创建缓存的插件实例
+ * ConnectorFactory factory = discovery.getOrCreatePlugin("mysql");
+ * }
+ * + * @param 插件接口类型 + */ +public final class PluginDiscovery { + + private static final ConcurrentHashMap, PluginDiscovery> DISCOVERIES = new ConcurrentHashMap<>(); + + private final Class type; + private final KeyedRegistry registry; + private final CachingFactory singletonFactory; + + private PluginDiscovery(Class type, KeyedRegistry registry) { + this.type = type; + this.registry = registry; + this.singletonFactory = new CachingFactory<>(registry::get); + } + + /** + * 获取单名称插件的 PluginDiscovery 实例。 + * 每个插件类只注册一个名称。 + */ + @SuppressWarnings("unchecked") + public static PluginDiscovery getPluginDiscovery(Class type, Function keyExtractor) { + return (PluginDiscovery) DISCOVERIES.computeIfAbsent(type, + k -> { + KeyedRegistry reg = KeyedRegistry.load(type, keyExtractor, type.getSimpleName()); + return new PluginDiscovery<>(type, reg); + }); + } + + /** + * 获取多名称插件的 PluginDiscovery 实例。 + * 每个插件类可以注册多个名称（如 engine-prefixed names）。 + */ + @SuppressWarnings("unchecked") + public static PluginDiscovery getMultiKeyPluginDiscovery(Class type, Function> keysExtractor) { + return (PluginDiscovery) DISCOVERIES.computeIfAbsent(type, + k -> { + KeyedRegistry reg = KeyedRegistry.loadMultiKey(type, keysExtractor, type.getSimpleName()); + return new PluginDiscovery<>(type, reg); + }); + } + + /** + * 获取或创建缓存的插件实例（单例模式）。 + * 等价于旧 PluginLoader.getOrCreatePlugin(name)。 + */ + public T getOrCreatePlugin(String name) { + return singletonFactory.get(name); + } + + /** + * 每次创建新实例。 + * 等价于旧 PluginLoader.getNewPlugin(name)。 + */ + public T getNewPlugin(String name) { + T provider = registry.get(name); + try { + @SuppressWarnings("unchecked") + T newInstance = (T) provider.getClass().newInstance(); + return newInstance; + } catch (Exception e) { + throw new IllegalStateException( + "Plugin instance (name: " + name + ", class: " + type.getName() + + ") couldn't be instantiated: " + e.getMessage(), e); + } + } + + /** + * 获取已加载的插件实例，未加载则返回 null。 + * 等价于旧 PluginLoader.getLoadedPlugin(name)。 + */ + public T getLoadedPlugin(String name) { + return singletonFactory.getIfLoaded(name); + } + + /** + * 判断是否支持指定名称的插件。 + * 等价于旧 PluginLoader.hasPlugin(name)。 + */ + public boolean hasPlugin(String name) { + return registry.supports(name); + } + + /** + * 返回所有支持的插件名称。 + * 等价于旧 PluginLoader.getSupportedPlugins()。 + */ + public Set getSupportedPlugins() { + return registry.getSupportedKeys(); + } + + /** + * 获取所有支持的插件实例（按需创建并缓存）。 + * 等价于旧 PluginLoader.getSupportedPluginInstances()。 + */ + public Set getSupportedPluginInstances() { + Set instances = new LinkedHashSet<>(); + for (String name : getSupportedPlugins()) { + instances.add(getOrCreatePlugin(name)); + } + return Collections.unmodifiableSet(instances); + } + + /** + * 返回已加载的插件名称集合。 + * 等价于旧 PluginLoader.getLoadedPlugins()。 + */ + public Set getLoadedPlugins() { + return singletonFactory.loadedKeys(); + } + + /** + * 重置指定类型的 Discovery（主要用于测试）。 + */ + public static void resetPluginDiscovery(Class type) { + DISCOVERIES.remove(type); + } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java b/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java deleted file mode 100644 index da57ecad1..000000000 --- a/datavines-spi/src/main/java/io/datavines/spi/PluginLoader.java +++ /dev/null @@ -1,486 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.spi; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.datavines.spi.utils.ClassUtils; -import io.datavines.spi.utils.Holder; -import io.datavines.spi.utils.UnsafeStringWriter; - -/** - * PluginLoader - */ -public class PluginLoader { - - private static final Logger logger = LoggerFactory.getLogger(PluginLoader.class); - - private static final Pattern NAME_SEPARATOR = Pattern.compile("\\s*[,]+\\s*"); - - private static final String PLUGIN_DIR = "META-INF/plugins/"; - - private static final ConcurrentMap, PluginLoader> PLUGIN_LOADERS = new ConcurrentHashMap<>(64); - - private static final ConcurrentMap, Object> PLUGIN_INSTANCES = new ConcurrentHashMap<>(64); - - private final Class type; - - private final ConcurrentMap, String> cachedNames = new ConcurrentHashMap<>(); - - private final Holder>> cachedClasses = new Holder<>(); - - private final ConcurrentMap> cachedInstances = new ConcurrentHashMap<>(); - - private final Map exceptions = new ConcurrentHashMap<>(); - - private final Object pluginLock = new Object(); - - private PluginLoader(Class type) { - this.type = type; - } - - private static boolean withPluginAnnotation(Class type) { - return type.isAnnotationPresent(SPI.class); - } - - @SuppressWarnings("unchecked") - public static PluginLoader getPluginLoader(Class type) { - if (type == null) { - throw new IllegalArgumentException("Plugin type == null"); - } - - if (!type.isInterface()) { - throw new IllegalArgumentException("Plugin type (" + type + ") is not an interface!"); - } - - if (!withPluginAnnotation(type)) { - throw new IllegalArgumentException("Plugin type (" + type + - ") is not an plugin, because it is NOT annotated with @" + SPI.class.getSimpleName() + "!"); - } - - PluginLoader loader = (PluginLoader) PLUGIN_LOADERS.get(type); - if (loader == null) { - PLUGIN_LOADERS.putIfAbsent(type, new PluginLoader(type)); - loader = (PluginLoader) PLUGIN_LOADERS.get(type); - } - - return loader; - } - - public static void resetPluginLoader(Class type) { - PluginLoader loader = PLUGIN_LOADERS.get(type); - if (loader != null) { - // Remove all instances associated with this loader as well - Map> classes = loader.getPluginClasses(); - for (Map.Entry> entry : classes.entrySet()) { - PLUGIN_INSTANCES.remove(entry.getValue()); - } - classes.clear(); - PLUGIN_LOADERS.remove(type); - } - } - - private static ClassLoader findClassLoader() { - return ClassUtils.getClassLoader(PluginLoader.class); - } - - public String getPluginName(T pluginInstance) { - return getPluginName(pluginInstance.getClass()); - } - - public String getPluginName(Class pluginClass) { - // load class - getPluginClasses(); - return cachedNames.get(pluginClass); - } - - @SuppressWarnings("unchecked") - public T getLoadedPlugin(String name) { - if (StringUtils.isEmpty(name)) { - throw new IllegalArgumentException("Plugin name == null"); - } - Holder holder = getOrCreateHolder(name); - return (T) holder.get(); - } - - private Holder getOrCreateHolder(String name) { - Holder holder = cachedInstances.get(name); - if (holder == null) { - cachedInstances.putIfAbsent(name, new Holder<>()); - holder = cachedInstances.get(name); - } - return holder; - } - - /** - * Return the list of plugins which are already loaded. - */ - public Set getLoadedPlugins() { - return Collections.unmodifiableSet(new TreeSet<>(cachedInstances.keySet())); - } - - @SuppressWarnings("unchecked") - public List getLoadedPluginInstances() { - List instances = new ArrayList<>(); - cachedInstances.values().forEach(holder -> instances.add((T) holder.get())); - return instances; - } - - /** - * Find the plugin with the given name. If the specified name is not found, then {@link IllegalStateException} - * will be thrown. - */ - @SuppressWarnings("unchecked") - public T getOrCreatePlugin(String name) { - if (StringUtils.isEmpty(name)) { - throw new IllegalArgumentException("Plugin name == null"); - } - - final Holder holder = getOrCreateHolder(name); - Object instance = holder.get(); - if (instance == null) { - synchronized (pluginLock) { - instance = holder.get(); - if (instance == null) { - instance = createPlugin(name); - holder.set(instance); - } - } - } - return (T) instance; - } - - @SuppressWarnings("unchecked") - public T getNewPlugin(String name){ - Class clazz = getPluginClasses().get(name); - if (clazz == null) { - throw findException(name); - } - try { - T instance = (T) clazz.newInstance(); - PLUGIN_INSTANCES.putIfAbsent(clazz,instance); - return instance; - } catch (Throwable t) { - throw new IllegalStateException("Plugin instance (name: " + name + ", class: " + - type + ") couldn't be instantiated: " + t.getMessage(), t); - } - } - - public boolean hasPlugin(String name) { - if (StringUtils.isEmpty(name)) { - throw new IllegalArgumentException("Plugin name == null"); - } - Class c = this.getPluginClass(name); - return c != null; - } - - public Set getSupportedPlugins() { - Map> classes = getPluginClasses(); - return Collections.unmodifiableSet(new TreeSet<>(classes.keySet())); - } - - public Set getSupportedPluginInstances() { - List instances = new LinkedList<>(); - Set supportedPlugins = getSupportedPlugins(); - if (CollectionUtils.isNotEmpty(supportedPlugins)) { - for (String name : supportedPlugins) { - instances.add(getOrCreatePlugin(name)); - } - } - // sort the Prioritized instances - instances.sort(Prioritized.COMPARATOR); - return new LinkedHashSet<>(instances); - } - - /** - * Add Or Replace the existing plugin via API - * - * @param name plugin name - * @param clazz plugin class - * @throws IllegalStateException when plugin to be placed doesn't exist - * @deprecated not recommended any longer, and use only when test - */ - public void addOrReplacePlugin(String name, Class clazz) { - getPluginClasses(); // load classes - - if (!type.isAssignableFrom(clazz)) { - throw new IllegalStateException("Input type " + - clazz + " doesn't implement Plugin " + type); - } - if (clazz.isInterface()) { - throw new IllegalStateException("Input type " + - clazz + " can't be interface!"); - } - - if (StringUtils.isBlank(name)) { - throw new IllegalStateException("Plugin name is blank (Plugin " + type + ")!"); - } - if (!cachedClasses.get().containsKey(name)) { - throw new IllegalStateException("Plugin name " + - name + " doesn't exist (Plugin " + type + ")!"); - } - - cachedNames.put(clazz, name); - cachedClasses.get().put(name, clazz); - cachedInstances.remove(name); - } - - private IllegalStateException findException(String name) { - for (Map.Entry entry : exceptions.entrySet()) { - if (entry.getKey().toLowerCase().contains(name.toLowerCase())) { - return entry.getValue(); - } - } - StringBuilder buf = new StringBuilder("No such plugin " + type.getName() + " by name " + name); - - int i = 1; - for (Map.Entry entry : exceptions.entrySet()) { - if (i == 1) { - buf.append(", possible causes: "); - } - - buf.append("\r\n("); - buf.append(i++); - buf.append(") "); - buf.append(entry.getKey()); - buf.append(":\r\n"); - buf.append(toString(entry.getValue())); - } - return new IllegalStateException(buf.toString()); - } - - @SuppressWarnings("unchecked") - private T createPlugin(String name) { - Class clazz = getPluginClasses().get(name); - if (clazz == null) { - throw findException(name); - } - try { - T instance = (T) PLUGIN_INSTANCES.get(clazz); - if (instance == null) { - PLUGIN_INSTANCES.putIfAbsent(clazz, clazz.newInstance()); - instance = (T) PLUGIN_INSTANCES.get(clazz); - } - return instance; - } catch (Throwable t) { - throw new IllegalStateException("Plugin instance (name: " + name + ", class: " + - type + ") couldn't be instantiated: " + t.getMessage(), t); - } - } - - private boolean containsPlugin(String name) { - return getPluginClasses().containsKey(name); - } - - private Class getPluginClass(String name) { - if (type == null) { - throw new IllegalArgumentException("Plugin type == null"); - } - if (name == null) { - throw new IllegalArgumentException("Plugin name == null"); - } - return getPluginClasses().get(name); - } - - private Map> getPluginClasses() { - Map> classes = cachedClasses.get(); - if (classes == null) { - synchronized (cachedClasses) { - classes = cachedClasses.get(); - if (classes == null) { - classes = loadPluginClasses(); - cachedClasses.set(classes); - } - } - } - return classes; - } - - /** - * synchronized in getPluginClasses - */ - private Map> loadPluginClasses() { - - Map> pluginClasses = new HashMap<>(); - loadDirectory(pluginClasses,type.getName()); - - return pluginClasses; - } - - private void loadDirectory(Map> pluginClasses, String type) { - String fileName = PLUGIN_DIR + type; - try { - Enumeration urls = null; - ClassLoader classLoader = findClassLoader(); - - if (classLoader != null) { - urls = classLoader.getResources(fileName); - } else { - urls = ClassLoader.getSystemResources(fileName); - } - - if (urls != null) { - while (urls.hasMoreElements()) { - java.net.URL resourceUrl = urls.nextElement(); - loadResource(pluginClasses, classLoader, resourceUrl); - } - } - } catch (Throwable t) { - logger.error("Exception occurred when loading plugin class (interface: " + - type + ", description file: " + fileName + ").", t); - } - } - - private void loadResource(Map> pluginClasses, ClassLoader classLoader, java.net.URL resourceUrl) { - try { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(resourceUrl.openStream(), StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - final int ci = line.indexOf('#'); - if (ci >= 0) { - line = line.substring(0, ci); - } - line = line.trim(); - if (!line.isEmpty()) { - try { - String name = null; - int i = line.indexOf('='); - if (i > 0) { - name = line.substring(0, i).trim(); - line = line.substring(i + 1).trim(); - } - if (!line.isEmpty()) { - loadClass(pluginClasses, resourceUrl, Class.forName(line, true, classLoader), name); - } - } catch (Throwable t) { - IllegalStateException e = new IllegalStateException("Failed to load plugin class (interface: " + type + ", class line: " + line + ") in " + resourceUrl + ", cause: " + t.getMessage(), t); - exceptions.put(line, e); - } - } - } - } - } catch (Throwable t) { - logger.error("Exception occurred when loading plugin class (interface: " + - type + ", class file: " + resourceUrl + ") in " + resourceUrl, t); - } - } - - private void loadClass(Map> pluginClasses, - java.net.URL resourceUrl, Class clazz, String name) { - if (!type.isAssignableFrom(clazz)) { - throw new IllegalStateException("Error occurred when loading plugin class (interface: " + - type + ", class line: " + clazz.getName() + "), class " - + clazz.getName() + " is not subtype of interface."); - } - - if (StringUtils.isEmpty(name)) { - name = findAnnotationName(clazz); - if (name.isEmpty()) { - throw new IllegalStateException("No such plugin name for the class " + clazz.getName() + " in the config " + resourceUrl); - } - } - - String[] names = NAME_SEPARATOR.split(name); - if (ArrayUtils.isNotEmpty(names)) { - for (String n : names) { - cacheName(clazz, n); - saveInPluginClass(pluginClasses, clazz, n); - } - } - } - - /** - * cache name - */ - private void cacheName(Class clazz, String name) { - if (!cachedNames.containsKey(clazz)) { - cachedNames.put(clazz, name); - } - } - - /** - * put clazz in pluginClasses - */ - private void saveInPluginClass(Map> pluginClasses, Class clazz, String name) { - Class c = pluginClasses.get(name); - if (c == null) { - pluginClasses.put(name, clazz); - } else if (c != clazz) { - String duplicateMsg = "Duplicate plugin " + type.getName() + " name " + name + " on " + c.getName() + " and " + clazz.getName(); - logger.error(duplicateMsg); - throw new IllegalStateException(duplicateMsg); - } - } - - private String findAnnotationName(Class clazz) { - - String name = clazz.getSimpleName(); - if (name.endsWith(type.getSimpleName())) { - name = name.substring(0, name.length() - type.getSimpleName().length()); - } - return name.toLowerCase(); - } - - @Override - public String toString() { - return this.getClass().getName() + "[" + type.getName() + "]"; - } - - /** - * @param e - * @return string - */ - public static String toString(Throwable e) { - UnsafeStringWriter w = new UnsafeStringWriter(); - PrintWriter p = new PrintWriter(w); - p.print(e.getClass().getName()); - if (e.getMessage() != null) { - p.print(": " + e.getMessage()); - } - p.println(); - try { - e.printStackTrace(p); - return w.toString(); - } finally { - p.close(); - } - } - -} diff --git a/datavines-spi/src/main/java/io/datavines/spi/PluginManager.java b/datavines-spi/src/main/java/io/datavines/spi/PluginManager.java new file mode 100644 index 000000000..5af48aa14 --- /dev/null +++ b/datavines-spi/src/main/java/io/datavines/spi/PluginManager.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.spi; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +/** + * 唯一型扩展的统一入口。 + * + *

内部组合 {@link KeyedRegistry} 和 {@link CachingFactory}， + * 适用于"一个 Key 对应一个 Provider"的场景。 + * + * @param Key 类型 + * @param

Provider 类型（在 datavines 中通常 P 就是最终实例类型） + */ +public final class PluginManager { + + private final KeyedRegistry registry; + private final CachingFactory factory; + + private PluginManager(KeyedRegistry registry, CachingFactory factory) { + this.registry = registry; + this.factory = factory; + } + + /** + * 从 ServiceLoader 加载并构建 PluginManager。 + * + * @param providerType SPI 接口类型 + * @param keyExtractor 从实例中提取 Key 的函数 + * @param instanceCreator 创建新实例的函数（从注册表中的 Provider 创建） + * @param managerName 管理器名称（用于日志和异常） + */ + public static PluginManager load( + Class

providerType, + Function keyExtractor, + Function instanceCreator, + String managerName) { + + KeyedRegistry registry = KeyedRegistry.load(providerType, keyExtractor, managerName); + CachingFactory factory = new CachingFactory<>(key -> instanceCreator.apply(registry.get(key))); + return new PluginManager<>(registry, factory); + } + + /** + * 简化版本：Provider 即 Instance（T = P），缓存 Provider 自身。 + */ + public static PluginManager loadSelfManaged( + Class

providerType, + Function keyExtractor, + String managerName) { + + KeyedRegistry registry = KeyedRegistry.load(providerType, keyExtractor, managerName); + // 缓存模式：直接返回注册表中的 Provider 实例 + CachingFactory factory = new CachingFactory<>(registry::get); + return new PluginManager<>(registry, factory); + } + + /** + * 获取缓存实例（单例模式）。 + */ + public P get(K key) { + return factory.get(key); + } + + /** + * 创建新实例，不走缓存。 + * 通过反射创建新实例（用于需要每次新建的场景）。 + */ + public P createNew(K key) { + P provider = registry.get(key); + try { + @SuppressWarnings("unchecked") + P newInstance = (P) provider.getClass().newInstance(); + return newInstance; + } catch (Exception e) { + throw new IllegalStateException( + "Failed to create new instance of " + provider.getClass().getName(), e); + } + } + + public P getIfLoaded(K key) { + return factory.getIfLoaded(key); + } + + public boolean supports(K key) { + return registry.supports(key); + } + + public P getProvider(K key) { + return registry.get(key); + } + + public P findProvider(K key) { + return registry.find(key); + } + + public Set getSupportedKeys() { + return registry.getSupportedKeys(); + } + + public Collection

getSupportedProviders() { + return registry.providers(); + } + + public Set getLoadedKeys() { + return factory.loadedKeys(); + } + + public List

getLoadedInstances() { + return factory.loadedInstances(); + } + + public void clearCache() { + factory.clear(); + } + + public void evict(K key) { + factory.evict(key); + } + + public KeyedRegistry getRegistry() { + return registry; + } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/Prioritized.java b/datavines-spi/src/main/java/io/datavines/spi/Prioritized.java deleted file mode 100644 index 2ada779af..000000000 --- a/datavines-spi/src/main/java/io/datavines/spi/Prioritized.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.spi; - -import static java.lang.Integer.compare; - -import java.util.Comparator; - -public interface Prioritized extends Comparable { - - /** - * The {@link Comparator} of {@link Prioritized} - */ - Comparator COMPARATOR = (one, two) -> { - boolean b1 = one instanceof Prioritized; - boolean b2 = two instanceof Prioritized; - if (b1 && !b2) { // one is Prioritized, two is not - return -1; - } else if (b2 && !b1) { // two is Prioritized, one is not - return 1; - } else if (b1 && b2) { // one and two both are Prioritized - return ((Prioritized) one).compareTo((Prioritized) two); - } else { // no different - return 0; - } - }; - - /** - * The maximum priority - */ - int MAX_PRIORITY = Integer.MIN_VALUE; - - /** - * The minimum priority - */ - int MIN_PRIORITY = Integer.MAX_VALUE; - - /** - * Normal Priority - */ - int NORMAL_PRIORITY = 0; - - /** - * Get the priority - * - * @return the default is {@link #MIN_PRIORITY minimum one} - */ - default int getPriority() { - return NORMAL_PRIORITY; - } - - @Override - default int compareTo(Prioritized that) { - return compare(this.getPriority(), that.getPriority()); - } -} diff --git a/datavines-spi/src/main/java/io/datavines/spi/SPI.java b/datavines-spi/src/main/java/io/datavines/spi/ProviderNotFoundException.java similarity index 53% rename from datavines-spi/src/main/java/io/datavines/spi/SPI.java rename to datavines-spi/src/main/java/io/datavines/spi/ProviderNotFoundException.java index 886b474ea..8f5c71a61 100644 --- a/datavines-spi/src/main/java/io/datavines/spi/SPI.java +++ b/datavines-spi/src/main/java/io/datavines/spi/ProviderNotFoundException.java @@ -16,17 +16,25 @@ */ package io.datavines.spi; -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +import java.util.Set; -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) -public @interface SPI { +/** + * 找不到指定 Key 对应的 Provider 时抛出。 + */ +public class ProviderNotFoundException extends RuntimeException { + + private final String registryName; + private final Object key; - String value() default ""; + public ProviderNotFoundException(String registryName, Object key, Set availableKeys) { + super(String.format( + "[%s] No provider found for key '%s'. Available keys: %s. " + + "Ensure the corresponding plugin JAR is on the classpath.", + registryName, key, availableKeys)); + this.registryName = registryName; + this.key = key; + } -} \ No newline at end of file + public String getRegistryName() { return registryName; } + public Object getKey() { return key; } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/ServiceLoaderUtils.java b/datavines-spi/src/main/java/io/datavines/spi/ServiceLoaderUtils.java new file mode 100644 index 000000000..8387a4c21 --- /dev/null +++ b/datavines-spi/src/main/java/io/datavines/spi/ServiceLoaderUtils.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.spi; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; + +/** + * ServiceLoader 的安全封装。 + * + *

JDK ServiceLoader 在遇到单个 Provider 加载失败时， + * 默认行为是抛出 ServiceConfigurationError 并中断整个迭代。 + * 本工具类提供容错模式：跳过失败的 Provider，继续加载其余的。 + */ +public final class ServiceLoaderUtils { + + private static final Logger log = LoggerFactory.getLogger(ServiceLoaderUtils.class); + + private ServiceLoaderUtils() {} + + /** + * 安全加载所有 Provider。单个 Provider 加载失败时记录错误并跳过，不影响其他 Provider。 + * + * @param serviceType SPI 服务接口 + * @return 成功加载的 Provider 列表 + */ + public static List loadAll(Class serviceType) { + return loadAll(serviceType, Thread.currentThread().getContextClassLoader()); + } + + /** + * 使用指定 ClassLoader 安全加载所有 Provider。 + */ + public static List loadAll(Class serviceType, ClassLoader classLoader) { + List result = new ArrayList<>(); + ServiceLoader loader = ServiceLoader.load(serviceType, classLoader); + Iterator iterator = loader.iterator(); + + while (iterator.hasNext()) { + try { + result.add(iterator.next()); + } catch (ServiceConfigurationError e) { + log.error("Failed to load provider for {}: {}", + serviceType.getName(), e.getMessage(), e); + } + } + + log.debug("Loaded {} providers for {}", result.size(), serviceType.getName()); + return result; + } + + /** + * 严格加载所有 Provider。任何 Provider 加载失败都会抛出异常。 + * + * @param serviceType SPI 服务接口 + * @return 成功加载的 Provider 列表 + * @throws ServiceConfigurationError 如果任何 Provider 加载失败 + */ + public static List loadAllStrict(Class serviceType) { + List result = new ArrayList<>(); + for (S provider : ServiceLoader.load(serviceType)) { + result.add(provider); + } + return result; + } +} diff --git a/datavines-spi/src/main/java/io/datavines/spi/utils/ClassUtils.java b/datavines-spi/src/main/java/io/datavines/spi/utils/ClassUtils.java deleted file mode 100644 index cf25e4a88..000000000 --- a/datavines-spi/src/main/java/io/datavines/spi/utils/ClassUtils.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.spi.utils; - -public class ClassUtils { - - /** - * get class loader - * - * @param clazz - * @return class loader - */ - public static ClassLoader getClassLoader(Class clazz) { - ClassLoader cl = null; - try { - cl = Thread.currentThread().getContextClassLoader(); - } catch (Throwable ex) { - // Cannot access thread context ClassLoader - falling back to system class loader... - } - if (cl == null) { - // No thread context class loader -> use class loader of this class. - cl = clazz.getClassLoader(); - if (cl == null) { - // getClassLoader() returning null indicates the bootstrap ClassLoader - try { - cl = ClassLoader.getSystemClassLoader(); - } catch (Throwable ex) { - // Cannot access system ClassLoader - oh well, maybe the caller can live with null... - } - } - } - - return cl; - } - -} diff --git a/datavines-spi/src/main/java/io/datavines/spi/utils/UnsafeStringWriter.java b/datavines-spi/src/main/java/io/datavines/spi/utils/UnsafeStringWriter.java deleted file mode 100644 index eb35a1678..000000000 --- a/datavines-spi/src/main/java/io/datavines/spi/utils/UnsafeStringWriter.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.datavines.spi.utils; - -import java.io.IOException; -import java.io.Writer; - -/** - * Thread-unsafe StringWriter. - */ -public class UnsafeStringWriter extends Writer { - - private final StringBuilder mBuffer; - - public UnsafeStringWriter() { - lock = mBuffer = new StringBuilder(); - } - - public UnsafeStringWriter(int size) { - if (size < 0) { - throw new IllegalArgumentException("Negative buffer size"); - } - - lock = mBuffer = new StringBuilder(); - } - - @Override - public void write(int c) { - mBuffer.append((char) c); - } - - @Override - public void write(char[] cs) throws IOException { - mBuffer.append(cs, 0, cs.length); - } - - @Override - public void write(char[] cs, int off, int len) throws IOException { - if ((off < 0) || (off > cs.length) || (len < 0) || - ((off + len) > cs.length) || ((off + len) < 0)) { - throw new IndexOutOfBoundsException(); - } - - if (len > 0) { - mBuffer.append(cs, off, len); - } - } - - @Override - public void write(String str) { - mBuffer.append(str); - } - - @Override - public void write(String str, int off, int len) { - mBuffer.append(str, off, off + len); - } - - @Override - public Writer append(CharSequence csq) { - if (csq == null) { - write("null"); - } else { - write(csq.toString()); - } - return this; - } - - @Override - public Writer append(CharSequence csq, int start, int end) { - CharSequence cs = (csq == null ? "null" : csq); - write(cs.subSequence(start, end).toString()); - return this; - } - - @Override - public Writer append(char c) { - mBuffer.append(c); - return this; - } - - @Override - public void close() { - } - - @Override - public void flush() { - } - - @Override - public String toString() { - return mBuffer.toString(); - } -} \ No newline at end of file