From 204ecfdad72f3b6317a8dc725d4a6d2a179f8e74 Mon Sep 17 00:00:00 2001 From: jiayang Date: Mon, 23 Oct 2023 10:39:35 +0800 Subject: [PATCH 1/6] [feature] add doris mongodb mongodbcdc datasource --- .../datasource/AbstractDataSourceClient.java | 3 +- .../classloader/DatasourceLoadConfig.java | 16 +- .../ElasticSearchDataSourceChannel.java | 2 +- .../ElasticSearchDataSourceChannelTest.java | 3 +- .../plugin/hive/HiveDataSourceChannel.java | 2 +- .../jdbc/ClickhouseJdbcDataSourceChannel.java | 2 +- .../datasource-jdbc-common/pom.xml | 28 +++ .../JdbcDataSourceLRUConnectionPool.java | 148 +++++++++++ .../datasource-jdbc-doris/pom.xml | 69 +++++ .../doris/jdbc/DorisDataSourceConfig.java | 24 ++ .../jdbc/DorisJdbcDataSourceChannel.java | 237 ++++++++++++++++++ .../jdbc/DorisJdbcDataSourceFactory.java | 31 +++ .../plugin/doris/jdbc/DorisOptionRule.java | 51 ++++ .../hive/jdbc/HiveJdbcDataSourceChannel.java | 2 +- .../jdbc/MysqlJdbcDataSourceChannel.java | 2 +- .../oracle/jdbc/OracleDataSourceChannel.java | 2 +- .../jdbc/PostgresqlDataSourceChannel.java | 2 +- .../jdbc/RedshiftDataSourceChannel.java | 2 +- .../jdbc/SqlServerDataSourceChannel.java | 2 +- .../jdbc/StarRocksJdbcDataSourceChannel.java | 2 +- .../tidb/jdbc/TidbJdbcDataSourceChannel.java | 2 +- .../plugin/kafka/KafkaDataSourceChannel.java | 2 +- .../kafka/KafkaDataSourceChannelTest.java | 3 +- .../datasource-mongodb/pom.xml | 50 ++++ .../mongodb/MongoDataSoueceChannel.java | 89 +++++++ .../mongodb/MongoDataSourceFactory.java | 40 +++ .../plugin/mongodb/MongoOptionRule.java | 34 +++ .../mongodb/MongoRequestParamsUtils.java | 16 ++ .../datasource-mongodbcdc/pom.xml | 50 ++++ .../MongodbCDCDataSourceChannel.java | 101 ++++++++ .../MongodbCDCDataSourceFactory.java | 40 +++ .../mongodbcdc/MongodbCDCOptionRule.java | 108 ++++++++ .../cdc/mysql/MysqlCDCDataSourceChannel.java | 2 +- .../plugin/api/DataSourceChannel.java | 31 ++- .../plugin/s3/S3DatasourceChannel.java | 2 +- .../SqlServerCDCDataSourceChannel.java | 2 +- .../starrocks/StarRocksDataSourceChannel.java | 2 +- .../seatunnel-datasource-plugins/pom.xml | 4 + .../datasource/SchemaGenerator.java | 123 +++++++++ .../impl/DorisDatasourceConfigSwitcher.java | 98 ++++++++ .../MongoDBCDCDataSourceConfigSwitcher.java | 102 ++++++++ .../impl/MongoDBDataSourceConfigSwitcher.java | 98 ++++++++ .../connector-datasource-mapper.yaml | 42 ++++ 43 files changed, 1643 insertions(+), 28 deletions(-) create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java create mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java index 34fb2850e..e5552ce16 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/AbstractDataSourceClient.java @@ -143,8 +143,7 @@ public OptionRule queryDataSourceFieldByName(String pluginName) { public OptionRule queryMetadataFieldByName(String pluginName) { updateClassLoader(pluginName); OptionRule datasourceMetadataFieldsByDataSourceName = - getDataSourceChannel(pluginName) - .getDatasourceMetadataFieldsByDataSourceName(pluginName); + getDataSourceChannel(pluginName).getVirtualTableOptions(pluginName); classLoaderRestore(); return datasourceMetadataFieldsByDataSourceName; } diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java index ef3dc0d77..6dc1fad62 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java @@ -92,6 +92,14 @@ public class DatasourceLoadConfig { classLoaderFactoryName.put( "JDBC-STARROCKS", "org.apache.seatunnel.datasource.plugin.starrocks.jdbc.StarRocksJdbcDataSourceFactory"); + classLoaderFactoryName.put( + "DORIS", + "com.apache.seatunnel.datasource.plugin.doris.jdbc.DorisJdbcDataSourceFactory"); + classLoaderFactoryName.put( + "MONGODB", "com.apache.seatunnel.datasource.plugin.mongodb.MongoDataSourceFactory"); + classLoaderFactoryName.put( + "MONGODB-CDC", + "com.apache.seatunnel.datasource.plugin.mongodbcdc.MongodbCDCDataSourceFactory"); classLoaderJarName.put("JDBC-ORACLE", "datasource-jdbc-oracle-"); classLoaderJarName.put("JDBC-CLICKHOUSE", "datasource-jdbc-clickhouse-"); @@ -111,6 +119,9 @@ public class DatasourceLoadConfig { classLoaderJarName.put("STARROCKS", "datasource-starrocks-"); classLoaderJarName.put("S3-REDSHIFT", "datasource-s3redshift-"); classLoaderJarName.put("JDBC-STARROCKS", "datasource-jdbc-starrocks-"); + classLoaderJarName.put("DORIS", "datasource-jdbc-doris-"); + classLoaderJarName.put("MONGODB", "datasource-mongodb-"); + classLoaderJarName.put("MONGODB-CDC", "datasource-mongodbcdc-"); } public static final Set pluginSet = @@ -127,7 +138,10 @@ public class DatasourceLoadConfig { "MySQL-CDC", "S3", "SqlServer-CDC", - "StarRocks"); + "Doris", + "StarRocks", + "MongoDB", + "MongoDB-CDC"); public static Map datasourceClassLoaders = new HashMap<>(); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java index 77c12f292..1dc3e933b 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/main/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannel.java @@ -49,7 +49,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return ElasticSearchOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java index e5d164c80..bbf78f56d 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-elasticsearch/src/test/java/org/apache/seatunnel/datasource/plugin/elasticsearch/ElasticSearchDataSourceChannelTest.java @@ -63,8 +63,7 @@ void getDataSourceOptions() { @Test void getDatasourceMetadataFieldsByDataSourceName() { Assertions.assertNotNull( - ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName( - PLUGIN_NAME)); + ELASTIC_SEARCH_DATA_SOURCE_CHANNEL.getVirtualTableOptions(PLUGIN_NAME)); } // @Test diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java index cebdbf8e8..b333df96c 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/com/whaleops/datasource/datasource/plugin/hive/HiveDataSourceChannel.java @@ -44,7 +44,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return HiveOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java index 68ce5f177..c4358013b 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-clickhouse/src/main/java/org/apache/seatunnel/datasource/plugin/clickhouse/jdbc/ClickhouseJdbcDataSourceChannel.java @@ -49,7 +49,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return ClickhouseDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml new file mode 100644 index 000000000..e6219da8f --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + org.apache.seatunnel + seatunnel-datasource-plugins + ${revision} + + + datasource-jdbc-common + + + 4.0.3 + + + + + com.google.guava + guava + + + com.zaxxer + HikariCP + ${hikaricp.version} + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java new file mode 100644 index 000000000..758930831 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java @@ -0,0 +1,148 @@ +package com.whaleops.datasource.plugin.jdbc.common; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.zaxxer.hikari.HikariDataSource; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +public abstract class JdbcDataSourceLRUConnectionPool implements AutoCloseable { + private static final Duration SLOW_CONNECTION_CREATE_LIMIT = Duration.ofSeconds(2); + private static final Duration CLEAN_CONNECTION_INTERVAL = Duration.ofMinutes(30); + private static Cache datasourceCache; + private static ScheduledExecutorService cleanUpExecutor; + + public Connection getConnection(Map requestParams) throws SQLException { + initialize(); + + HikariDataSource dataSource; + try { + dataSource = + datasourceCache.get( + datasourceCacheKey(requestParams), + createDataSourceLoader(requestParams)); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + + long start = System.currentTimeMillis(); + Connection connection = dataSource.getConnection(); + long end = System.currentTimeMillis(); + if (end - start > SLOW_CONNECTION_CREATE_LIMIT.toMillis()) { + log.warn( + "Slow jdbc connection creation, cost {}ms for {}", + end - start, + dataSource.getJdbcUrl()); + } + return connection; + } + + protected abstract HikariDataSource createDataSource(Map requestParams); + + protected abstract String datasourceCacheKey(Map requestParams); + + private Callable createDataSourceLoader(Map requestParams) { + return () -> { + log.info("Creating jdbc datasource for {}", desensitize(requestParams)); + HikariDataSource dataSource = createDataSource(requestParams); + log.info("Created jdbc datasource for {}", desensitize(requestParams)); + return dataSource; + }; + } + + protected Map desensitize(Map requestParams) { + Map newRequestParams = new HashMap<>(requestParams); + newRequestParams.remove("password"); + return newRequestParams; + } + + public void initialize() { + if (cleanUpExecutor == null) { + synchronized (JdbcDataSourceLRUConnectionPool.class) { + if (cleanUpExecutor == null) { + log.info("Initializing jdbc datasource connection pool"); + datasourceCache = + CacheBuilder.newBuilder() + .expireAfterAccess( + CLEAN_CONNECTION_INTERVAL.toMillis(), + TimeUnit.MILLISECONDS) + .removalListener( + (RemovalListener) + notification -> { + try { + log.info( + "Closing expired jdbc datasource connection pool for {}", + notification.getKey()); + notification.getValue().close(); + log.info( + "Closed expired jdbc datasource connection pool for {}", + notification.getKey()); + } catch (Exception e) { + log.error( + "Failed to close expired jdbc datasource connection pool for {}", + notification.getKey(), + e); + } + }) + .build(); + cleanUpExecutor = + Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat( + "datasource-plugin-jdbc-connection-pool-clean-up-%d") + .build()); + cleanUpExecutor.scheduleWithFixedDelay( + () -> { + log.debug("Cleaning up jdbc datasource cache"); + datasourceCache.cleanUp(); + log.debug("Cleaned up jdbc datasource cache"); + }, + CLEAN_CONNECTION_INTERVAL.toMillis(), + CLEAN_CONNECTION_INTERVAL.toMillis(), + TimeUnit.MILLISECONDS); + log.info("Initialized jdbc datasource connection pool"); + } + } + } + } + + @Override + public void close() throws Exception { + if (cleanUpExecutor != null) { + synchronized (JdbcDataSourceLRUConnectionPool.class) { + log.info("Closing jdbc datasource connection pool"); + cleanUpExecutor.shutdownNow(); + datasourceCache.cleanUp(); + cleanUpExecutor = null; + log.info("Closed jdbc datasource connection pool"); + } + } + } + + public static String extractDbName(String jdbcUrl) { + String dbName = null; + Pattern pattern = Pattern.compile("[:/](\\w+)(?:\\?|&|$)"); + Matcher matcher = pattern.matcher(jdbcUrl); + + while (matcher.find()) { + dbName = matcher.group(1); + } + + return dbName; + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml new file mode 100644 index 000000000..afc9a7c62 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml @@ -0,0 +1,69 @@ + + + 4.0.0 + + org.apache.seatunnel + seatunnel-datasource-plugins + ${revision} + + + datasource-jdbc-doris + + + 8.0.28 + + + + + org.apache.seatunnel + datasource-plugins-api + ${project.version} + provided + + + org.apache.seatunnel + datasource-jdbc-common + ${project.version} + + + org.apache.commons + commons-lang3 + + + + com.google.auto.service + auto-service + + + org.apache.seatunnel + seatunnel-api + provided + + + + + mysql + mysql-connector-java + ${mysql-connector.version} + provided + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java new file mode 100644 index 000000000..4a08e24d7 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java @@ -0,0 +1,24 @@ +package com.apache.seatunnel.datasource.plugin.doris.jdbc; + +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; +import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; + +import com.google.common.collect.Sets; + +import java.util.Set; + +public class DorisDataSourceConfig { + + public static final String PLUGIN_NAME = "Doris"; + + public static final DataSourcePluginInfo DORIS_DATASOURCE_PLUGIN_INFO = + DataSourcePluginInfo.builder() + .name(PLUGIN_NAME) + .icon(PLUGIN_NAME) + .version("1.0.0") + .type(DatasourcePluginTypeEnum.DATABASE.getCode()) + .build(); + + public static final Set DORIS_SYSTEM_DATABASES = + Sets.newHashSet("information_schema", "__internal_schema"); +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java new file mode 100644 index 000000000..29dc9b1b8 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java @@ -0,0 +1,237 @@ +package com.apache.seatunnel.datasource.plugin.doris.jdbc; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import com.whaleops.datasource.plugin.jdbc.common.JdbcDataSourceLRUConnectionPool; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import lombok.NonNull; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class DorisJdbcDataSourceChannel extends JdbcDataSourceLRUConnectionPool + implements DataSourceChannel { + + @Override + public OptionRule getDataSourceOptions(@NonNull String pluginName) { + return DorisOptionRule.optionRule(); + } + + @Override + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { + return DorisOptionRule.metadataRule(); + } + + @Override + public List getTables( + @NonNull String pluginName, + Map requestParams, + String database, + Map options) { + List tableNames = new ArrayList<>(); + String filterName = options.get("filterName"); + String size = options.get("size"); + boolean isSize = StringUtils.isNotEmpty(size); + if (StringUtils.isNotEmpty(filterName) && !filterName.contains("%")) { + filterName = "%" + filterName + "%"; + } else if (StringUtils.equals(filterName, "")) { + filterName = null; + } + try (Connection connection = getConnection(requestParams); + ResultSet resultSet = + connection + .getMetaData() + .getTables(database, null, filterName, new String[] {"TABLE"})) { + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + if (StringUtils.isNotBlank(tableName)) { + tableNames.add(tableName); + if (isSize && tableNames.size() >= Integer.parseInt(size)) { + break; + } + } + } + return tableNames; + } catch (SQLException e) { + throw new DataSourcePluginException("get table names failed", e); + } + } + + @Override + public List getDatabases( + @NonNull String pluginName, @NonNull Map requestParams) { + List dbNames = new ArrayList<>(); + try (Connection connection = getConnection(requestParams); + PreparedStatement statement = connection.prepareStatement("SHOW DATABASES;"); + ResultSet re = statement.executeQuery()) { + // filter system databases + while (re.next()) { + String dbName = re.getString("database"); + if (StringUtils.isNotBlank(dbName) + && !DorisDataSourceConfig.DORIS_SYSTEM_DATABASES.contains(dbName)) { + dbNames.add(dbName); + } + } + return dbNames; + } catch (SQLException e) { + throw new DataSourcePluginException("Get databases failed", e); + } + } + + @Override + public boolean checkDataSourceConnectivity( + @NonNull String pluginName, @NonNull Map requestParams) { + try (Connection ignored = getConnection(requestParams)) { + return true; + } catch (Exception e) { + throw new DataSourcePluginException("check jdbc connectivity failed", e); + } + } + + @Override + public List getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { + List tableFields = new ArrayList<>(); + try (Connection connection = getConnection(requestParams)) { + DatabaseMetaData metaData = connection.getMetaData(); + String primaryKey = getPrimaryKey(metaData, database, table); + try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) { + while (resultSet.next()) { + TableField tableField = new TableField(); + String columnName = resultSet.getString("COLUMN_NAME"); + tableField.setPrimaryKey(false); + if (StringUtils.isNotBlank(primaryKey) && primaryKey.equals(columnName)) { + tableField.setPrimaryKey(true); + } + tableField.setName(columnName); + tableField.setType(resultSet.getString("TYPE_NAME")); + tableField.setComment(resultSet.getString("REMARKS")); + Object nullable = resultSet.getObject("IS_NULLABLE"); + tableField.setNullable(Boolean.TRUE.toString().equals(nullable.toString())); + tableFields.add(tableField); + } + } + } catch (SQLException e) { + throw new DataSourcePluginException("get table fields failed", e); + } + return tableFields; + } + + private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String tableName) + throws SQLException { + try (ResultSet primaryKeysInfo = metaData.getPrimaryKeys(dbName, "%", tableName)) { + while (primaryKeysInfo.next()) { + return primaryKeysInfo.getString("COLUMN_NAME"); + } + } + return null; + } + + @Override + protected HikariDataSource createDataSource(final Map requestParams) { + checkNotNull(requestParams.get(DorisOptionRule.BASE_URL.key()), "Jdbc url cannot be null"); + String url = requestParams.get(DorisOptionRule.BASE_URL.key()); + String username = requestParams.get(DorisOptionRule.USERNAME.key()); + String password = requestParams.get(DorisOptionRule.PASSWORD.key()); + + HikariConfig config = new HikariConfig(); + config.setPoolName(String.format("datasource-plugin-doris[%s, %s]", url, username)); + config.setJdbcUrl(url); + config.setUsername(username); + config.setPassword(password); + config.setMinimumIdle(1); + config.setMaximumPoolSize(2); + config.setConnectionTimeout(30000); + config.setIdleTimeout(600000); + config.setMaxLifetime(1800000); + config.setConnectionTestQuery("SELECT 1"); + return new HikariDataSource(config); + } + + @Override + protected String datasourceCacheKey(Map requestParams) { + String url = requestParams.get(DorisOptionRule.BASE_URL.key()); + if (null == requestParams.get(DorisOptionRule.BASE_URL.key())) { + throw new DataSourcePluginException("Jdbc url is null"); + } + String username = requestParams.get(DorisOptionRule.USERNAME.key()); + return String.format("{jdbc-url=%s, username=%s}", url, username); + } + + @Override + public Pair getTableSyncMaxValue( + String pluginName, + Map requestParams, + String databaseName, + String tableName, + String updateFieldType) { + String updateFieldName = requestParams.get("updateFieldName"); + StringBuilder sql = new StringBuilder(); + String replaceTableName = parseDatabaseTableName(tableName, databaseName); + sql.append("SELECT MAX(`") + .append(updateFieldName) + .append("`) AS `") + .append(updateFieldName) + .append("` FROM ") + .append(replaceTableName) + .append(";"); + + if (sql.toString().equals("")) { + return null; + } + try (Connection connection = getConnection(requestParams); + PreparedStatement statement = connection.prepareStatement(sql.toString()); + ResultSet re = statement.executeQuery()) { + ResultSetMetaData metaData = re.getMetaData(); + String columnTypeName = metaData.getColumnTypeName(1); + while (re.next()) { + String fieldName = re.getString(updateFieldName); + if (StringUtils.isEmpty(fieldName) + || StringUtils.equalsIgnoreCase("null", fieldName)) { + throw new RuntimeException(updateFieldName + " max value is null"); + } + if (StringUtils.equals(updateFieldType, "primary")) { + return Pair.of(fieldName, columnTypeName); + } else if (StringUtils.equals(updateFieldType, "datetime")) { + return Pair.of(fieldName, columnTypeName); + } + } + } catch (Exception ex) { + throw new RuntimeException("get table max value sync failed", ex); + } + return DataSourceChannel.super.getTableSyncMaxValue( + pluginName, requestParams, databaseName, tableName, updateFieldType); + } + + private String parseDatabaseTableName(String tableName, String databaseName) { + return "`" + databaseName + "`.`" + tableName + "`"; + } + + @Override + public Connection getConnection(String pluginName, Map requestParams) { + try { + return getConnection(requestParams); + } catch (Exception e) { + throw new DataSourcePluginException("Get connection failed", e); + } + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java new file mode 100644 index 000000000..addcbe378 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java @@ -0,0 +1,31 @@ +package com.apache.seatunnel.datasource.plugin.doris.jdbc; + +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; + +import com.google.auto.service.AutoService; +import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; + +import java.util.Set; + +@Slf4j +@AutoService(DataSourceFactory.class) +public class DorisJdbcDataSourceFactory implements DataSourceFactory { + + @Override + public String factoryIdentifier() { + return DorisDataSourceConfig.PLUGIN_NAME; + } + + @Override + public Set supportedDataSources() { + return Sets.newHashSet(DorisDataSourceConfig.DORIS_DATASOURCE_PLUGIN_INFO); + } + + @Override + public DataSourceChannel createChannel() { + return new DorisJdbcDataSourceChannel(); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java new file mode 100644 index 000000000..35b8365bf --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java @@ -0,0 +1,51 @@ +package com.apache.seatunnel.datasource.plugin.doris.jdbc; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.util.OptionRule; + +public class DorisOptionRule { + + private static final String DEFAULT_PASSWORD = ""; + + public static final Option FENODES = + Options.key("fenodes") + .stringType() + .noDefaultValue() + .withDescription( + "Doris cluster fenodes address, the format is \"fe_ip:fe_http_port, ...\""); + + public static final Option USERNAME = + Options.key("username") + .stringType() + .noDefaultValue() + .withDescription("Doris user username"); + + public static final Option PASSWORD = + Options.key("password") + .stringType() + .defaultValue(DEFAULT_PASSWORD) + .withDescription("Doris user password"); + + public static final Option TABLE = + Options.key("table.identifier") + .stringType() + .noDefaultValue() + .withDescription("The name of Doris table"); + + public static final Option BASE_URL = + Options.key("base-url") + .stringType() + .noDefaultValue() + .withDescription( + "URL has to be without database, like \"jdbc:mysql://localhost:5432/\" or" + + "\"jdbc:mysql://localhost:5432\" rather than \"jdbc:mysql://localhost:5432/db\""); + + public static OptionRule optionRule() { + return OptionRule.builder().required(FENODES, USERNAME, PASSWORD, BASE_URL).build(); + } + + public static OptionRule metadataRule() { + return OptionRule.builder().required(TABLE).build(); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java index 2fa5b1961..d13a444c4 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/jdbc/HiveJdbcDataSourceChannel.java @@ -51,7 +51,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return HiveJdbcOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java index 78a7f62e9..cf29c5582 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-mysql/src/main/java/org/apache/seatunnel/datasource/plugin/mysql/jdbc/MysqlJdbcDataSourceChannel.java @@ -49,7 +49,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return MysqlDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java index 775decb1c..a70e672d1 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java @@ -51,7 +51,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return OracleDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java index b21bf9930..2271ff8d8 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-postgresql/src/main/java/org/apache/seatunnel/datasource/plugin/postgresql/jdbc/PostgresqlDataSourceChannel.java @@ -50,7 +50,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return PostgresqlDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java index 98c4db848..f7185da4e 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/jdbc/RedshiftDataSourceChannel.java @@ -47,7 +47,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return RedshiftDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java index e560e7f1f..59bea2e3f 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-sqlserver/src/main/java/org/apache/seatunnel/datasource/plugin/sqlserver/jdbc/SqlServerDataSourceChannel.java @@ -50,7 +50,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return SqlServerDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java index 271653388..21f311eaa 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/jdbc/StarRocksJdbcDataSourceChannel.java @@ -47,7 +47,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return StarRocksDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java index 90f929370..b2cb5b23a 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-tidb/src/main/java/org/apache/seatunnel/datasource/plugin/tidb/jdbc/TidbJdbcDataSourceChannel.java @@ -49,7 +49,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return TidbDataSourceConfig.METADATA_RULE; } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java index 1f377b4b5..195ef80fc 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/main/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannel.java @@ -52,7 +52,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return KafkaOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java index bd30e9ccb..687765209 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-kafka/src/test/java/org/apache/seatunnel/datasource/plugin/kafka/KafkaDataSourceChannelTest.java @@ -58,8 +58,7 @@ public void getDataSourceOptions() { @Test public void getDatasourceMetadataFieldsByDataSourceName() { OptionRule datasourceMetadataFieldsByDataSourceName = - KAFKA_DATA_SOURCE_CHANNEL.getDatasourceMetadataFieldsByDataSourceName( - KAFKA_PLUGIN_NAME); + KAFKA_DATA_SOURCE_CHANNEL.getVirtualTableOptions(KAFKA_PLUGIN_NAME); Assertions.assertEquals( 2, datasourceMetadataFieldsByDataSourceName.getOptionalOptions().size()); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml new file mode 100644 index 000000000..c57a7f26e --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + org.apache.seatunnel + seatunnel-datasource-plugins + ${revision} + + + datasource-mongodb + + + + org.apache.seatunnel + datasource-plugins-api + ${project.version} + provided + + + + org.mongodb + mongodb-driver-sync + 4.7.1 + + + + org.apache.commons + commons-lang3 + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java new file mode 100644 index 000000000..6ed7fd334 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java @@ -0,0 +1,89 @@ +package com.apache.seatunnel.datasource.plugin.mongodb; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.collect.ImmutableList; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoIterable; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; + +@Slf4j +public class MongoDataSoueceChannel implements DataSourceChannel { + + private static final String DATABASE = "default"; + + @Override + public OptionRule getDataSourceOptions(@NonNull String pluginName) { + return MongoOptionRule.optionRule(); + } + + @Override + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { + return MongoOptionRule.metadataRule(); + } + + public List getTables( + @NonNull String pluginName, + Map requestParams, + String database, + Map options) { + checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); + + return Collections.emptyList(); + } + + @Override + public List getDatabases( + @NonNull String pluginName, @NonNull Map requestParams) { + return ImmutableList.of(DATABASE); + } + + @Override + public List getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { + checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); + return Collections.emptyList(); + } + + @Override + public boolean checkDataSourceConnectivity( + @NonNull String pluginName, @NonNull Map requestParams) { + + try (MongoClient mongoClient = createMongoClient(requestParams)) { + // Verify if the connection to mongodb was successful + MongoIterable databaseNames = mongoClient.listDatabaseNames(); + if (databaseNames.iterator().hasNext()) { + log.info("mongoDB connection successful"); + return true; + } else { + return false; + } + } catch (Exception e) { + throw new DataSourcePluginException( + "check MongoDB connectivity failed, " + e.getMessage(), e); + } + } + + // Resolve the URI in requestParams of Map type + private MongoClient createMongoClient(Map requestParams) { + + return MongoClients.create( + MongoRequestParamsUtils.parseStringFromRequestParams(requestParams)); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java new file mode 100644 index 000000000..e893ddcb8 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java @@ -0,0 +1,40 @@ +package com.apache.seatunnel.datasource.plugin.mongodb; + +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; +import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; + +import com.google.auto.service.AutoService; +import com.google.common.collect.Sets; + +import java.util.Set; + +@AutoService(DataSourceFactory.class) +public class MongoDataSourceFactory implements DataSourceFactory { + public static final String MONGO_PLUGIN_NAME = "MongoDB"; + public static final String MONGO_PLUGIN_ICON = "MongoDB"; + public static final String MONGO_PLUGIN_VERSION = "1.0.0"; + + @Override + public String factoryIdentifier() { + return MONGO_PLUGIN_NAME; + } + + @Override + public Set supportedDataSources() { + return Sets.newHashSet( + DataSourcePluginInfo.builder() + .name(MONGO_PLUGIN_NAME) + .icon(MONGO_PLUGIN_ICON) + .version(MONGO_PLUGIN_VERSION) + .supportVirtualTables(true) + .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) + .build()); + } + + @Override + public DataSourceChannel createChannel() { + return new MongoDataSoueceChannel(); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java new file mode 100644 index 000000000..69f05d3c3 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java @@ -0,0 +1,34 @@ +package com.apache.seatunnel.datasource.plugin.mongodb; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.util.OptionRule; + +public class MongoOptionRule { + + public static final Option URI = + Options.key("uri") + .stringType() + .noDefaultValue() + .withDescription("The MongoDB connection uri."); + + public static final Option DATABASE = + Options.key("database") + .stringType() + .noDefaultValue() + .withDescription("The name of MongoDB database to read or write."); + + public static final Option COLLECTION = + Options.key("collection") + .stringType() + .noDefaultValue() + .withDescription("The name of MongoDB collection to read or write."); + + public static OptionRule optionRule() { + return OptionRule.builder().required(URI).build(); + } + + public static OptionRule metadataRule() { + return OptionRule.builder().required(DATABASE, COLLECTION).build(); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java new file mode 100644 index 000000000..1329a39e1 --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java @@ -0,0 +1,16 @@ +package com.apache.seatunnel.datasource.plugin.mongodb; + +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; + +public class MongoRequestParamsUtils { + + public static String parseStringFromRequestParams(Map requestParams) { + checkArgument( + requestParams.containsKey(MongoOptionRule.URI.key()), + String.format("Missing %s in requestParams", MongoOptionRule.URI.key())); + + return requestParams.get(MongoOptionRule.URI.key()); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml new file mode 100644 index 000000000..a859d39ad --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + org.apache.seatunnel + seatunnel-datasource-plugins + ${revision} + + + datasource-mongodbcdc + + + + org.apache.seatunnel + datasource-plugins-api + ${project.version} + provided + + + + org.mongodb + mongodb-driver-sync + 4.7.1 + + + + org.apache.commons + commons-lang3 + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${e2e.dependency.skip} + true + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java new file mode 100644 index 000000000..35dbbcdda --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java @@ -0,0 +1,101 @@ +package com.apache.seatunnel.datasource.plugin.mongodbcdc; + +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.collect.ImmutableList; +import com.mongodb.ConnectionString; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoIterable; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static com.apache.seatunnel.datasource.plugin.mongodbcdc.MongodbCDCOptionRule.buildConnectionString; +import static com.google.common.base.Preconditions.checkArgument; + +@Slf4j +public class MongodbCDCDataSourceChannel implements DataSourceChannel { + + private static final String DATABASE = "default"; + + @Override + public OptionRule getDataSourceOptions(@NonNull String pluginName) { + return MongodbCDCOptionRule.optionRule(); + } + + @Override + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { + return MongodbCDCOptionRule.metadataRule(); + } + + public List getTables( + @NonNull String pluginName, + Map requestParams, + String database, + Map options) { + checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); + + return Collections.emptyList(); + } + + @Override + public List getDatabases( + @NonNull String pluginName, @NonNull Map requestParams) { + return ImmutableList.of(DATABASE); + } + + @Override + public List getTableFields( + @NonNull String pluginName, + @NonNull Map requestParams, + @NonNull String database, + @NonNull String table) { + checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); + return Collections.emptyList(); + } + + @Override + public boolean checkDataSourceConnectivity( + @NonNull String pluginName, @NonNull Map requestParams) { + + try (MongoClient mongoClient = createMongoClient(requestParams)) { + // Verify if the connection to mongodb was successful + MongoIterable databaseNames = mongoClient.listDatabaseNames(); + if (databaseNames.iterator().hasNext()) { + log.info("mongoDB connection successful"); + return true; + } else { + return false; + } + } catch (Exception e) { + throw new DataSourcePluginException( + "check MongoDB connectivity failed, " + e.getMessage(), e); + } + } + + // Resolve the URI in requestParams of Map type + private MongoClient createMongoClient(Map requestParams) { + + checkArgument( + requestParams.containsKey(MongodbCDCOptionRule.HOSTS.key()), + String.format("Missing %s in requestParams", MongodbCDCOptionRule.HOSTS.key())); + + ConnectionString connectionString = + buildConnectionString( + requestParams.get(MongodbCDCOptionRule.USERNAME.key()), + requestParams.get(MongodbCDCOptionRule.PASSWORD.key()), + requestParams.get(MongodbCDCOptionRule.HOSTS.key()), + requestParams.get(MongodbCDCOptionRule.CONNECTION_OPTIONS.key())); + + return MongoClients.create(connectionString); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java new file mode 100644 index 000000000..98c2277cb --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java @@ -0,0 +1,40 @@ +package com.apache.seatunnel.datasource.plugin.mongodbcdc; + +import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; +import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; +import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; +import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; + +import com.google.auto.service.AutoService; +import com.google.common.collect.Sets; + +import java.util.Set; + +@AutoService(DataSourceFactory.class) +public class MongodbCDCDataSourceFactory implements DataSourceFactory { + public static final String MONGO_PLUGIN_NAME = "MongoDB-CDC"; + public static final String MONGO_PLUGIN_ICON = "MongoDB-CDC"; + public static final String MONGO_PLUGIN_VERSION = "1.0.0"; + + @Override + public String factoryIdentifier() { + return MONGO_PLUGIN_NAME; + } + + @Override + public Set supportedDataSources() { + return Sets.newHashSet( + DataSourcePluginInfo.builder() + .name(MONGO_PLUGIN_NAME) + .icon(MONGO_PLUGIN_ICON) + .version(MONGO_PLUGIN_VERSION) + .supportVirtualTables(true) + .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) + .build()); + } + + @Override + public DataSourceChannel createChannel() { + return new MongodbCDCDataSourceChannel(); + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java new file mode 100644 index 000000000..bc12f845e --- /dev/null +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java @@ -0,0 +1,108 @@ +package com.apache.seatunnel.datasource.plugin.mongodbcdc; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.common.utils.SeaTunnelException; + +import org.apache.commons.lang3.StringUtils; + +import com.mongodb.ConnectionString; + +import javax.annotation.Nonnull; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +public class MongodbCDCOptionRule { + + public static final Option HOSTS = + Options.key("hosts") + .stringType() + .noDefaultValue() + .withDescription( + "The comma-separated list of hostname and port pairs of the MongoDB servers. " + + "eg. localhost:27017,localhost:27018"); + + public static final Option CONNECTION_OPTIONS = + Options.key("connection.options") + .stringType() + .noDefaultValue() + .withDescription( + "The ampersand-separated MongoDB connection options. " + + "eg. replicaSet=test&connectTimeoutMS=300000"); + + public static final Option USERNAME = + Options.key("username") + .stringType() + .noDefaultValue() + .withDescription( + "Name of the database user to be used when connecting to MongoDB. " + + "This is required only when MongoDB is configured to use authentication."); + + public static final Option PASSWORD = + Options.key("password") + .stringType() + .noDefaultValue() + .withDescription( + "Password to be used when connecting to MongoDB. " + + "This is required only when MongoDB is configured to use authentication."); + + public static final Option DATABASE = + Options.key("database") + .stringType() + .noDefaultValue() + .withDescription("The name of MongoDB database to read or write."); + + public static final Option COLLECTION = + Options.key("collection") + .stringType() + .noDefaultValue() + .withDescription("The name of MongoDB collection to read or write."); + + public static OptionRule optionRule() { + return OptionRule.builder() + .required(HOSTS) + .optional(CONNECTION_OPTIONS, USERNAME, PASSWORD) + .build(); + } + + public static OptionRule metadataRule() { + return OptionRule.builder().required(DATABASE, COLLECTION).build(); + } + + public static @Nonnull ConnectionString buildConnectionString( + String username, String password, String hosts, String connectionOptions) { + StringBuilder sb = new StringBuilder("mongodb://"); + + if (hasCredentials(username, password)) { + appendCredentials(sb, username, password); + } + + sb.append(hosts); + + if (StringUtils.isNotEmpty(connectionOptions)) { + sb.append("/?").append(connectionOptions); + } + + return new ConnectionString(sb.toString()); + } + + private static boolean hasCredentials(String username, String password) { + return StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password); + } + + private static void appendCredentials( + @Nonnull StringBuilder sb, String username, String password) { + sb.append(encodeValue(username)).append(":").append(encodeValue(password)).append("@"); + } + + public static String encodeValue(String value) { + try { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new SeaTunnelException(e.getMessage()); + } + } +} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java index 1cd99d362..54d492bf4 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mysql-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/mysql/MysqlCDCDataSourceChannel.java @@ -56,7 +56,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return MysqlCDCOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java index 72a2ea208..73353f4ee 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-plugins-api/src/main/java/org/apache/seatunnel/datasource/plugin/api/DataSourceChannel.java @@ -28,13 +28,15 @@ import java.sql.Connection; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public interface DataSourceChannel { List DEFAULT_DATABASES = ImmutableList.of("default"); /** - * get datasource metadata fields by datasource name + * This method is used to define the datasource options which is used to create a datasource. * * @param pluginName plugin name * @return datasource metadata fields @@ -42,12 +44,13 @@ public interface DataSourceChannel { OptionRule getDataSourceOptions(@NonNull String pluginName); /** - * get datasource metadata fields by datasource name + * This method is used to define the datasource options which is used to create the virtual + * table. * * @param pluginName plugin name * @return datasource metadata fields */ - OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName); + OptionRule getVirtualTableOptions(@NonNull String pluginName); List getTables( @NonNull String pluginName, @@ -71,11 +74,19 @@ List getTableFields( @NonNull String database, @NonNull String table); - Map> getTableFields( + default Map> getTableFields( @NonNull String pluginName, @NonNull Map requestParams, @NonNull String database, - @NonNull List tables); + @NonNull List tables) { + return tables.parallelStream() + .collect( + Collectors.toMap( + Function.identity(), + table -> + getTableFields( + pluginName, requestParams, database, table))); + } /** * just check metadata field is right and used by virtual table @@ -87,6 +98,16 @@ default Boolean checkMetadataFieldIsRight(Map requestParams) { return true; } + /** + * Gets the maximum value of synchronization at the time of synchronization + * + * @param pluginName pluginName + * @param requestParams Map connector params include url, driver, updateType …… + * @param databaseName database name + * @param tableName table name + * @param updateFieldType table updateValue type + * @return return str + */ default Pair getTableSyncMaxValue( String pluginName, Map requestParams, diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java index c045b5a4b..20db9c032 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3/src/main/java/org/apache/seatunnel/datasource/plugin/s3/S3DatasourceChannel.java @@ -39,7 +39,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return S3OptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java index fb47afd2b..2eb7032c1 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-sqlserver-cdc/src/main/java/org/apache/seatunnel/datasource/plugin/cdc/sqlserver/SqlServerCDCDataSourceChannel.java @@ -59,7 +59,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return SqlServerCDCOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java index 9ae74fff5..56bffa20e 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-starrocks/src/main/java/org/apache/seatunnel/datasource/plugin/starrocks/StarRocksDataSourceChannel.java @@ -54,7 +54,7 @@ public OptionRule getDataSourceOptions(@NonNull String pluginName) { } @Override - public OptionRule getDatasourceMetadataFieldsByDataSourceName(@NonNull String pluginName) { + public OptionRule getVirtualTableOptions(@NonNull String pluginName) { return StarRocksOptionRule.metadataRule(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml index 5f2296bb3..a751dd7e2 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml @@ -44,6 +44,10 @@ datasource-s3 datasource-sqlserver-cdc datasource-jdbc-tidb + datasource-jdbc-common + datasource-jdbc-doris + datasource-mongodb + datasource-mongodbcdc diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java new file mode 100644 index 000000000..76ac60585 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java @@ -0,0 +1,123 @@ +package org.whaleops.whaletunnel.web.app.thirdpart.datasource; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.app.domain.request.job.DatabaseTableSchemaReq; +import org.apache.seatunnel.app.domain.request.job.SelectTableFields; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; +import org.apache.seatunnel.datasource.plugin.api.model.TableField; + +import org.apache.commons.collections4.CollectionUtils; + +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +@Slf4j +public class SchemaGenerator { + + private SchemaGenerator() {} + + /** + * Generate the schema of the table. + * + * 
+     * fields {
+     *        name = "string"
+     *        age = "int"
+     *       }
+     *
+ * + * @param virtualTableDetailRes virtual table detail. + * @param selectTableFields select table fields which need to be placed in the schema. + * @return schema. + */ + public static Config generateSchemaBySelectTableFields( + VirtualTableDetailRes virtualTableDetailRes, SelectTableFields selectTableFields) { + checkNotNull(selectTableFields, "selectTableFields cannot be null"); + checkArgument( + CollectionUtils.isNotEmpty(selectTableFields.getTableFields()), + "selectTableFields.tableFields cannot be empty"); + + checkNotNull(virtualTableDetailRes, "virtualTableDetailRes cannot be null"); + checkArgument( + CollectionUtils.isNotEmpty(virtualTableDetailRes.getFields()), + "virtualTableDetailRes.fields cannot be empty"); + + Map fieldTypeMap = + virtualTableDetailRes.getFields().stream() + .collect( + Collectors.toMap( + VirtualTableFieldRes::getFieldName, Function.identity())); + + Config schema = ConfigFactory.empty(); + for (String fieldName : selectTableFields.getTableFields()) { + VirtualTableFieldRes virtualTableFieldRes = + checkNotNull( + fieldTypeMap.get(fieldName), + String.format( + "Cannot find the field: %s from virtual table", fieldName)); + schema = + schema.withValue( + fieldName, + ConfigValueFactory.fromAnyRef(virtualTableFieldRes.getFieldType())); + } + return schema.atKey("fields"); + } + + // todo: how to deal with multiple table has the same field? + /** + * Generate the schema of the table. + * + * 
+     * fields {
+     *        name = "string"
+     *        age = "int"
+     *       }
+     *
+ * + * @param outputSchemas schema in the ui. + * @param selectTableFields select table fields which need to be placed in the schema. + * @return schema. + */ + public static Config generateSchemaBySelectTableFields( + List outputSchemas, SelectTableFields selectTableFields) { + checkNotNull(selectTableFields, "selectTableFields cannot be null"); + checkArgument( + CollectionUtils.isNotEmpty(selectTableFields.getTableFields()), + "selectTableFields.tableFields cannot be empty"); + + checkArgument( + CollectionUtils.isNotEmpty(outputSchemas), + "selectTableFields.tableFields cannot be empty"); + Config schema = ConfigFactory.empty(); + + Map tableFieldMap = + outputSchemas.stream() + .flatMap( + databaseTableSchemaReq -> + databaseTableSchemaReq.getFields().stream()) + .collect(Collectors.toMap(TableField::getName, Function.identity())); + for (String fieldName : selectTableFields.getTableFields()) { + TableField tableField = + checkNotNull( + tableFieldMap.get(fieldName), + String.format( + "Cannot find the field: %s from virtual table", fieldName)); + schema = + schema.withValue( + fieldName, + ConfigValueFactory.fromAnyRef(tableField.getOutputDataType())); + } + return schema.atKey("fields"); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java new file mode 100644 index 000000000..ce8e3b4b0 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java @@ -0,0 +1,98 @@ +package org.apache.seatunnel.app.thirdparty.datasource.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; +import org.apache.seatunnel.app.domain.request.connector.BusinessMode; +import org.apache.seatunnel.app.domain.request.job.DataSourceOption; +import org.apache.seatunnel.app.domain.request.job.SelectTableFields; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.dynamicforms.FormStructure; +import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; +import org.apache.seatunnel.common.constants.PluginType; + +import com.google.auto.service.AutoService; + +import java.util.List; + +@AutoService(DataSourceConfigSwitcher.class) +public class DorisDatasourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { + + private static final String TABLE = "table.identifier"; + + @Override + public String getDataSourceName() { + return "DORIS"; + } + + @Override + public FormStructure filterOptionRule( + String connectorName, + OptionRule dataSourceOptionRule, + OptionRule virtualTableOptionRule, + BusinessMode businessMode, + PluginType pluginType, + OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, + List excludedKeys) { + if (PluginType.SOURCE.equals(pluginType)) { + throw new UnsupportedOperationException("Unsupported PluginType: " + pluginType); + } else if (PluginType.SINK.equals(pluginType)) { + excludedKeys.add(TABLE); + } else { + throw new UnsupportedOperationException("Unsupported plugin type: " + pluginType); + } + return super.filterOptionRule( + connectorName, + dataSourceOptionRule, + virtualTableOptionRule, + businessMode, + pluginType, + connectorOptionRule, + addRequiredOptions, + addOptionalOptions, + excludedKeys); + } + + @Override + public Config mergeDatasourceConfig( + Config dataSourceInstanceConfig, + VirtualTableDetailRes virtualTableDetail, + DataSourceOption dataSourceOption, + SelectTableFields selectTableFields, + BusinessMode businessMode, + PluginType pluginType, + Config connectorConfig) { + if (PluginType.SOURCE.equals(pluginType)) { + throw new UnsupportedOperationException("Unsupported PluginType: " + pluginType); + } else if (PluginType.SINK.equals(pluginType)) { + if (businessMode.equals(BusinessMode.DATA_INTEGRATION)) { + connectorConfig = + connectorConfig.withValue( + TABLE, + ConfigValueFactory.fromAnyRef( + dataSourceOption + .getDatabases() + .get(0) + .concat(".") + .concat(dataSourceOption.getTables().get(0)))); + } + } else { + throw new UnsupportedOperationException("Unsupported plugin type: " + pluginType); + } + + return super.mergeDatasourceConfig( + dataSourceInstanceConfig, + virtualTableDetail, + dataSourceOption, + selectTableFields, + businessMode, + pluginType, + connectorConfig); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java new file mode 100644 index 000000000..74080b8f6 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java @@ -0,0 +1,102 @@ +package org.apache.seatunnel.app.thirdparty.datasource.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; +import org.apache.seatunnel.app.domain.request.connector.BusinessMode; +import org.apache.seatunnel.app.domain.request.job.DataSourceOption; +import org.apache.seatunnel.app.domain.request.job.SelectTableFields; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.dynamicforms.FormStructure; +import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; +import org.apache.seatunnel.common.constants.PluginType; + +import org.whaleops.whaletunnel.web.app.thirdpart.datasource.SchemaGenerator; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.List; + +@AutoService(DataSourceConfigSwitcher.class) +@Slf4j +public class MongoDBCDCDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { + + private static final String DATABASE = "database"; + private static final String COLLECTION = "collection"; + private static final String SCHEMA = "schema"; + + @Override + public String getDataSourceName() { + return "MONGODB-CDC"; + } + + @Override + public FormStructure filterOptionRule( + String connectorName, + OptionRule dataSourceOptionRule, + OptionRule virtualTableOptionRule, + BusinessMode businessMode, + PluginType pluginType, + OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, + List excludedKeys) { + excludedKeys.add(SCHEMA); + return super.filterOptionRule( + connectorName, + dataSourceOptionRule, + virtualTableOptionRule, + businessMode, + pluginType, + connectorOptionRule, + addRequiredOptions, + addOptionalOptions, + excludedKeys); + } + + @Override + public Config mergeDatasourceConfig( + Config dataSourceInstanceConfig, + VirtualTableDetailRes virtualTableDetail, + DataSourceOption dataSourceOption, + SelectTableFields selectTableFields, + BusinessMode businessMode, + PluginType pluginType, + Config connectorConfig) { + // Use field to generate the schema + + String databaseName = virtualTableDetail.getDatasourceProperties().get(DATABASE); + String collectionName = virtualTableDetail.getDatasourceProperties().get(COLLECTION); + + connectorConfig = + connectorConfig.withValue( + DATABASE, + ConfigValueFactory.fromIterable(Collections.singleton(databaseName))); + connectorConfig = + connectorConfig.withValue( + COLLECTION, + ConfigValueFactory.fromIterable( + Collections.singleton(databaseName + "." + collectionName))); + connectorConfig = + connectorConfig.withValue( + SCHEMA, + SchemaGenerator.generateSchemaBySelectTableFields( + virtualTableDetail, selectTableFields) + .root()); + + return super.mergeDatasourceConfig( + dataSourceInstanceConfig, + virtualTableDetail, + dataSourceOption, + selectTableFields, + businessMode, + pluginType, + connectorConfig); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java new file mode 100644 index 000000000..b6fb1f799 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java @@ -0,0 +1,98 @@ +package org.apache.seatunnel.app.thirdparty.datasource.impl; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.configuration.util.RequiredOption; +import org.apache.seatunnel.app.domain.request.connector.BusinessMode; +import org.apache.seatunnel.app.domain.request.job.DataSourceOption; +import org.apache.seatunnel.app.domain.request.job.SelectTableFields; +import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; +import org.apache.seatunnel.app.dynamicforms.FormStructure; +import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; +import org.apache.seatunnel.common.constants.PluginType; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@AutoService(DataSourceConfigSwitcher.class) +@Slf4j +public class MongoDBDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { + private static final String DATABASE = "database"; + private static final String COLLECTION = "collection"; + private static final String SCHEMA = "schema"; + + @Override + public String getDataSourceName() { + return "MONGODB"; + } + + @Override + public FormStructure filterOptionRule( + String connectorName, + OptionRule dataSourceOptionRule, + OptionRule virtualTableOptionRule, + BusinessMode businessMode, + PluginType pluginType, + OptionRule connectorOptionRule, + List addRequiredOptions, + List> addOptionalOptions, + List excludedKeys) { + excludedKeys.add(SCHEMA); + return super.filterOptionRule( + connectorName, + dataSourceOptionRule, + virtualTableOptionRule, + businessMode, + pluginType, + connectorOptionRule, + addRequiredOptions, + addOptionalOptions, + excludedKeys); + } + + @Override + public Config mergeDatasourceConfig( + Config dataSourceInstanceConfig, + VirtualTableDetailRes virtualTableDetail, + DataSourceOption dataSourceOption, + SelectTableFields selectTableFields, + BusinessMode businessMode, + PluginType pluginType, + Config connectorConfig) { + // Use field to generate the schema + connectorConfig = + connectorConfig.withValue( + DATABASE, + ConfigValueFactory.fromAnyRef( + virtualTableDetail.getDatasourceProperties().get(DATABASE))); + connectorConfig = + connectorConfig.withValue( + COLLECTION, + ConfigValueFactory.fromAnyRef( + virtualTableDetail.getDatasourceProperties().get(COLLECTION))); + if (pluginType == PluginType.SOURCE) { + connectorConfig = + connectorConfig.withValue( + SCHEMA, + KafkaKingbaseDataSourceConfigSwitcher.SchemaGenerator + .generateSchemaBySelectTableFields( + virtualTableDetail, selectTableFields) + .root()); + } + + return super.mergeDatasourceConfig( + dataSourceInstanceConfig, + virtualTableDetail, + dataSourceOption, + selectTableFields, + businessMode, + pluginType, + connectorConfig); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml index 917c677b7..f7e62212c 100644 --- a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml +++ b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml @@ -54,6 +54,18 @@ connector-datasource-mapper: dataSources: - Postgres-CDC + Doris: + dataSources: + - Doris + + MongoDB: + dataSources: + - MongoDB + + MongoDB-CDC: + dataSources: + - MongoDB-CDC + sourceDatasourceFeatures: JDBC-Mysql: businessMode: @@ -123,6 +135,22 @@ connector-datasource-mapper: - MULTIPLE_TABLE - SPLIT_TABLE + MongoDB: + businessMode: + - DATA_INTEGRATION + sceneMode: + - SINGLE_TABLE + jobMode: + - BATCH + + MongoDB-CDC: + businessMode: + - DATA_INTEGRATION + sceneMode: + - SINGLE_TABLE + jobMode: + - STREAMING + sinkDatasourceFeatures: @@ -196,3 +224,17 @@ connector-datasource-mapper: - DATA_INTEGRATION sceneMode: - SINGLE_TABLE + + Doris: + businessMode: + - DATA_INTEGRATION + sceneMode: + - SINGLE_TABLE + + MongoDB: + businessMode: + - DATA_INTEGRATION + - DATA_REPLICA + sceneMode: + - SINGLE_TABLE + - MULTIPLE_TABLE From 051fa97c0fae861153ceec004fb1fdade810b5c6 Mon Sep 17 00:00:00 2001 From: jiayang Date: Mon, 23 Oct 2023 10:45:38 +0800 Subject: [PATCH 2/6] [feature] add license --- .../datasource-jdbc-common/pom.xml | 14 ++++++++++++++ .../common/JdbcDataSourceLRUConnectionPool.java | 17 +++++++++++++++++ .../datasource-jdbc-doris/pom.xml | 14 ++++++++++++++ .../doris/jdbc/DorisDataSourceConfig.java | 17 +++++++++++++++++ .../doris/jdbc/DorisJdbcDataSourceChannel.java | 17 +++++++++++++++++ .../doris/jdbc/DorisJdbcDataSourceFactory.java | 17 +++++++++++++++++ .../plugin/doris/jdbc/DorisOptionRule.java | 17 +++++++++++++++++ .../datasource-mongodb/pom.xml | 14 ++++++++++++++ .../plugin/mongodb/MongoDataSoueceChannel.java | 17 +++++++++++++++++ .../plugin/mongodb/MongoDataSourceFactory.java | 17 +++++++++++++++++ .../plugin/mongodb/MongoOptionRule.java | 17 +++++++++++++++++ .../plugin/mongodb/MongoRequestParamsUtils.java | 17 +++++++++++++++++ .../datasource-mongodbcdc/pom.xml | 14 ++++++++++++++ .../mongodbcdc/MongodbCDCDataSourceChannel.java | 17 +++++++++++++++++ .../mongodbcdc/MongodbCDCDataSourceFactory.java | 17 +++++++++++++++++ .../plugin/mongodbcdc/MongodbCDCOptionRule.java | 17 +++++++++++++++++ 16 files changed, 260 insertions(+) diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml index e6219da8f..0cc26999c 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java index 758930831..140e61f40 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-common/src/main/java/com/whaleops/datasource/plugin/jdbc/common/JdbcDataSourceLRUConnectionPool.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.whaleops.datasource.plugin.jdbc.common; import com.google.common.cache.Cache; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml index afc9a7c62..62d4f65fb 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java index 4a08e24d7..0b01f3684 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisDataSourceConfig.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.doris.jdbc; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java index 29dc9b1b8..b8135dac6 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceChannel.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.doris.jdbc; import org.apache.seatunnel.api.configuration.util.OptionRule; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java index addcbe378..1246a3898 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisJdbcDataSourceFactory.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.doris.jdbc; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java index 35b8365bf..6bfba3ae5 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-doris/src/main/java/com/apache/seatunnel/datasource/plugin/doris/jdbc/DorisOptionRule.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.doris.jdbc; import org.apache.seatunnel.api.configuration.Option; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml index c57a7f26e..8bbf4dcf7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java index 6ed7fd334..c052840bd 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodb; import org.apache.seatunnel.api.configuration.util.OptionRule; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java index e893ddcb8..b08f5b305 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodb; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java index 69f05d3c3..cde0f89fd 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodb; import org.apache.seatunnel.api.configuration.Option; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java index 1329a39e1..eb342acff 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodb; import java.util.Map; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml index a859d39ad..657eba3c2 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java index 35dbbcdda..c504e39b1 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodbcdc; import org.apache.seatunnel.api.configuration.util.OptionRule; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java index 98c2277cb..70b6f4f20 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodbcdc; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java index bc12f845e..9d5b124ec 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.apache.seatunnel.datasource.plugin.mongodbcdc; import org.apache.seatunnel.api.configuration.Option; From c6d19e6423fa7787bde9ed7b95769fafce1af72d Mon Sep 17 00:00:00 2001 From: jiayang Date: Mon, 23 Oct 2023 15:05:48 +0800 Subject: [PATCH 3/6] [feature] add licensed --- .../datasource/SchemaGenerator.java | 19 ++++++++++++++++++- .../impl/DorisDatasourceConfigSwitcher.java | 17 +++++++++++++++++ .../MongoDBCDCDataSourceConfigSwitcher.java | 17 +++++++++++++++++ .../impl/MongoDBDataSourceConfigSwitcher.java | 17 +++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java index 76ac60585..198ee44d8 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java @@ -1,4 +1,21 @@ -package org.whaleops.whaletunnel.web.app.thirdpart.datasource; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.thirdparty.datasource; import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java index ce8e3b4b0..88273a963 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/DorisDatasourceConfigSwitcher.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.app.thirdparty.datasource.impl; import org.apache.seatunnel.shade.com.typesafe.config.Config; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java index 74080b8f6..140993a70 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.app.thirdparty.datasource.impl; import org.apache.seatunnel.shade.com.typesafe.config.Config; diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java index b6fb1f799..2153c52e7 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.app.thirdparty.datasource.impl; import org.apache.seatunnel.shade.com.typesafe.config.Config; From efcde6eaf2471f8d259690ef56ca33bf9d69a1c4 Mon Sep 17 00:00:00 2001 From: jiayang Date: Mon, 23 Oct 2023 18:28:40 +0800 Subject: [PATCH 4/6] [feature] update package name --- .../datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java index 140993a70..2417e0119 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java @@ -30,10 +30,9 @@ import org.apache.seatunnel.app.dynamicforms.FormStructure; import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; +import org.apache.seatunnel.app.thirdparty.datasource.SchemaGenerator; import org.apache.seatunnel.common.constants.PluginType; -import org.whaleops.whaletunnel.web.app.thirdpart.datasource.SchemaGenerator; - import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; From 9a26af41911aec679d8dd83ebc0e8a58a64a898d Mon Sep 17 00:00:00 2001 From: jiayang Date: Thu, 26 Oct 2023 20:10:12 +0800 Subject: [PATCH 5/6] [feature] add doris --- .../classloader/DatasourceLoadConfig.java | 11 +- .../datasource-mongodb/pom.xml | 64 --------- .../mongodb/MongoDataSoueceChannel.java | 106 --------------- .../mongodb/MongoDataSourceFactory.java | 57 -------- .../plugin/mongodb/MongoOptionRule.java | 51 ------- .../mongodb/MongoRequestParamsUtils.java | 33 ----- .../datasource-mongodbcdc/pom.xml | 64 --------- .../MongodbCDCDataSourceChannel.java | 118 ----------------- .../MongodbCDCDataSourceFactory.java | 57 -------- .../mongodbcdc/MongodbCDCOptionRule.java | 125 ------------------ .../seatunnel-datasource-plugins/pom.xml | 2 - .../src/main/bin/download_datasource.sh | 1 + seatunnel-web-dist/pom.xml | 13 +- 13 files changed, 14 insertions(+), 688 deletions(-) delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java delete mode 100644 seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java diff --git a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java index 6dc1fad62..97063ff4d 100644 --- a/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-client/src/main/java/org/apache/seatunnel/datasource/classloader/DatasourceLoadConfig.java @@ -95,11 +95,6 @@ public class DatasourceLoadConfig { classLoaderFactoryName.put( "DORIS", "com.apache.seatunnel.datasource.plugin.doris.jdbc.DorisJdbcDataSourceFactory"); - classLoaderFactoryName.put( - "MONGODB", "com.apache.seatunnel.datasource.plugin.mongodb.MongoDataSourceFactory"); - classLoaderFactoryName.put( - "MONGODB-CDC", - "com.apache.seatunnel.datasource.plugin.mongodbcdc.MongodbCDCDataSourceFactory"); classLoaderJarName.put("JDBC-ORACLE", "datasource-jdbc-oracle-"); classLoaderJarName.put("JDBC-CLICKHOUSE", "datasource-jdbc-clickhouse-"); @@ -120,8 +115,6 @@ public class DatasourceLoadConfig { classLoaderJarName.put("S3-REDSHIFT", "datasource-s3redshift-"); classLoaderJarName.put("JDBC-STARROCKS", "datasource-jdbc-starrocks-"); classLoaderJarName.put("DORIS", "datasource-jdbc-doris-"); - classLoaderJarName.put("MONGODB", "datasource-mongodb-"); - classLoaderJarName.put("MONGODB-CDC", "datasource-mongodbcdc-"); } public static final Set pluginSet = @@ -139,9 +132,7 @@ public class DatasourceLoadConfig { "S3", "SqlServer-CDC", "Doris", - "StarRocks", - "MongoDB", - "MongoDB-CDC"); + "StarRocks"); public static Map datasourceClassLoaders = new HashMap<>(); diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml deleted file mode 100644 index 8bbf4dcf7..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-datasource-plugins - ${revision} - - - datasource-mongodb - - - - org.apache.seatunnel - datasource-plugins-api - ${project.version} - provided - - - - org.mongodb - mongodb-driver-sync - 4.7.1 - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - ${e2e.dependency.skip} - true - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java deleted file mode 100644 index c052840bd..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSoueceChannel.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodb; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; -import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; -import org.apache.seatunnel.datasource.plugin.api.model.TableField; - -import org.apache.commons.lang3.StringUtils; - -import com.google.common.collect.ImmutableList; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoIterable; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; - -@Slf4j -public class MongoDataSoueceChannel implements DataSourceChannel { - - private static final String DATABASE = "default"; - - @Override - public OptionRule getDataSourceOptions(@NonNull String pluginName) { - return MongoOptionRule.optionRule(); - } - - @Override - public OptionRule getVirtualTableOptions(@NonNull String pluginName) { - return MongoOptionRule.metadataRule(); - } - - public List getTables( - @NonNull String pluginName, - Map requestParams, - String database, - Map options) { - checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); - - return Collections.emptyList(); - } - - @Override - public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { - return ImmutableList.of(DATABASE); - } - - @Override - public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { - checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); - return Collections.emptyList(); - } - - @Override - public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { - - try (MongoClient mongoClient = createMongoClient(requestParams)) { - // Verify if the connection to mongodb was successful - MongoIterable databaseNames = mongoClient.listDatabaseNames(); - if (databaseNames.iterator().hasNext()) { - log.info("mongoDB connection successful"); - return true; - } else { - return false; - } - } catch (Exception e) { - throw new DataSourcePluginException( - "check MongoDB connectivity failed, " + e.getMessage(), e); - } - } - - // Resolve the URI in requestParams of Map type - private MongoClient createMongoClient(Map requestParams) { - - return MongoClients.create( - MongoRequestParamsUtils.parseStringFromRequestParams(requestParams)); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java deleted file mode 100644 index b08f5b305..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoDataSourceFactory.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodb; - -import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; -import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; -import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; -import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; - -import com.google.auto.service.AutoService; -import com.google.common.collect.Sets; - -import java.util.Set; - -@AutoService(DataSourceFactory.class) -public class MongoDataSourceFactory implements DataSourceFactory { - public static final String MONGO_PLUGIN_NAME = "MongoDB"; - public static final String MONGO_PLUGIN_ICON = "MongoDB"; - public static final String MONGO_PLUGIN_VERSION = "1.0.0"; - - @Override - public String factoryIdentifier() { - return MONGO_PLUGIN_NAME; - } - - @Override - public Set supportedDataSources() { - return Sets.newHashSet( - DataSourcePluginInfo.builder() - .name(MONGO_PLUGIN_NAME) - .icon(MONGO_PLUGIN_ICON) - .version(MONGO_PLUGIN_VERSION) - .supportVirtualTables(true) - .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) - .build()); - } - - @Override - public DataSourceChannel createChannel() { - return new MongoDataSoueceChannel(); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java deleted file mode 100644 index cde0f89fd..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoOptionRule.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodb; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; -import org.apache.seatunnel.api.configuration.util.OptionRule; - -public class MongoOptionRule { - - public static final Option URI = - Options.key("uri") - .stringType() - .noDefaultValue() - .withDescription("The MongoDB connection uri."); - - public static final Option DATABASE = - Options.key("database") - .stringType() - .noDefaultValue() - .withDescription("The name of MongoDB database to read or write."); - - public static final Option COLLECTION = - Options.key("collection") - .stringType() - .noDefaultValue() - .withDescription("The name of MongoDB collection to read or write."); - - public static OptionRule optionRule() { - return OptionRule.builder().required(URI).build(); - } - - public static OptionRule metadataRule() { - return OptionRule.builder().required(DATABASE, COLLECTION).build(); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java deleted file mode 100644 index eb342acff..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodb/src/main/java/com/apache/seatunnel/datasource/plugin/mongodb/MongoRequestParamsUtils.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodb; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkArgument; - -public class MongoRequestParamsUtils { - - public static String parseStringFromRequestParams(Map requestParams) { - checkArgument( - requestParams.containsKey(MongoOptionRule.URI.key()), - String.format("Missing %s in requestParams", MongoOptionRule.URI.key())); - - return requestParams.get(MongoOptionRule.URI.key()); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml deleted file mode 100644 index 657eba3c2..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-datasource-plugins - ${revision} - - - datasource-mongodbcdc - - - - org.apache.seatunnel - datasource-plugins-api - ${project.version} - provided - - - - org.mongodb - mongodb-driver-sync - 4.7.1 - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - ${e2e.dependency.skip} - true - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java deleted file mode 100644 index c504e39b1..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceChannel.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodbcdc; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; -import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; -import org.apache.seatunnel.datasource.plugin.api.model.TableField; - -import org.apache.commons.lang3.StringUtils; - -import com.google.common.collect.ImmutableList; -import com.mongodb.ConnectionString; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoIterable; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static com.apache.seatunnel.datasource.plugin.mongodbcdc.MongodbCDCOptionRule.buildConnectionString; -import static com.google.common.base.Preconditions.checkArgument; - -@Slf4j -public class MongodbCDCDataSourceChannel implements DataSourceChannel { - - private static final String DATABASE = "default"; - - @Override - public OptionRule getDataSourceOptions(@NonNull String pluginName) { - return MongodbCDCOptionRule.optionRule(); - } - - @Override - public OptionRule getVirtualTableOptions(@NonNull String pluginName) { - return MongodbCDCOptionRule.metadataRule(); - } - - public List getTables( - @NonNull String pluginName, - Map requestParams, - String database, - Map options) { - checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); - - return Collections.emptyList(); - } - - @Override - public List getDatabases( - @NonNull String pluginName, @NonNull Map requestParams) { - return ImmutableList.of(DATABASE); - } - - @Override - public List getTableFields( - @NonNull String pluginName, - @NonNull Map requestParams, - @NonNull String database, - @NonNull String table) { - checkArgument(StringUtils.equalsIgnoreCase(database, DATABASE), "database must be default"); - return Collections.emptyList(); - } - - @Override - public boolean checkDataSourceConnectivity( - @NonNull String pluginName, @NonNull Map requestParams) { - - try (MongoClient mongoClient = createMongoClient(requestParams)) { - // Verify if the connection to mongodb was successful - MongoIterable databaseNames = mongoClient.listDatabaseNames(); - if (databaseNames.iterator().hasNext()) { - log.info("mongoDB connection successful"); - return true; - } else { - return false; - } - } catch (Exception e) { - throw new DataSourcePluginException( - "check MongoDB connectivity failed, " + e.getMessage(), e); - } - } - - // Resolve the URI in requestParams of Map type - private MongoClient createMongoClient(Map requestParams) { - - checkArgument( - requestParams.containsKey(MongodbCDCOptionRule.HOSTS.key()), - String.format("Missing %s in requestParams", MongodbCDCOptionRule.HOSTS.key())); - - ConnectionString connectionString = - buildConnectionString( - requestParams.get(MongodbCDCOptionRule.USERNAME.key()), - requestParams.get(MongodbCDCOptionRule.PASSWORD.key()), - requestParams.get(MongodbCDCOptionRule.HOSTS.key()), - requestParams.get(MongodbCDCOptionRule.CONNECTION_OPTIONS.key())); - - return MongoClients.create(connectionString); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java deleted file mode 100644 index 70b6f4f20..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCDataSourceFactory.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodbcdc; - -import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; -import org.apache.seatunnel.datasource.plugin.api.DataSourceFactory; -import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; -import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; - -import com.google.auto.service.AutoService; -import com.google.common.collect.Sets; - -import java.util.Set; - -@AutoService(DataSourceFactory.class) -public class MongodbCDCDataSourceFactory implements DataSourceFactory { - public static final String MONGO_PLUGIN_NAME = "MongoDB-CDC"; - public static final String MONGO_PLUGIN_ICON = "MongoDB-CDC"; - public static final String MONGO_PLUGIN_VERSION = "1.0.0"; - - @Override - public String factoryIdentifier() { - return MONGO_PLUGIN_NAME; - } - - @Override - public Set supportedDataSources() { - return Sets.newHashSet( - DataSourcePluginInfo.builder() - .name(MONGO_PLUGIN_NAME) - .icon(MONGO_PLUGIN_ICON) - .version(MONGO_PLUGIN_VERSION) - .supportVirtualTables(true) - .type(DatasourcePluginTypeEnum.NO_STRUCTURED.getCode()) - .build()); - } - - @Override - public DataSourceChannel createChannel() { - return new MongodbCDCDataSourceChannel(); - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java deleted file mode 100644 index 9d5b124ec..000000000 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-mongodbcdc/src/main/java/com/apache/seatunnel/datasource/plugin/mongodbcdc/MongodbCDCOptionRule.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apache.seatunnel.datasource.plugin.mongodbcdc; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.common.utils.SeaTunnelException; - -import org.apache.commons.lang3.StringUtils; - -import com.mongodb.ConnectionString; - -import javax.annotation.Nonnull; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; - -public class MongodbCDCOptionRule { - - public static final Option HOSTS = - Options.key("hosts") - .stringType() - .noDefaultValue() - .withDescription( - "The comma-separated list of hostname and port pairs of the MongoDB servers. " - + "eg. localhost:27017,localhost:27018"); - - public static final Option CONNECTION_OPTIONS = - Options.key("connection.options") - .stringType() - .noDefaultValue() - .withDescription( - "The ampersand-separated MongoDB connection options. " - + "eg. replicaSet=test&connectTimeoutMS=300000"); - - public static final Option USERNAME = - Options.key("username") - .stringType() - .noDefaultValue() - .withDescription( - "Name of the database user to be used when connecting to MongoDB. " - + "This is required only when MongoDB is configured to use authentication."); - - public static final Option PASSWORD = - Options.key("password") - .stringType() - .noDefaultValue() - .withDescription( - "Password to be used when connecting to MongoDB. " - + "This is required only when MongoDB is configured to use authentication."); - - public static final Option DATABASE = - Options.key("database") - .stringType() - .noDefaultValue() - .withDescription("The name of MongoDB database to read or write."); - - public static final Option COLLECTION = - Options.key("collection") - .stringType() - .noDefaultValue() - .withDescription("The name of MongoDB collection to read or write."); - - public static OptionRule optionRule() { - return OptionRule.builder() - .required(HOSTS) - .optional(CONNECTION_OPTIONS, USERNAME, PASSWORD) - .build(); - } - - public static OptionRule metadataRule() { - return OptionRule.builder().required(DATABASE, COLLECTION).build(); - } - - public static @Nonnull ConnectionString buildConnectionString( - String username, String password, String hosts, String connectionOptions) { - StringBuilder sb = new StringBuilder("mongodb://"); - - if (hasCredentials(username, password)) { - appendCredentials(sb, username, password); - } - - sb.append(hosts); - - if (StringUtils.isNotEmpty(connectionOptions)) { - sb.append("/?").append(connectionOptions); - } - - return new ConnectionString(sb.toString()); - } - - private static boolean hasCredentials(String username, String password) { - return StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password); - } - - private static void appendCredentials( - @Nonnull StringBuilder sb, String username, String password) { - sb.append(encodeValue(username)).append(":").append(encodeValue(password)).append("@"); - } - - public static String encodeValue(String value) { - try { - return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - throw new SeaTunnelException(e.getMessage()); - } - } -} diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml index a751dd7e2..e3ef533ac 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml +++ b/seatunnel-datasource/seatunnel-datasource-plugins/pom.xml @@ -46,8 +46,6 @@ datasource-jdbc-tidb datasource-jdbc-common datasource-jdbc-doris - datasource-mongodb - datasource-mongodbcdc diff --git a/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh b/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh index 1c09af2f7..8ebea6d59 100644 --- a/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh +++ b/seatunnel-server/seatunnel-app/src/main/bin/download_datasource.sh @@ -43,6 +43,7 @@ datasource_list=( "datasource-s3" "datasource-sqlserver-cdc" "datasource-starrocks" + "datasource-jdbc-doris" ) # the datasource default version is 1.0.0, you can also choose a custom version. eg: 1.1.2: sh install-datasource.sh 2.1.2 diff --git a/seatunnel-web-dist/pom.xml b/seatunnel-web-dist/pom.xml index 7e83b1acb..b20507f37 100644 --- a/seatunnel-web-dist/pom.xml +++ b/seatunnel-web-dist/pom.xml @@ -349,7 +349,18 @@ - + + org.apache.seatunnel + datasource-jdbc-doris + ${project.version} + provided + + + * + * + + + com.oracle.database.jdbc From ba4b547e65d68a776595d99b8c3ef06b4fe21e06 Mon Sep 17 00:00:00 2001 From: jiayang Date: Thu, 26 Oct 2023 20:15:11 +0800 Subject: [PATCH 6/6] [feature] split --- .../datasource/SchemaGenerator.java | 140 ------------------ .../MongoDBCDCDataSourceConfigSwitcher.java | 118 --------------- .../impl/MongoDBDataSourceConfigSwitcher.java | 115 -------------- .../connector-datasource-mapper.yaml | 28 ---- 4 files changed, 401 deletions(-) delete mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java delete mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java delete mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java deleted file mode 100644 index 198ee44d8..000000000 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/SchemaGenerator.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.app.thirdparty.datasource; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; - -import org.apache.seatunnel.app.domain.request.job.DatabaseTableSchemaReq; -import org.apache.seatunnel.app.domain.request.job.SelectTableFields; -import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; -import org.apache.seatunnel.app.domain.response.datasource.VirtualTableFieldRes; -import org.apache.seatunnel.datasource.plugin.api.model.TableField; - -import org.apache.commons.collections4.CollectionUtils; - -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -@Slf4j -public class SchemaGenerator { - - private SchemaGenerator() {} - - /** - * Generate the schema of the table. - * - * 
-     * fields {
-     *        name = "string"
-     *        age = "int"
-     *       }
-     *
- * - * @param virtualTableDetailRes virtual table detail. - * @param selectTableFields select table fields which need to be placed in the schema. - * @return schema. - */ - public static Config generateSchemaBySelectTableFields( - VirtualTableDetailRes virtualTableDetailRes, SelectTableFields selectTableFields) { - checkNotNull(selectTableFields, "selectTableFields cannot be null"); - checkArgument( - CollectionUtils.isNotEmpty(selectTableFields.getTableFields()), - "selectTableFields.tableFields cannot be empty"); - - checkNotNull(virtualTableDetailRes, "virtualTableDetailRes cannot be null"); - checkArgument( - CollectionUtils.isNotEmpty(virtualTableDetailRes.getFields()), - "virtualTableDetailRes.fields cannot be empty"); - - Map fieldTypeMap = - virtualTableDetailRes.getFields().stream() - .collect( - Collectors.toMap( - VirtualTableFieldRes::getFieldName, Function.identity())); - - Config schema = ConfigFactory.empty(); - for (String fieldName : selectTableFields.getTableFields()) { - VirtualTableFieldRes virtualTableFieldRes = - checkNotNull( - fieldTypeMap.get(fieldName), - String.format( - "Cannot find the field: %s from virtual table", fieldName)); - schema = - schema.withValue( - fieldName, - ConfigValueFactory.fromAnyRef(virtualTableFieldRes.getFieldType())); - } - return schema.atKey("fields"); - } - - // todo: how to deal with multiple table has the same field? - /** - * Generate the schema of the table. - * - * 
-     * fields {
-     *        name = "string"
-     *        age = "int"
-     *       }
-     *
- * - * @param outputSchemas schema in the ui. - * @param selectTableFields select table fields which need to be placed in the schema. - * @return schema. - */ - public static Config generateSchemaBySelectTableFields( - List outputSchemas, SelectTableFields selectTableFields) { - checkNotNull(selectTableFields, "selectTableFields cannot be null"); - checkArgument( - CollectionUtils.isNotEmpty(selectTableFields.getTableFields()), - "selectTableFields.tableFields cannot be empty"); - - checkArgument( - CollectionUtils.isNotEmpty(outputSchemas), - "selectTableFields.tableFields cannot be empty"); - Config schema = ConfigFactory.empty(); - - Map tableFieldMap = - outputSchemas.stream() - .flatMap( - databaseTableSchemaReq -> - databaseTableSchemaReq.getFields().stream()) - .collect(Collectors.toMap(TableField::getName, Function.identity())); - for (String fieldName : selectTableFields.getTableFields()) { - TableField tableField = - checkNotNull( - tableFieldMap.get(fieldName), - String.format( - "Cannot find the field: %s from virtual table", fieldName)); - schema = - schema.withValue( - fieldName, - ConfigValueFactory.fromAnyRef(tableField.getOutputDataType())); - } - return schema.atKey("fields"); - } -} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java deleted file mode 100644 index 2417e0119..000000000 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBCDCDataSourceConfigSwitcher.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.app.thirdparty.datasource.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.RequiredOption; -import org.apache.seatunnel.app.domain.request.connector.BusinessMode; -import org.apache.seatunnel.app.domain.request.job.DataSourceOption; -import org.apache.seatunnel.app.domain.request.job.SelectTableFields; -import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; -import org.apache.seatunnel.app.dynamicforms.FormStructure; -import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.SchemaGenerator; -import org.apache.seatunnel.common.constants.PluginType; - -import com.google.auto.service.AutoService; -import lombok.extern.slf4j.Slf4j; - -import java.util.Collections; -import java.util.List; - -@AutoService(DataSourceConfigSwitcher.class) -@Slf4j -public class MongoDBCDCDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - - private static final String DATABASE = "database"; - private static final String COLLECTION = "collection"; - private static final String SCHEMA = "schema"; - - @Override - public String getDataSourceName() { - return "MONGODB-CDC"; - } - - @Override - public FormStructure filterOptionRule( - String connectorName, - OptionRule dataSourceOptionRule, - OptionRule virtualTableOptionRule, - BusinessMode businessMode, - PluginType pluginType, - OptionRule connectorOptionRule, - List addRequiredOptions, - List> addOptionalOptions, - List excludedKeys) { - excludedKeys.add(SCHEMA); - return super.filterOptionRule( - connectorName, - dataSourceOptionRule, - virtualTableOptionRule, - businessMode, - pluginType, - connectorOptionRule, - addRequiredOptions, - addOptionalOptions, - excludedKeys); - } - - @Override - public Config mergeDatasourceConfig( - Config dataSourceInstanceConfig, - VirtualTableDetailRes virtualTableDetail, - DataSourceOption dataSourceOption, - SelectTableFields selectTableFields, - BusinessMode businessMode, - PluginType pluginType, - Config connectorConfig) { - // Use field to generate the schema - - String databaseName = virtualTableDetail.getDatasourceProperties().get(DATABASE); - String collectionName = virtualTableDetail.getDatasourceProperties().get(COLLECTION); - - connectorConfig = - connectorConfig.withValue( - DATABASE, - ConfigValueFactory.fromIterable(Collections.singleton(databaseName))); - connectorConfig = - connectorConfig.withValue( - COLLECTION, - ConfigValueFactory.fromIterable( - Collections.singleton(databaseName + "." + collectionName))); - connectorConfig = - connectorConfig.withValue( - SCHEMA, - SchemaGenerator.generateSchemaBySelectTableFields( - virtualTableDetail, selectTableFields) - .root()); - - return super.mergeDatasourceConfig( - dataSourceInstanceConfig, - virtualTableDetail, - dataSourceOption, - selectTableFields, - businessMode, - pluginType, - connectorConfig); - } -} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java deleted file mode 100644 index 2153c52e7..000000000 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/MongoDBDataSourceConfigSwitcher.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.app.thirdparty.datasource.impl; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.configuration.util.RequiredOption; -import org.apache.seatunnel.app.domain.request.connector.BusinessMode; -import org.apache.seatunnel.app.domain.request.job.DataSourceOption; -import org.apache.seatunnel.app.domain.request.job.SelectTableFields; -import org.apache.seatunnel.app.domain.response.datasource.VirtualTableDetailRes; -import org.apache.seatunnel.app.dynamicforms.FormStructure; -import org.apache.seatunnel.app.thirdparty.datasource.AbstractDataSourceConfigSwitcher; -import org.apache.seatunnel.app.thirdparty.datasource.DataSourceConfigSwitcher; -import org.apache.seatunnel.common.constants.PluginType; - -import com.google.auto.service.AutoService; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -@AutoService(DataSourceConfigSwitcher.class) -@Slf4j -public class MongoDBDataSourceConfigSwitcher extends AbstractDataSourceConfigSwitcher { - private static final String DATABASE = "database"; - private static final String COLLECTION = "collection"; - private static final String SCHEMA = "schema"; - - @Override - public String getDataSourceName() { - return "MONGODB"; - } - - @Override - public FormStructure filterOptionRule( - String connectorName, - OptionRule dataSourceOptionRule, - OptionRule virtualTableOptionRule, - BusinessMode businessMode, - PluginType pluginType, - OptionRule connectorOptionRule, - List addRequiredOptions, - List> addOptionalOptions, - List excludedKeys) { - excludedKeys.add(SCHEMA); - return super.filterOptionRule( - connectorName, - dataSourceOptionRule, - virtualTableOptionRule, - businessMode, - pluginType, - connectorOptionRule, - addRequiredOptions, - addOptionalOptions, - excludedKeys); - } - - @Override - public Config mergeDatasourceConfig( - Config dataSourceInstanceConfig, - VirtualTableDetailRes virtualTableDetail, - DataSourceOption dataSourceOption, - SelectTableFields selectTableFields, - BusinessMode businessMode, - PluginType pluginType, - Config connectorConfig) { - // Use field to generate the schema - connectorConfig = - connectorConfig.withValue( - DATABASE, - ConfigValueFactory.fromAnyRef( - virtualTableDetail.getDatasourceProperties().get(DATABASE))); - connectorConfig = - connectorConfig.withValue( - COLLECTION, - ConfigValueFactory.fromAnyRef( - virtualTableDetail.getDatasourceProperties().get(COLLECTION))); - if (pluginType == PluginType.SOURCE) { - connectorConfig = - connectorConfig.withValue( - SCHEMA, - KafkaKingbaseDataSourceConfigSwitcher.SchemaGenerator - .generateSchemaBySelectTableFields( - virtualTableDetail, selectTableFields) - .root()); - } - - return super.mergeDatasourceConfig( - dataSourceInstanceConfig, - virtualTableDetail, - dataSourceOption, - selectTableFields, - businessMode, - pluginType, - connectorConfig); - } -} diff --git a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml index f7e62212c..296077a5b 100644 --- a/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml +++ b/seatunnel-server/seatunnel-app/src/main/resources/connector-datasource-mapper.yaml @@ -58,13 +58,6 @@ connector-datasource-mapper: dataSources: - Doris - MongoDB: - dataSources: - - MongoDB - - MongoDB-CDC: - dataSources: - - MongoDB-CDC sourceDatasourceFeatures: JDBC-Mysql: @@ -135,21 +128,7 @@ connector-datasource-mapper: - MULTIPLE_TABLE - SPLIT_TABLE - MongoDB: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE - jobMode: - - BATCH - MongoDB-CDC: - businessMode: - - DATA_INTEGRATION - sceneMode: - - SINGLE_TABLE - jobMode: - - STREAMING @@ -231,10 +210,3 @@ connector-datasource-mapper: sceneMode: - SINGLE_TABLE - MongoDB: - businessMode: - - DATA_INTEGRATION - - DATA_REPLICA - sceneMode: - - SINGLE_TABLE - - MULTIPLE_TABLE