From 14ec230e66e2dbebed1cedbe0a5a5de34a2aeda6 Mon Sep 17 00:00:00 2001 From: lsy3993 <110876560+lsy3993@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:35:02 +0800 Subject: [PATCH 01/12] [refactor](fe) Remove bilingual descriptions from variable annotations (#66342) ### What problem does this PR solve? Issue Number: N/A Problem Summary: Session and global variable annotations currently store both Chinese and English descriptions as a two-element array. The Chinese description is unused and the array shape makes callers cumbersome. This change keeps only the English description, updates the VarAttr annotation type, and adjusts the affected unit tests. ### Release note None ### Check List (For Author) - Test: Unit Test - `./run-fe-ut.sh --run org.apache.doris.qe.SessionVariablesTest` - Behavior changed: No - Does this need documentation: No --- .../java/org/apache/doris/common/Config.java | 1935 ++++++++--------- .../org/apache/doris/common/ConfigBase.java | 5 +- .../org/apache/doris/qe/GlobalVariable.java | 102 +- .../java/org/apache/doris/qe/VarAttrDef.java | 7 +- .../org/apache/doris/common/ConfigTest.java | 13 + .../org/apache/doris/qe/SessionVariable.java | 1281 ++++------- .../apache/doris/qe/SessionVariablesTest.java | 15 +- 7 files changed, 1428 insertions(+), 1930 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 765baf5393d1b2..b626cba8cf15a3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -21,12 +21,12 @@ public class Config extends ConfigBase { - @ConfField(description = {"The path of the user-defined configuration file, used to store fe_custom.conf. " - + "Configurations in this file will override those in fe.conf"}) + @ConfField(description = "The path of the user-defined configuration file, used to store fe_custom.conf. " + + "Configurations in this file will override those in fe.conf") public static String custom_config_dir = EnvUtils.getDorisHome() + "/conf"; - @ConfField(description = { - "The maximum file size of fe.log and fe.audit.log. Once this size is exceeded, the log file will be split"}) + @ConfField(description = "The maximum file size of fe.log and fe.audit.log. Once this size is exceeded, the log " + + "file will be split") public static int log_roll_size_mb = 1024; // 1 GB /** @@ -64,732 +64,688 @@ public class Config extends ConfigBase { * default is false. if true, will compress fe.log & fe.warn.log by gzip */ @Deprecated // use env var LOG_DIR instead - @ConfField(description = {"The path of the FE log file, used to store fe.log"}) + @ConfField(description = "The path of the FE log file, used to store fe.log") public static String sys_log_dir = ""; - @ConfField(description = {"The level of FE log"}, options = {"INFO", "WARN", "ERROR", "FATAL"}) + @ConfField(description = "The level of FE log", options = {"INFO", "WARN", "ERROR", "FATAL"}) public static String sys_log_level = "INFO"; - @ConfField(description = { - "The output mode of the FE log. " - + "NORMAL mode is synchronous output with location information; " - + "ASYNC mode is the default mode, asynchronous output with location information; " - + "BRIEF mode is asynchronous output without location information. " - + "Performance improves in the order: NORMAL, ASYNC, BRIEF"}, + @ConfField(description = "The output mode of the FE log. NORMAL mode is synchronous output with location " + + "information; ASYNC mode is the default mode, asynchronous output with location " + + "information; BRIEF mode is asynchronous output without location information. " + + "Performance improves in the order: NORMAL, ASYNC, BRIEF", options = {"NORMAL", "ASYNC", "BRIEF"}) public static String sys_log_mode = "ASYNC"; - @ConfField(description = { - "The maximum number of FE log files that can be retained within the " - + "sys_log_roll_interval (log roll interval). The default value is 10, which means the system " - + "will keep up to 10 log files during each log roll interval."}) + @ConfField(description = "The maximum number of FE log files that can be retained within the " + + "sys_log_roll_interval (log roll interval). The default value is 10, which means the " + + "system will keep up to 10 log files during each log roll interval.") public static int sys_log_roll_num = 10; - @ConfField(description = {"Verbose modules. VERBOSE level logging is implemented by the DEBUG level of log4j. " - + "If set to `org.apache.doris.catalog`, " - + "DEBUG logs of classes under this package will be printed."}) + @ConfField(description = "Verbose modules. VERBOSE level logging is implemented by the DEBUG level of log4j. If " + + "set to `org.apache.doris.catalog`, DEBUG logs of classes under this package will be " + "printed.") public static String[] sys_log_verbose_modules = {}; - @ConfField(description = {"The split cycle of the FE log file"}, options = {"DAY", "HOUR"}) + @ConfField(description = "The split cycle of the FE log file", options = {"DAY", "HOUR"}) public static String sys_log_roll_interval = "DAY"; - @ConfField(description = { - "The maximum retention time of the FE log file. After this time, the log file will be deleted. " - + "Supported formats include: 7d, 10h, 60m, 120s"}) + @ConfField(description = "The maximum retention time of the FE log file. After this time, the log file will be " + + "deleted. Supported formats include: 7d, 10h, 60m, 120s") public static String sys_log_delete_age = "7d"; - @ConfField(description = {"Whether to enable compression for FE log files"}) + @ConfField(description = "Whether to enable compression for FE log files") public static boolean sys_log_enable_compress = false; - @ConfField(description = {"The path of the FE audit log file, used to store fe.audit.log"}) + @ConfField(description = "The path of the FE audit log file, used to store fe.audit.log") public static String audit_log_dir = System.getenv("LOG_DIR"); - @ConfField(description = {"The maximum number of FE audit log files. " - + "After exceeding this number, the oldest log file will be deleted"}) + @ConfField(description = "The maximum number of FE audit log files. After exceeding this number, the oldest log " + + "file will be deleted") public static int audit_log_roll_num = 90; - @ConfField(description = {"The type of FE audit log file"}, + @ConfField(description = "The type of FE audit log file", options = {"slow_query", "query", "load", "stream_load"}) public static String[] audit_log_modules = {"slow_query", "query", "load", "stream_load"}; - @ConfField(mutable = true, description = {"The threshold of slow query, in milliseconds. " - + "If the response time of a query exceeds this threshold, it will be recorded in audit log."}) + @ConfField(mutable = true, description = "The threshold of slow query, in milliseconds. If the response time of a " + + "query exceeds this threshold, it will be recorded in audit log.") public static long qe_slow_log_ms = 5000; - @ConfField(mutable = true, description = {"The threshold of sql_digest generation, in milliseconds. " - + "If the response time of a query exceeds this threshold, " - + "sql_digest will be generated for it."}) + @ConfField(mutable = true, description = "The threshold of sql_digest generation, in milliseconds. If the " + + "response time of a query exceeds this threshold, sql_digest will be " + "generated for it.") public static long sql_digest_generation_threshold_ms = 5000; - @ConfField(description = {"The split cycle of the FE audit log file"}, + @ConfField(description = "The split cycle of the FE audit log file", options = {"DAY", "HOUR"}) public static String audit_log_roll_interval = "DAY"; - @ConfField(description = {"The maximum retention time of the FE audit log file. " - + "After this time, the log file will be deleted. " - + "Supported formats include: 7d, 10h, 60m, 120s"}) + @ConfField(description = "The maximum retention time of the FE audit log file. After this time, the log file will " + + "be deleted. Supported formats include: 7d, 10h, 60m, 120s") public static String audit_log_delete_age = "30d"; - @ConfField(description = {"Whether to enable compression for FE audit log files"}) + @ConfField(description = "Whether to enable compression for FE audit log files") public static boolean audit_log_enable_compress = false; - @ConfField(description = {"Active lineage plugins. Specify the name returned by LineagePlugin.name()"}) + @ConfField(description = "Active lineage plugins. Specify the name returned by LineagePlugin.name()") public static String[] activate_lineage_plugin = {}; - @ConfField(description = {"Whether to use a file to record logs. When starting FE with --console, " - + "all logs will be written to both standard output and file. " - + "Disabling this option will stop writing logs to files."}) + @ConfField(description = "Whether to use a file to record logs. When starting FE with --console, all logs will be " + + "written to both standard output and file. Disabling this option will stop writing logs " + "to files.") public static boolean enable_file_logger = true; @ConfField(mutable = false, masterOnly = false, - description = {"Whether to check for table lock leaks"}) + description = "Whether to check for table lock leaks") public static boolean check_table_lock_leaky = false; - @ConfField(mutable = false, description = {"当前 FE 节点所属的 Resource Group。可通过命令行参数 " - + "`--local_resource_group` 或环境变量 `DORIS_LOCAL_RESOURCE_GROUP` 覆盖。空字符串表示未设置。", - "The Resource Group that the current FE node belongs to. It can be overridden by the " - + "`--local_resource_group` command line option or the " - + "`DORIS_LOCAL_RESOURCE_GROUP` environment variable. An empty string means unset."}) + @ConfField(mutable = false, description = "The Resource Group that the current FE node belongs to. It can be " + + "overridden by the `--local_resource_group` command line option or the " + + "`DORIS_LOCAL_RESOURCE_GROUP` environment variable. An empty string " + "means unset.") public static String local_resource_group = ""; @ConfField(mutable = true, masterOnly = false, - description = {"PreparedStatement stmtId starting position, used for testing only"}) + description = "PreparedStatement stmtId starting position, used for testing only") public static long prepared_stmt_start_id = -1; - @ConfField(description = {"The installation directory of the plugin"}) + @ConfField(description = "The installation directory of the plugin") public static String plugin_dir = EnvUtils.getDorisHome() + "/plugins"; - @ConfField(mutable = true, masterOnly = true, description = {"Whether to enable the plugin"}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable the plugin") public static boolean plugin_enable = true; - @ConfField(description = {"The path to save JDBC drivers. When creating a JDBC Catalog, " - + "if the specified driver file path is not an absolute path, Doris will look for jars in this path"}) + @ConfField(description = "The path to save JDBC drivers. When creating a JDBC Catalog, if the specified driver " + + "file path is not an absolute path, Doris will look for jars in this path") public static String jdbc_drivers_dir = EnvUtils.getDorisHome() + "/plugins/jdbc_drivers"; - @ConfField(description = {"The safe path of the JDBC driver. When creating a JDBC Catalog, " - + "you can configure multiple files or network paths that are allowed to be used, " - + "separated by semicolons. " - + "The default is * to allow all; if set to empty, it also means to allow all. " - + "When set to concrete paths, driver URLs are matched structurally (component-based), " - + "so path traversal and prefix confusion are rejected."}) + @ConfField(description = "The safe path of the JDBC driver. When creating a JDBC Catalog, you can configure " + + "multiple files or network paths that are allowed to be used, separated by semicolons. " + + "The default is * to allow all; if set to empty, it also means to allow all. When set to " + + "concrete paths, driver URLs are matched structurally (component-based), so path " + + "traversal and prefix confusion are rejected.") public static String jdbc_driver_secure_path = "*"; - @ConfField(description = {"Functions that MySQL JDBC Catalog does not support pushing down"}) + @ConfField(description = "Functions that MySQL JDBC Catalog does not support pushing down") public static String[] jdbc_mysql_unsupported_pushdown_functions = {"date_trunc", "money_format", "negative"}; - @ConfField(mutable = true, description = { - "MySQL compatibility variable whitelist. These variables will be silently ignored in SET statements " - + "instead of throwing an error. This is mainly used for compatibility with MySQL client tools " - + "(such as phpMyAdmin, mysqldump). Doris does not need to understand the specific meaning of " - + "these variables, it just needs to accept them without error."}) + @ConfField(mutable = true, description = "MySQL compatibility variable whitelist. These variables will be " + + "silently ignored in SET statements instead of throwing an error. This " + + "is mainly used for compatibility with MySQL client tools (such as " + + "phpMyAdmin, mysqldump). Doris does not need to understand the specific " + + "meaning of these variables, it just needs to accept them without error.") public static String[] mysql_compat_var_whitelist = {}; - @ConfField(description = {"Force SQLServer Jdbc Catalog encrypt to false. " - + "This is a security-sensitive switch (it disables SQLServer JDBC transport encryption), " - + "so it can only be set in fe.conf and is not modifiable at runtime via ADMIN SET FRONTEND CONFIG."}) + @ConfField(description = "Force SQLServer Jdbc Catalog encrypt to false. This is a security-sensitive switch (it " + + "disables SQLServer JDBC transport encryption), so it can only be set in fe.conf and is " + + "not modifiable at runtime via ADMIN SET FRONTEND CONFIG.") public static boolean force_sqlserver_jdbc_encrypt_false = false; - @ConfField(mutable = true, masterOnly = true, description = { - "The default parallelism of the load execution plan on a single node when the broker load is submitted"}) + @ConfField(mutable = true, masterOnly = true, description = "The default parallelism of the load execution plan " + + "on a single node when the broker load is submitted") public static int default_load_parallelism = 8; - @ConfField(mutable = true, masterOnly = true, description = { - "Labels of finished or cancelled load jobs will be removed after this time. " - + "The removed labels can be reused."}) + @ConfField(mutable = true, masterOnly = true, description = "Labels of finished or cancelled load jobs will be " + + "removed after this time. The removed labels can be " + "reused.") public static int label_keep_max_second = 3 * 24 * 3600; // 3 days - @ConfField(mutable = true, masterOnly = true, description = { - "For some high-frequency load jobs such as INSERT, STREAMING LOAD, ROUTINE_LOAD_TASK, and DELETE, " - + "remove the finished job or task if expired. The removed labels can be reused."}) + @ConfField(mutable = true, masterOnly = true, description = "For some high-frequency load jobs such as INSERT, " + + "STREAMING LOAD, ROUTINE_LOAD_TASK, and DELETE, " + "remove the finished job or task if expired. The " + + "removed labels can be reused.") public static int streaming_label_keep_max_second = 43200; // 12 hour - @ConfField(mutable = true, masterOnly = true, description = { - "For ALTER and EXPORT jobs, remove the finished job if expired."}) + @ConfField(mutable = true, masterOnly = true, description = "For ALTER and EXPORT jobs, remove the finished job " + + "if expired.") public static int history_job_keep_max_second = 7 * 24 * 3600; // 7 days - @ConfField(mutable = true, masterOnly = true, description = { - "For EXPORT jobs, if the number of EXPORT jobs in the system exceeds this value, " - + "the oldest records will be deleted."}) + @ConfField(mutable = true, masterOnly = true, description = "For EXPORT jobs, if the number of EXPORT jobs in the " + + "system exceeds this value, the oldest records will " + "be deleted.") public static int max_export_history_job_num = 1000; - @ConfField(description = {"The cleanup interval for transactions, in seconds. " - + "In each cycle, expired historical transactions will be cleaned up"}) + @ConfField(description = "The cleanup interval for transactions, in seconds. In each cycle, expired historical " + + "transactions will be cleaned up") public static int transaction_clean_interval_second = 30; - @ConfField(description = {"The cleanup interval for load jobs, in seconds. " - + "In each cycle, expired historical load jobs will be cleaned up"}) + @ConfField(description = "The cleanup interval for load jobs, in seconds. In each cycle, expired historical load " + + "jobs will be cleaned up") public static int label_clean_interval_second = 1 * 3600; // 1 hours - @ConfField(mutable = true, masterOnly = true, description = {"Time interval for cleaning up discarded temporary " - + "partitions after an Insert Overwrite task fails, in milliseconds"}) + @ConfField(mutable = true, masterOnly = true, description = "Time interval for cleaning up discarded temporary " + + "partitions after an Insert Overwrite task fails, in " + "milliseconds") public static int overwrite_clean_interval_ms = 10000; - @ConfField(description = {"The directory to save Doris meta data"}) + @ConfField(description = "The directory to save Doris meta data") public static String meta_dir = EnvUtils.getDorisHome() + "/doris-meta"; - @ConfField(description = {"The directory to save Doris temp data"}) + @ConfField(description = "The directory to save Doris temp data") public static String tmp_dir = EnvUtils.getDorisHome() + "/temp_dir"; - @ConfField(description = {"The storage type of the metadata log. BDB: Logs are stored in BDBJE. " - + "LOCAL: logs are stored in a local file (for testing only)"}, options = {"BDB", "LOCAL"}) + @ConfField(description = "The storage type of the metadata log. BDB: Logs are stored " + + "in BDBJE. LOCAL: logs are stored in a local file (for " + "testing only)", options = {"BDB", "LOCAL"}) public static String edit_log_type = "bdb"; - @ConfField(description = {"The port of BDBJE"}) + @ConfField(description = "The port of BDBJE") public static int edit_log_port = 9010; - @ConfField(mutable = true, masterOnly = true, description = { - "The log roll size of BDBJE. When the number of log entries exceeds this value, the log will be rolled"}) + @ConfField(mutable = true, masterOnly = true, description = "The log roll size of BDBJE. When the number of log " + + "entries exceeds this value, the log will be rolled") public static int edit_log_roll_num = 50000; - @ConfField(mutable = true, masterOnly = true, description = {"The max number of log entries for batching BDBJE"}) + @ConfField(mutable = true, masterOnly = true, description = "The max number of log entries for batching BDBJE") public static int batch_edit_log_max_item_num = 100; - @ConfField(mutable = true, masterOnly = true, description = {"The max size for batching BDBJE"}) + @ConfField(mutable = true, masterOnly = true, description = "The max size for batching BDBJE") public static long batch_edit_log_max_byte_size = 640 * 1024L; - @ConfField(mutable = true, masterOnly = true, description = { - "The sleep time after writing multiple batching BDBJE entries continuously"}) + @ConfField(mutable = true, masterOnly = true, description = "The sleep time after writing multiple batching BDBJE " + + "entries continuously") public static long batch_edit_log_rest_time_ms = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "After writing multiple batching BDBJE entries continuously, a short rest is needed. " - + "This indicates the write count before a rest"}) + @ConfField(mutable = true, masterOnly = true, description = "After writing multiple batching BDBJE entries " + + "continuously, a short rest is needed. This indicates " + "the write count before a rest") public static long batch_edit_log_continuous_count_for_rest = 1000; - @ConfField(description = {"Batch EditLog writing"}) + @ConfField(description = "Batch EditLog writing") public static boolean enable_batch_editlog = true; - @ConfField(description = {"The tolerated delay time of metadata synchronization, in seconds. " - + "If the metadata delay exceeds this value, non-master FE will stop offering service"}) + @ConfField(description = "The tolerated delay time of metadata synchronization, in seconds. If the metadata delay " + + "exceeds this value, non-master FE will stop offering service") public static int meta_delay_toleration_second = 300; // 5 min - @ConfField(description = {"The sync policy of meta data log. If you only deploy one Follower FE, " - + "set this to `SYNC`. If you deploy more than 3 Follower FE, " - + "you can set this and the following `replica_sync_policy` to `WRITE_NO_SYNC`. " - + "See: http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.SyncPolicy.html"}, + @ConfField(description = "The sync policy of meta data log. If you only deploy one Follower FE, set this to " + + "`SYNC`. If you deploy more than 3 Follower FE, you can set this and the following " + + "`replica_sync_policy` to `WRITE_NO_SYNC`. See: " + + "http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.SyncPolicy.htm" + "l", options = {"SYNC", "NO_SYNC", "WRITE_NO_SYNC"}) public static String master_sync_policy = "SYNC"; // SYNC, NO_SYNC, WRITE_NO_SYNC - @ConfField(description = {"Same as `master_sync_policy`"}, + @ConfField(description = "Same as `master_sync_policy`", options = {"SYNC", "NO_SYNC", "WRITE_NO_SYNC"}) public static String replica_sync_policy = "SYNC"; // SYNC, NO_SYNC, WRITE_NO_SYNC - @ConfField(description = {"The replica ack policy of bdbje. " - + "See: http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.ReplicaAckPolicy.html"}, + @ConfField(description = "The replica ack policy of bdbje. See: " + + "http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.ReplicaAckPoli" + "cy.html", options = {"ALL", "NONE", "SIMPLE_MAJORITY"}) public static String replica_ack_policy = "SIMPLE_MAJORITY"; // ALL, NONE, SIMPLE_MAJORITY - @ConfField(description = {"The heartbeat timeout of BDBJE between master and follower, in seconds. " - + "The default is 30 seconds, which is the same as the default value in BDBJE. " - + "If the network is experiencing transient problems, " - + "or some unexpected long Java GC is bothering you, " - + "you can try to increase this value to decrease the chances of false timeouts"}) + @ConfField(description = "The heartbeat timeout of BDBJE between master and follower, in seconds. The default is " + + "30 seconds, which is the same as the default value in BDBJE. If the network is " + + "experiencing transient problems, or some unexpected long Java GC is bothering you, you " + + "can try to increase this value to decrease the chances of false timeouts") public static int bdbje_heartbeat_timeout_second = 30; - @ConfField(description = {"The lock timeout of bdbje operation, in seconds. " - + "If there are many LockTimeoutException in FE WARN log, you can try to increase this value"}) + @ConfField(description = "The lock timeout of bdbje operation, in seconds. If there are many LockTimeoutException " + + "in FE WARN log, you can try to increase this value") public static int bdbje_lock_timeout_second = 5; - @ConfField(description = {"The replica ack timeout of bdbje between master and follower, in seconds. " - + "If there are many ReplicaWriteException in FE WARN log, you can try to increase this value"}) + @ConfField(description = "The replica ack timeout of bdbje between master and follower, in seconds. If there are " + + "many ReplicaWriteException in FE WARN log, you can try to increase this value") public static int bdbje_replica_ack_timeout_second = 10; - @ConfField(description = {"The desired upper limit on the number of bytes of reserved space to retain " - + "in a replicated JE Environment. " - + "This parameter is ignored in a non-replicated JE Environment."}) + @ConfField(description = "The desired upper limit on the number of bytes of reserved space to retain in a " + + "replicated JE Environment. This parameter is ignored in a non-replicated JE Environment.") public static long bdbje_reserved_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G - @ConfField(description = {"Amount of free disk space required by BDBJE. " - + "If the free disk space is less than this value, BDBJE will not be able to write."}) + @ConfField(description = "Amount of free disk space required by BDBJE. If the free disk space is less than this " + + "value, BDBJE will not be able to write.") public static long bdbje_free_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G - @ConfField(description = {"Amount of memory used by BDBJE as cache."}) + @ConfField(description = "Amount of memory used by BDBJE as cache.") public static long bdbje_cache_size_bytes = 10 * 1024 * 1024; // 10 MB - @ConfField(description = {"Maximum message size of BDBJE."}) + @ConfField(description = "Maximum message size of BDBJE.") public static long bdbje_max_message_size_bytes = Integer.MAX_VALUE; // 2 GB - @ConfField(masterOnly = true, description = {"Number of threads to handle heartbeat events"}) + @ConfField(masterOnly = true, description = "Number of threads to handle heartbeat events") public static int heartbeat_mgr_threads_num = 8; - @ConfField(masterOnly = true, description = {"Queue size to store heartbeat tasks in heartbeat_mgr"}) + @ConfField(masterOnly = true, description = "Queue size to store heartbeat tasks in heartbeat_mgr") public static int heartbeat_mgr_blocking_queue_size = 1024; - @ConfField(masterOnly = true, description = {"Number of threads to update tablet statistics"}) + @ConfField(masterOnly = true, description = "Number of threads to update tablet statistics") public static int tablet_stat_mgr_threads_num = -1; - @ConfField(masterOnly = true, description = { - "Number of threads to handle agent tasks in the agent task thread pool."}) + @ConfField(masterOnly = true, description = "Number of threads to handle agent tasks in the agent task thread " + + "pool.") public static int max_agent_task_threads_num = 4096; - @ConfField(description = { - "The maximum number of transactions that BDBJE can roll back when trying to rejoin the group. " - + "If the number of transactions to roll back is larger than this value, " - + "BDBJE will not be able to rejoin the group, and you need to clean up BDBJE data manually."}) + @ConfField(description = "The maximum number of transactions that BDBJE can roll back when trying to rejoin the " + + "group. If the number of transactions to roll back is larger than this value, BDBJE will " + + "not be able to rejoin the group, and you need to clean up BDBJE data manually.") public static int txn_rollback_limit = 100; - @ConfField(description = {"The preferred network address. If FE has multiple network addresses, " - + "this configuration can be used to specify the preferred network address. " - + "This is a semicolon-separated list, " - + "each element is a CIDR representation of the network address"}) + @ConfField(description = "The preferred network address. If FE has multiple network addresses, this configuration " + + "can be used to specify the preferred network address. This is a semicolon-separated " + + "list, each element is a CIDR representation of the network address") public static String priority_networks = ""; - @ConfField(mutable = true, description = { - "If true, non-master FE will ignore the metadata delay gap between Master FE and itself, " - + "even if the metadata delay gap exceeds the threshold. " - + "Non-master FE will still offer read service. " - + "This is helpful when you need to stop the Master FE for a relatively long time for some reason, " - + "but still want the non-master FE to offer read service."}) + @ConfField(mutable = true, description = "If true, non-master FE will ignore the metadata delay gap between " + + "Master FE and itself, even if the metadata delay gap exceeds the " + + "threshold. Non-master FE will still offer read service. This is helpful " + + "when you need to stop the Master FE for a relatively long time for some " + + "reason, but still want the non-master FE to offer read service.") public static boolean ignore_meta_check = false; - @ConfField(description = {"The maximum clock skew between non-master FE to Master FE host, in milliseconds. " - + "This value is checked whenever a non-master FE establishes a connection to master FE via BDBJE. " - + "The connection is abandoned if the clock skew is larger than this value."}) + @ConfField(description = "The maximum clock skew between non-master FE to Master FE host, in milliseconds. This " + + "value is checked whenever a non-master FE establishes a connection to master FE via " + + "BDBJE. The connection is abandoned if the clock skew is larger than this value.") public static long max_bdbje_clock_delta_ms = 5000; // 5s - @ConfField(mutable = true, description = { - "Whether to enable authentication for all HTTP interfaces"}, varType = VariableAnnotation.EXPERIMENTAL) + @ConfField(mutable = true, description = "Whether to enable " + + "authentication for all HTTP " + "interfaces", varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_all_http_auth = false; - @ConfField(description = {"Whether to enable FE unified TLS configuration. When enabled, protocols not listed in " - + "tls_excluded_protocols will use TLS implementation."}) + @ConfField(description = "Whether to enable FE unified TLS configuration. When enabled, protocols not listed in " + + "tls_excluded_protocols will use TLS implementation.") public static boolean enable_tls = false; - @ConfField(description = {"Verify mode used by FE TLS. Supported values are verify_peer, verify_none and " - + "verify_fail_if_no_peer_cert."}) + @ConfField(description = "Verify mode used by FE TLS. Supported values are verify_peer, verify_none and " + + "verify_fail_if_no_peer_cert.") public static String tls_verify_mode = "verify_peer"; - @ConfField(description = {"Path to the FE TLS server certificate."}) + @ConfField(description = "Path to the FE TLS server certificate.") public static String tls_certificate_path = ""; - @ConfField(description = {"Path to the FE TLS private key."}) + @ConfField(description = "Path to the FE TLS private key.") public static String tls_private_key_path = ""; - @ConfField(description = {"Password for the FE TLS private key."}) + @ConfField(description = "Password for the FE TLS private key.") public static String tls_private_key_password = ""; - @ConfField(description = {"Path to the FE TLS CA certificate."}) + @ConfField(description = "Path to the FE TLS CA certificate.") public static String tls_ca_certificate_path = ""; - @ConfField(description = {"Refresh interval for FE TLS certificate reload, in seconds."}) + @ConfField(description = "Refresh interval for FE TLS certificate reload, in seconds.") public static int tls_cert_refresh_interval_seconds = 3600; - @ConfField(description = {"Comma-separated list of protocols that should not use TLS. Supported values are " - + "thrift,mysql,http,arrowflight."}) + @ConfField(description = "Comma-separated list of protocols that should not use TLS. Supported values are " + + "thrift,mysql,http,arrowflight.") public static String tls_excluded_protocols = ""; - @ConfField(description = {"Peer certificate DNS SAN allowlist for private protocols. " - + "Syntax: protocol=dns1,dns2;... . Currently supported protocols are thrift and brpc."}) + @ConfField(description = "Peer certificate DNS SAN allowlist for private protocols. Syntax: " + + "protocol=dns1,dns2;... . Currently supported protocols are thrift and brpc.") public static String tls_peer_cert_required_san_dns = ""; - @ConfField(mutable = true, description = { - "Whether password verification can be skipped after cert-based auth succeeds."}) + @ConfField(mutable = true, description = "Whether password verification can be skipped after cert-based auth " + + "succeeds.") public static boolean tls_cert_based_auth_ignore_password = false; - @ConfField(description = {"FE HTTP port. Currently, all FEs' HTTP port must be the same"}) + @ConfField(description = "FE HTTP port. Currently, all FEs' HTTP port must be the same") public static int http_port = 8030; - @ConfField(description = {"FE HTTPS port. Currently, all FEs' HTTPS port must be the same"}) + @ConfField(description = "FE HTTPS port. Currently, all FEs' HTTPS port must be the same") public static int https_port = 8050; - @ConfField(description = {"The key store path of FE https service"}) + @ConfField(description = "The key store path of FE https service") public static String key_store_path = EnvUtils.getDorisHome() + "/conf/ssl/doris_ssl_certificate.keystore"; - @ConfField(description = {"The key store password of FE https service"}) + @ConfField(description = "The key store password of FE https service") public static String key_store_password = ""; - @ConfField(description = {"The key store type of FE https service"}) + @ConfField(description = "The key store type of FE https service") public static String key_store_type = "JKS"; - @ConfField(description = {"The key store alias of FE https service"}) + @ConfField(description = "The key store alias of FE https service") public static String key_store_alias = "doris_ssl_certificate"; - @ConfField(description = {"Whether to enable https, if enabled, http port will not be available"}, + @ConfField(description = "Whether to enable https, if enabled, http port will not be available", varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_https = false; - @ConfField(description = { - "The number of acceptor threads for Jetty. Jetty's thread architecture model is very simple, " - + "divided into three thread pools: acceptor, selector and worker. " - + "The acceptor is responsible for accepting new connections, " - + "and then handing it over to the selector to process the unpacking of the HTTP message protocol, " - + "and finally the worker processes the request. " - + "The first two thread pools adopt a non-blocking model, " - + "and one thread can handle many socket reads and writes, " - + "so the number of thread pools is small. For most projects, " - + "only 1-2 acceptor threads are needed, 2 to 4 should be enough. " - + "The number of workers depends on the ratio of QPS and IO events of the application. " - + "The higher the QPS, or the higher the IO ratio, the more threads are waiting, " - + "and the more threads are required."}) + @ConfField(description = "The number of acceptor threads for Jetty. Jetty's thread architecture model is very " + + "simple, divided into three thread pools: acceptor, selector and worker. The acceptor is " + + "responsible for accepting new connections, and then handing it over to the selector to " + + "process the unpacking of the HTTP message protocol, and finally the worker processes " + + "the request. The first two thread pools adopt a non-blocking model, and one thread can " + + "handle many socket reads and writes, so the number of thread pools is small. For most " + + "projects, only 1-2 acceptor threads are needed, 2 to 4 should be enough. The number of " + + "workers depends on the ratio of QPS and IO events of the application. The higher the " + + "QPS, or the higher the IO ratio, the more threads are waiting, and the more threads are " + "required.") public static int jetty_server_acceptors = 2; - @ConfField(description = {"The number of selector threads for Jetty."}) + @ConfField(description = "The number of selector threads for Jetty.") public static int jetty_server_selectors = 4; - @ConfField(description = {"The number of worker threads for Jetty. 0 means using the default thread pool."}) + @ConfField(description = "The number of worker threads for Jetty. 0 means using the default thread pool.") public static int jetty_server_workers = 0; - @ConfField(description = {"The default minimum number of threads for jetty."}) + @ConfField(description = "The default minimum number of threads for jetty.") public static int jetty_threadPool_minThreads = 20; - @ConfField(description = {"The default maximum number of threads for jetty."}) + @ConfField(description = "The default maximum number of threads for jetty.") public static int jetty_threadPool_maxThreads = 400; - @ConfField(description = {"The maximum HTTP POST size of Jetty, in bytes, the default value is 100MB."}) + @ConfField(description = "The maximum HTTP POST size of Jetty, in bytes, the default value is 100MB.") public static int jetty_server_max_http_post_size = 100 * 1024 * 1024; - @ConfField(description = { - "Jetty 在应用未消费完请求体时，额外尝试读取剩余内容的最大次数。" - + "-1 表示不限制，0 表示不额外读取，正数表示最大读取次数。", - "The maximum number of extra reads Jetty performs for unconsumed request content. " - + "-1 means unlimited, 0 means disabled, and a positive value limits the read attempts."}) + @ConfField(description = "The maximum number of extra reads Jetty performs for unconsumed request content. -1 " + + "means unlimited, 0 means disabled, and a positive value limits the read attempts.") public static int jetty_server_max_unconsumed_request_content_reads = -1; - @ConfField(description = {"The maximum HTTP header size of Jetty, in bytes, the default value is 1MB."}) + @ConfField(description = "The maximum HTTP header size of Jetty, in bytes, the default value is 1MB.") public static int jetty_server_max_http_header_size = 1048576; - @ConfField(description = {"Whether to disable mini load, disabled by default"}) + @ConfField(description = "Whether to disable mini load, disabled by default") public static boolean disable_mini_load = true; - @ConfField(description = {"The backlog number of the MySQL NIO server. " - + "If you increase this value, you should also increase the value in " - + "`/proc/sys/net/core/somaxconn` at the same time"}) + @ConfField(description = "The backlog number of the MySQL NIO server. If you increase this value, you should also " + + "increase the value in `/proc/sys/net/core/somaxconn` at the same time") public static int mysql_nio_backlog_num = 1024; - @ConfField(description = {"Whether to enable TCP Keep-Alive for MySQL connections, disabled by default"}) + @ConfField(description = "Whether to enable TCP Keep-Alive for MySQL connections, disabled by default") public static boolean mysql_nio_enable_keep_alive = false; - @ConfField(description = {"The connection timeout of thrift client, in milliseconds. 0 means no timeout."}) + @ConfField(description = "The connection timeout of thrift client, in milliseconds. 0 means no timeout.") public static int thrift_client_timeout_ms = 0; @ConfField(mutable = true, masterOnly = false, - description = {"Thrift RPC 连接阶段的超时时间（毫秒），包括 TCP connect 和可能的 TLS 握手。" - + "用于防止 reopen() 时因网络异常长时间阻塞。0 表示不设置。", - "Timeout in milliseconds for the connect phase of Thrift RPC connections, " - + "including TCP connect and potential TLS handshake. " - + "Prevents long blocking during reopen() when network is unreachable. " - + "0 means no timeout."}) + description = "Timeout in milliseconds for the connect phase of Thrift RPC connections, including TCP " + + "connect and potential TLS handshake. Prevents long blocking during reopen() when network " + + "is unreachable. 0 means no timeout.") public static int thrift_rpc_connect_timeout_ms = 10000; // The default value is inherited from org.apache.thrift.TConfiguration - @ConfField(description = {"The maximum size of a received message of the Thrift server, in bytes"}) + @ConfField(description = "The maximum size of a received message of the Thrift server, in bytes") public static int thrift_max_message_size = 100 * 1024 * 1024; // The default value is inherited from org.apache.thrift.TConfiguration - @ConfField(description = {"The size limit of one frame for the Thrift server transport"}) + @ConfField(description = "The size limit of one frame for the Thrift server transport") public static int thrift_max_frame_size = 16384000; - @ConfField(description = {"The backlog number of the Thrift server. " - + "If you increase this value, you should also increase the value in " - + "`/proc/sys/net/core/somaxconn` at the same time"}) + @ConfField(description = "The backlog number of the Thrift server. If you increase this value, you should also " + + "increase the value in `/proc/sys/net/core/somaxconn` at the same time") public static int thrift_backlog_num = 1024; - @ConfField(description = {"The port of FE thrift server"}) + @ConfField(description = "The port of FE thrift server") public static int rpc_port = 9020; - @ConfField(description = {"The port of FE MySQL server"}) + @ConfField(description = "The port of FE MySQL server") public static int query_port = 9030; - @ConfField(description = {"The port of FE Arrow-Flight-SQL server"}) + @ConfField(description = "The port of FE Arrow-Flight-SQL server") public static int arrow_flight_sql_port = 8070; - @ConfField(description = {"The number of IO threads in the MySQL service"}) + @ConfField(description = "The number of IO threads in the MySQL service") public static int mysql_service_io_threads_num = 4; - @ConfField(description = {"The maximum number of task threads in the MySQL service"}) + @ConfField(description = "The maximum number of task threads in the MySQL service") public static int max_mysql_service_task_threads_num = 4096; - @ConfField(description = {"BackendServiceProxy pool size for pooling GRPC channels."}) + @ConfField(description = "BackendServiceProxy pool size for pooling GRPC channels.") public static int backend_proxy_num = 48; - @ConfField(description = { - "Cluster ID used for internal authentication. Usually a random integer generated when the master FE " - + "starts for the first time. You can also specify one."}) + @ConfField(description = "Cluster ID used for internal authentication. Usually a random integer generated when " + + "the master FE starts for the first time. You can also specify one.") public static int cluster_id = -1; - @ConfField(sensitive = true, description = {"Cluster token used for internal authentication."}) + @ConfField(sensitive = true, description = "Cluster token used for internal authentication.") public static String auth_token = ""; @ConfField(mutable = true, masterOnly = true, - description = {"Maximal waiting time for creating a single replica, in seconds. " - + "eg. if you create a table with #m tablets and #n replicas for each tablet, " - + "the create table request will run at most " - + "(m * n * tablet_create_timeout_second) before timeout"}) + description = "Maximal waiting time for creating a single replica, in seconds. eg. if you create a table " + + "with #m tablets and #n replicas for each tablet, the create table request will run at most " + + "(m * n * tablet_create_timeout_second) before timeout") public static int tablet_create_timeout_second = 2; - @ConfField(mutable = true, masterOnly = true, description = { - "Minimal waiting time for creating a table, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Minimal waiting time for creating a table, in " + + "seconds.") public static int min_create_table_timeout_second = 30; - @ConfField(mutable = true, masterOnly = true, description = { - "Maximal waiting time for creating a table, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximal waiting time for creating a table, in " + + "seconds.") public static int max_create_table_timeout_second = 3600; - @ConfField(mutable = true, masterOnly = true, description = { - "Maximal waiting time for all publish version tasks of one transaction to be finished, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximal waiting time for all publish version tasks " + + "of one transaction to be finished, in seconds.") public static int publish_version_timeout_second = 30; // 30 seconds - @ConfField(mutable = true, masterOnly = true, description = { - "Waiting time for a transaction to reach \"at least one replica success\", in seconds. " - + "If time exceeds this and each tablet has at least one replica published successfully, " - + "then the load task will be considered successful."}) + @ConfField(mutable = true, masterOnly = true, description = "Waiting time for a transaction to reach \"at least " + + "one replica success\", in seconds. If time exceeds " + "this and each tablet has at least one replica " + + "published successfully, then the load task will be " + "considered successful.") public static int publish_wait_time_second = 300; - @ConfField(mutable = true, masterOnly = true, description = { - "Check the replicas that are undergoing schema change when publishing a transaction. " - + "Do not turn off this check " - + "under normal circumstances. It only temporarily skips the check if " - + "publish version and schema change encounter a deadlock"}) + @ConfField(mutable = true, masterOnly = true, description = "Check the replicas that are undergoing schema change " + + "when publishing a transaction. Do not turn off this " + "check under normal circumstances. It only " + + "temporarily skips the check if publish version and " + "schema change encounter a deadlock") public static boolean publish_version_check_alter_replica = true; - @ConfField(mutable = true, masterOnly = true, description = { - "Log printing interval for failed publish transactions, in seconds"}) + @ConfField(mutable = true, masterOnly = true, description = "Log printing interval for failed publish " + + "transactions, in seconds") public static long publish_fail_log_interval_second = 5 * 60; - @ConfField(mutable = true, masterOnly = true, description = { - "The upper limit of failure logs for PUBLISH_VERSION tasks"}) + @ConfField(mutable = true, masterOnly = true, description = "The upper limit of failure logs for PUBLISH_VERSION " + + "tasks") public static long publish_version_task_failed_log_threshold = 80; - @ConfField(masterOnly = true, description = {"Number of threads to handle publish tasks"}) + @ConfField(masterOnly = true, description = "Number of threads to handle publish tasks") public static int publish_thread_pool_num = 128; - @ConfField(masterOnly = true, description = {"Queue size to store publish tasks in the publish thread pool"}) + @ConfField(masterOnly = true, description = "Queue size to store publish tasks in the publish thread pool") public static int publish_queue_size = 128; - @ConfField(mutable = true, description = {"Whether to enable parallel publish version"}) + @ConfField(mutable = true, description = "Whether to enable parallel publish version") public static boolean enable_parallel_publish_version = true; - @ConfField(masterOnly = true, description = {"Number of threads to handle tablet report tasks"}) + @ConfField(masterOnly = true, description = "Number of threads to handle tablet report tasks") public static int tablet_report_thread_pool_num = 10; - @ConfField(masterOnly = true, description = { - "Queue size to store tablet report tasks in the tablet report thread pool."}) + @ConfField(masterOnly = true, description = "Queue size to store tablet report tasks in the tablet report thread " + + "pool.") public static int tablet_report_queue_size = 1024; - @ConfField(mutable = true, masterOnly = true, description = { - "Maximal waiting time for all data inserted before one transaction to be committed, in seconds. " - + "This parameter is only used for transactional insert operation"}) + @ConfField(mutable = true, masterOnly = true, description = "Maximal waiting time for all data inserted before " + + "one transaction to be committed, in seconds. This " + "parameter is only used for transactional insert " + + "operation") public static int commit_timeout_second = 30; // 30 seconds - @ConfField(masterOnly = true, description = {"The interval of the publish task trigger thread, in milliseconds"}) + @ConfField(masterOnly = true, description = "The interval of the publish task trigger thread, in milliseconds") public static int publish_version_interval_ms = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "If the number of publishing transactions of a table exceeds this value, new transactions will " - + "be rejected. Set to -1 to disable this limit."}) + @ConfField(mutable = true, masterOnly = true, description = "If the number of publishing transactions of a table " + + "exceeds this value, new transactions will be " + "rejected. Set to -1 to disable this limit.") public static long max_publishing_txn_num_per_table = 500; - @ConfField(description = {"The maximum number of worker threads of the Thrift server"}) + @ConfField(description = "The maximum number of worker threads of the Thrift server") public static int thrift_server_max_worker_threads = 4096; - @ConfField(mutable = true, masterOnly = true, description = {"Maximal timeout for delete job, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximal timeout for delete job, in seconds.") public static int delete_job_max_timeout_second = 300; - @ConfField(mutable = true, masterOnly = true, description = { - "Minimum number of successfully written replicas for a load job."}) + @ConfField(mutable = true, masterOnly = true, description = "Minimum number of successfully written replicas for " + + "a load job.") public static short min_load_replica_num = -1; - @ConfField(description = {"The interval of the load job scheduler, in seconds."}) + @ConfField(description = "The interval of the load job scheduler, in seconds.") public static int load_checker_interval_second = 5; - @ConfField(description = {"The interval of the ingestion load job scheduler, in seconds."}) + @ConfField(description = "The interval of the ingestion load job scheduler, in seconds.") public static int ingestion_load_checker_interval_second = 60; - @ConfField(mutable = true, masterOnly = true, description = {"Default timeout for broker load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Default timeout for broker load jobs, in seconds.") public static int broker_load_default_timeout_second = 14400; // 4 hour - @ConfField(description = {"The timeout of RPC between FE and Broker, in milliseconds"}) + @ConfField(description = "The timeout of RPC between FE and Broker, in milliseconds") public static int broker_timeout_ms = 10000; // 10s - @ConfField(description = {"The timeout of RPC for high-concurrency short-circuit queries"}) + @ConfField(description = "The timeout of RPC for high-concurrency short-circuit queries") public static int point_query_timeout_ms = 10000; // 10s - @ConfField(mutable = true, masterOnly = true, description = {"Default timeout for insert load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Default timeout for insert load jobs, in seconds.") public static int insert_load_default_timeout_second = 14400; // 4 hour - @ConfField(mutable = true, masterOnly = true, description = { - "Randomly set ORDER BY keys for MOW tables for testing."}) + @ConfField(mutable = true, masterOnly = true, description = "Randomly set ORDER BY keys for MOW tables for " + + "testing.") public static boolean random_add_order_by_keys_for_mow = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Randomly use V3 storage_format (ext_meta) for some tables in fuzzy tests to increase coverage"}) + @ConfField(mutable = true, masterOnly = true, description = "Randomly use V3 storage_format (ext_meta) for some " + + "tables in fuzzy tests to increase coverage") public static boolean random_use_v3_storage_format = true; - @ConfField(mutable = true, masterOnly = true, description = { - "The stale threshold of checkpoint image file in cloud mode (in seconds). " - + "If the image file is older than this threshold, a new checkpoint will be triggered " - + "even if there are no new journals. This helps keep table version, partition version, " - + "and tablet stats in the image up-to-date. If the value is less than or equal to 0, " - + "this feature is disabled."}) + @ConfField(mutable = true, masterOnly = true, description = "The stale threshold of checkpoint image file in " + + "cloud mode (in seconds). If the image file is older " + "than this threshold, a new checkpoint will be " + + "triggered even if there are no new journals. This " + "helps keep table version, partition version, and " + + "tablet stats in the image up-to-date. If the value " + + "is less than or equal to 0, this feature is disabled.") public static long cloud_checkpoint_image_stale_threshold_seconds = 3600; - @ConfField(mutable = true, masterOnly = true, description = { - "Wait for the internal batch to be written before returning; " - + "insert into and stream load use group commit by default."}) + @ConfField(mutable = true, masterOnly = true, description = "Wait for the internal batch to be written before " + + "returning; insert into and stream load use group " + "commit by default.") public static boolean wait_internal_group_commit_finish = false; - @ConfField(mutable = false, masterOnly = true, description = {"Default commit interval in ms for group commit"}) + @ConfField(mutable = false, masterOnly = true, description = "Default commit interval in ms for group commit") public static int group_commit_interval_ms_default_value = 10000; - @ConfField(mutable = false, masterOnly = true, description = {"Default commit data bytes for group commit"}) + @ConfField(mutable = false, masterOnly = true, description = "Default commit data bytes for group commit") public static int group_commit_data_bytes_default_value = 134217728; - @ConfField(mutable = true, masterOnly = true, description = { - "The internal group commit timeout is a multiple of the table's group_commit_interval_ms"}) + @ConfField(mutable = true, masterOnly = true, description = "The internal group commit timeout is a multiple of " + + "the table's group_commit_interval_ms") public static int group_commit_timeout_multipler = 10; - @ConfField(mutable = true, masterOnly = true, description = {"Default timeout for stream load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Default timeout for stream load jobs, in seconds.") public static int stream_load_default_timeout_second = 86400 * 3; // 3days - @ConfField(mutable = true, masterOnly = true, description = { - "Default pre-commit timeout for stream load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Default pre-commit timeout for stream load jobs, in " + + "seconds.") public static int stream_load_default_precommit_timeout_second = 3600; // 3600s - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to enable memtable on sink node by default in stream load"}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable memtable on sink node by default " + + "in stream load") public static boolean stream_load_default_memtable_on_sink_node = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to enable forwarding group commit stream load to follower nodes." - + " If true, stream load with group commit mode will be forwarded to a follower FE round robin."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable forwarding group commit stream " + + "load to follower nodes. If true, stream load with " + + "group commit mode will be forwarded to a follower FE " + "round robin.") public static boolean enable_forward_group_commit_stream_load_to_follower = false; - @ConfField(mutable = true, masterOnly = true, description = {"Maximum timeout for load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum timeout for load jobs, in seconds.") public static int max_load_timeout_second = 259200; // 3days - @ConfField(mutable = true, masterOnly = true, description = {"Maximum timeout for stream load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum timeout for stream load jobs, in seconds.") public static int max_stream_load_timeout_second = 259200; // 3days - @ConfField(mutable = true, masterOnly = true, description = {"Minimum timeout for load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Minimum timeout for load jobs, in seconds.") public static int min_load_timeout_second = 1; // 1s - @ConfField(mutable = true, masterOnly = true, description = { - "Default timeout for ingestion load jobs, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Default timeout for ingestion load jobs, in seconds.") public static int ingestion_load_default_timeout_second = 86400; // 1 day - @ConfField(mutable = true, masterOnly = true, description = { - "Maximum number of waiting jobs for Broker Load. This is a desired number. " - + "In some situations, such as switching the master, " - + "the current number may exceed this value."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum number of waiting jobs for Broker Load. This " + + "is a desired number. In some situations, such as " + + "switching the master, the current number may exceed " + "this value.") public static int desired_max_waiting_jobs = 100; - @ConfField(mutable = true, masterOnly = true, description = { - "The interval at which FE fetches stream load records from BE."}) + @ConfField(mutable = true, masterOnly = true, description = "The interval at which FE fetches stream load records " + + "from BE.") public static int fetch_stream_load_record_interval_second = 120; - @ConfField(mutable = true, masterOnly = true, description = { - "Default maximum number of recent stream load records that can be stored in memory."}) + @ConfField(mutable = true, masterOnly = true, description = "Default maximum number of recent stream load records " + + "that can be stored in memory.") public static int max_stream_load_record_size = 5000; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to disable show stream load and clear stream load records in memory."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to disable show stream load and clear stream " + + "load records in memory.") public static boolean disable_show_stream_load = false; - @ConfField(mutable = true, description = {"Whether to enable stream load profile"}) + @ConfField(mutable = true, description = "Whether to enable stream load profile") public static boolean enable_stream_load_profile = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to enable writing to a single replica for stream load and broker load."}, + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable writing to a single replica for " + + "stream load and broker load.", varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_single_replica_load = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Shuffle will not be enabled for DUPLICATE KEY tables if their tablet count is lower than this number"}, + @ConfField(mutable = true, masterOnly = true, description = "Shuffle will not be enabled for DUPLICATE KEY tables " + + "if their tablet count is lower than this number", varType = VariableAnnotation.EXPERIMENTAL) public static int min_tablets_for_dup_table_shuffle = 64; - @ConfField(mutable = true, masterOnly = true, description = { - "Maximum number of concurrently running transactions, including prepare and commit transactions, " - + "under a single database.", - "The transaction manager will reject incoming transactions once this limit is reached."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum number of concurrently running transactions, " + + "including prepare and commit transactions, under a " + + "single database. The transaction manager will reject " + + "incoming transactions once this limit is reached.") public static int max_running_txn_num_per_db = 10000; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to move transaction edit log writes outside the write lock to reduce lock contention. " - + "When enabled, edit log entries are enqueued inside the write lock (FIFO preserves ordering) " - + "and awaited outside the lock, reducing write lock hold time " - + "and improving concurrent transaction throughput. " - + "Default is true. Set to false to use the traditional in-lock synchronous write mode."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to move transaction edit log writes outside " + + "the write lock to reduce lock contention. When " + "enabled, edit log entries are enqueued inside the " + + "write lock (FIFO preserves ordering) and awaited " + + "outside the lock, reducing write lock hold time and " + + "improving concurrent transaction throughput. Default " + + "is true. Set to false to use the traditional in-lock " + "synchronous write mode.") public static boolean enable_txn_log_outside_lock = true; - @ConfField(mutable = true, description = { - "Whether to enable per-transaction parallel publish. When enabled, different transactions " - + "in the same database can finish publishing in parallel across executor threads, " - + "instead of being serialized per database. " - + "When disabled, falls back to per-database routing (old behavior) " - + "where transactions within a DB are published sequentially."}) + @ConfField(mutable = true, description = "Whether to enable per-transaction parallel publish. When enabled, " + + "different transactions in the same database can finish publishing in " + + "parallel across executor threads, instead of being serialized per " + + "database. When disabled, falls back to per-database routing (old " + + "behavior) where transactions within a DB are published sequentially.") public static boolean enable_per_txn_publish = true; - @ConfField(masterOnly = true, description = {"The pending load task executor pool size. " - + "This pool size limits the maximum number of running pending load tasks.", - "Currently, it only limits the pending load tasks of broker load and ingestion load.", - "It should be less than `max_running_txn_num_per_db`"}) + @ConfField(masterOnly = true, description = "The pending load task executor pool size. This pool size limits the " + + "maximum number of running pending load tasks. Currently, it only " + + "limits the pending load tasks of broker load and ingestion load. It " + + "should be less than `max_running_txn_num_per_db`") public static int async_pending_load_task_pool_size = 10; - @ConfField(masterOnly = true, description = {"The loading load task executor pool size. " - + "This pool size limits the maximum number of running loading load tasks.", - "Currently, it only limits the loading load tasks of broker load."}) + @ConfField(masterOnly = true, description = "The loading load task executor pool size. This pool size limits the " + + "maximum number of running loading load tasks. Currently, it only " + + "limits the loading load tasks of broker load.") public static int async_loading_load_task_pool_size = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "The same meaning as `tablet_create_timeout_second`, but used when deleting a tablet."}) + @ConfField(mutable = true, masterOnly = true, description = "The same meaning as `tablet_create_timeout_second`, " + + "but used when deleting a tablet.") public static int tablet_delete_timeout_second = 2; - @ConfField(mutable = true, masterOnly = true, description = { - "The high watermark of disk capacity usage percent. " - + "This is used for calculating the load score of a backend."}) + @ConfField(mutable = true, masterOnly = true, description = "The high watermark of disk capacity usage percent. " + + "This is used for calculating the load score of a " + "backend.") public static double capacity_used_percent_high_water = 0.75; - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum difference in disk capacity usage percent between BEs. " - + "It is used for calculating the load score of a backend."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum difference in disk capacity usage " + + "percent between BEs. It is used for calculating the " + "load score of a backend.") public static double used_capacity_percent_max_diff = 0.30; - @ConfField(mutable = true, masterOnly = true, description = { - "Sets a fixed disk usage factor in the BE load fraction. The BE load score is a combination of disk usage " - + "and replica count. The valid value range is [0, 1]. When it is out of this range, other " - + "methods are used to automatically calculate this coefficient."}) + @ConfField(mutable = true, masterOnly = true, description = "Sets a fixed disk usage factor in the BE load " + + "fraction. The BE load score is a combination of disk " + + "usage and replica count. The valid value range is " + + "[0, 1]. When it is out of this range, other methods " + + "are used to automatically calculate this coefficient.") public static double backend_load_capacity_coeficient = -1.0; - @ConfField(mutable = true, masterOnly = true, description = { - "Maximum timeout for ALTER TABLE requests. Set this long enough to accommodate your table data size."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum timeout for ALTER TABLE requests. Set this " + + "long enough to accommodate your table data size.") public static int alter_table_timeout_second = 86400 * 30; // 1month - @ConfField(mutable = true, masterOnly = true, description = { - "When disable_storage_medium_check is true, ReportHandler will not check the tablet's storage medium " - + "and will disable the storage cooldown function."}) + @ConfField(mutable = true, masterOnly = true, description = "When disable_storage_medium_check is true, " + + "ReportHandler will not check the tablet's storage " + "medium and will disable the storage cooldown " + + "function.") public static boolean disable_storage_medium_check = false; - @ConfField(description = {"When creating a table (or partition), you can specify its storage medium (HDD or SSD)."}) + @ConfField(description = "When creating a table (or partition), you can specify its storage medium (HDD or SSD).") public static String default_storage_medium = "HDD"; - @ConfField(mutable = true, masterOnly = true, description = { - "After dropping a database (table/partition), you can recover it by using the RECOVER statement.", - "This specifies the maximum data retention time. After this time, the data will be deleted permanently."}) + @ConfField(mutable = true, masterOnly = true, description = "After dropping a database (table/partition), you can " + + "recover it by using the RECOVER statement. This " + "specifies the maximum data retention time. After " + + "this time, the data will be deleted permanently.") public static long catalog_trash_expire_second = 86400L; // 1day @ConfField public static boolean catalog_trash_ignore_min_erase_latency = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Minimum bytes that a single broker scanner will read. When splitting files in broker load, " - + "if the size of a split file is less than this value, it will not be split."}) + @ConfField(mutable = true, masterOnly = true, description = "Minimum bytes that a single broker scanner will " + + "read. When splitting files in broker load, if the " + "size of a split file is less than this value, it " + + "will not be split.") public static long min_bytes_per_broker_scanner = 67108864L; // 64MB - @ConfField(mutable = true, masterOnly = true, description = {"Maximal concurrency of broker scanners."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximal concurrency of broker scanners.") public static int max_broker_concurrency = 100; // TODO(cmy): Disable by default because current checksum logic has some bugs. - @ConfField(mutable = true, masterOnly = true, description = { - "Start time of consistency check. Used with `consistency_check_end_time` " - + "to decide the start and end time of consistency check. " - + "If set to the same value, consistency check will not be scheduled."}) + @ConfField(mutable = true, masterOnly = true, description = "Start time of consistency check. Used with " + + "`consistency_check_end_time` to decide the start and " + + "end time of consistency check. If set to the same " + "value, consistency check will not be scheduled.") public static String consistency_check_start_time = "23"; - @ConfField(mutable = true, masterOnly = true, description = { - "End time of consistency check. Used with `consistency_check_start_time` " - + "to decide the start and end time of consistency check. " - + "If set to the same value, consistency check will not be scheduled."}) + @ConfField(mutable = true, masterOnly = true, description = "End time of consistency check. Used with " + + "`consistency_check_start_time` to decide the start " + + "and end time of consistency check. If set to the " + + "same value, consistency check will not be scheduled.") public static String consistency_check_end_time = "23"; - @ConfField(mutable = true, masterOnly = true, description = { - "Default timeout of a single consistency check task. Set long enough to fit your tablet size."}) + @ConfField(mutable = true, masterOnly = true, description = "Default timeout of a single consistency check task. " + + "Set long enough to fit your tablet size.") public static long check_consistency_default_timeout_second = 600; // 10 min - @ConfField(description = {"Maximum number of MySQL server connections per FE."}) + @ConfField(description = "Maximum number of MySQL server connections per FE.") public static int qe_max_connection = 1024; - @ConfField(mutable = true, description = {"Colocate join PlanFragment instance memory limit penalty factor.", - "The memory_limit for colocate join PlanFragment instance = " - + "`exec_mem_limit / min (query_colocate_join_memory_limit_penalty_factor, instance_num)`"}) + @ConfField(mutable = true, description = "Colocate join PlanFragment instance memory limit penalty factor. The " + + "memory_limit for colocate join PlanFragment instance = `exec_mem_limit " + + "/ min (query_colocate_join_memory_limit_penalty_factor, instance_num)`") public static int query_colocate_join_memory_limit_penalty_factor = 1; /** @@ -803,8 +759,7 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true, masterOnly = true) public static boolean disable_colocate_balance = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to allow colocate balance between all groups."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to allow colocate balance between all groups.") public static boolean disable_colocate_balance_between_groups = false; @ConfField public static boolean proxy_auth_enable = false; @@ -829,16 +784,14 @@ public class Config extends ConfigBase { // check token when download image file. @ConfField public static boolean enable_token_check = true; - @ConfField(sensitive = true, description = {"Cluster token for FE meta-service internal HTTP authentication. " - + "When set (non-empty), FE meta-service endpoints (such as image/role/check/put/journal_id) " - + "additionally require the caller to present a matching token header, on top of the existing " - + "node-host check. Empty (default) keeps the legacy behavior of node-host check only, so " - + "existing clusters and rolling upgrades are unaffected. Must be identical on all FEs and " - + "provisioned in fe.conf before enabling, otherwise FEs will reject each other.", - "FE meta-service 内部 HTTP 鉴权使用的集群 token。设置(非空)后,meta-service 端点(如 " - + "image/role/check/put/journal_id)在原有 node-host 校验之上,额外要求调用方携带匹配的 token 头。" - + "为空(默认)时维持仅 node-host 校验的旧行为,存量集群与滚动升级不受影响。必须在所有 FE 上取值一致," - + "并在启用前写入 fe.conf,否则 FE 之间会互相拒绝。"}) + @ConfField(sensitive = true, description = "Cluster token for FE meta-service internal HTTP authentication. When " + + "set (non-empty), FE meta-service endpoints (such as " + + "image/role/check/put/journal_id) additionally require the caller to " + + "present a matching token header, on top of the existing node-host " + + "check. Empty (default) keeps the legacy behavior of node-host check " + + "only, so existing clusters and rolling upgrades are unaffected. Must " + + "be identical on all FEs and provisioned in fe.conf before enabling, " + + "otherwise FEs will reject each other.") public static String fe_meta_auth_token = ""; /** @@ -948,16 +901,11 @@ public class Config extends ConfigBase { * * This can greatly reduce FE outbound network throughput when cache hit rate is high. */ - @ConfField(mutable = true, description = { - "是否启用 point query 轻量请求。开启后，FE 在 PreparedStatement 执行阶段会优先省略" - + " desc_tbl/output_expr/query_options，BE 若未命中可复用缓存则会要求 FE 补发完整请求。" - + "当 BE 侧缓存命中率较高时，可以显著降低 FE 的出网带宽。", - "Whether to enable lightweight point-query requests. When enabled, FE will omit" - + " desc_tbl/output_expr/query_options on the first PreparedStatement execute" - + " request, and BE will ask FE to resend the full request if reusable cache" - + " is missing. This can significantly reduce FE outbound bandwidth when the" - + " BE-side reusable cache hit rate is high." - }) + @ConfField(mutable = true, description = "Whether to enable lightweight point-query requests. When enabled, FE " + + "will omit desc_tbl/output_expr/query_options on the first " + + "PreparedStatement execute request, and BE will ask FE to resend the " + + "full request if reusable cache is missing. This can significantly " + + "reduce FE outbound bandwidth when the BE-side reusable cache hit rate " + "is high.") public static boolean enable_lightweight_lookup_request = false; /** @@ -1172,9 +1120,8 @@ public class Config extends ConfigBase { public static int report_queue_size = 100; // if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum number of batched tasks per RPC assigned to each BE when resending agent tasks, " - + "the default value is 10000."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum number of batched tasks per RPC assigned " + + "to each BE when resending agent tasks, the default " + "value is 10000.") public static int report_resend_batch_task_num_per_rpc = 10000; /** @@ -1307,11 +1254,10 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static String[] force_skip_journal_ids = {}; - @ConfField(description = { - "When replaying editlog encounters exceptions with specific operation types that prevent FE from starting, " - + "you can configure the editlog operation type enum values to be ignored, " - + "thereby skipping these exceptions and allowing the replay thread to continue " - + "replaying other logs."}) + @ConfField(description = "When replaying editlog encounters exceptions with specific operation types that prevent " + + "FE from starting, you can configure the editlog operation type enum values to be " + + "ignored, thereby skipping these exceptions and allowing the replay thread to continue " + + "replaying other logs.") public static short[] skip_operation_types_on_replay_exception = {-1, -1}; /** @@ -1436,11 +1382,9 @@ public class Config extends ConfigBase { mutable = true, masterOnly = false, callbackClassString = "org.apache.doris.common.cache.NereidsSqlCacheManager$UpdateConfig", - description = { - "The current default setting is 300, which is used to control the expiration time of SQL cache " - + "in NereidsSqlCacheManager. If the cache is not accessed for a period of time, " - + "it will be reclaimed."} - ) + description = "The current default setting is 300, which is used to control the expiration time of SQL " + + "cache in NereidsSqlCacheManager. If the cache is not accessed for a period of time, it " + + "will be reclaimed.") public static int expire_sql_cache_in_fe_second = 300; /** @@ -1450,11 +1394,9 @@ public class Config extends ConfigBase { mutable = true, masterOnly = false, callbackClassString = "org.apache.doris.nereids.stats.MemoryHboPlanStatisticsProvider$UpdateConfig", - description = { - "The default setting is 86400, which is used to control the expiration time of plan stats cache " - + "in MemoryHboPlanStatisticsProvider. If the cache is not accessed for a period of time, " - + "it will be reclaimed."} - ) + description = "The default setting is 86400, which is used to control the expiration time of plan stats " + + "cache in MemoryHboPlanStatisticsProvider. If the cache is not accessed for a period of " + + "time, it will be reclaimed.") public static int expire_hbo_plan_stats_cache_in_fe_second = 86400; /** @@ -1464,11 +1406,9 @@ public class Config extends ConfigBase { mutable = true, masterOnly = false, callbackClassString = "org.apache.doris.nereids.stats.HboPlanInfoProvider$UpdateConfig", - description = { - "The default setting is 100, which is used to control the expiration time of HBO plan info cache " - + "in HboPlanInfoProvider. If the cache is not accessed for a period of time, " - + "it will be reclaimed."} - ) + description = "The default setting is 100, which is used to control the expiration time of HBO plan info " + + "cache in HboPlanInfoProvider. If the cache is not accessed for a period of time, it will " + + "be reclaimed.") public static int expire_hbo_plan_info_cache_in_fe_second = 1000; /** @@ -1478,24 +1418,23 @@ public class Config extends ConfigBase { mutable = true, masterOnly = false, callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig", - description = {"The current default setting is 300, which is used to control the expiration time of " - + "the partition metadata cache in NereidsSortedPartitionsCacheManager. " - + "If the cache is not accessed for a period of time, it will be reclaimed."} - ) + description = "The current default setting is 300, which is used to control the expiration time of the " + + "partition metadata cache in NereidsSortedPartitionsCacheManager. If the cache is not " + + "accessed for a period of time, it will be reclaimed.") public static int expire_cache_partition_meta_table_in_fe_second = 300; /** * Set the maximum number of rows that can be cached */ - @ConfField(mutable = true, masterOnly = false, description = { - "Maximum number of rows that can be cached in SQL/Partition Cache, is 3000 by default."}) + @ConfField(mutable = true, masterOnly = false, description = "Maximum number of rows that can be cached in " + + "SQL/Partition Cache, is 3000 by default.") public static int cache_result_max_row_count = 3000; /** * Set the maximum data size that can be cached */ - @ConfField(mutable = true, masterOnly = false, description = { - "Maximum data size of rows that can be cached in SQL/Partition Cache. The default is 30MB."}) + @ConfField(mutable = true, masterOnly = false, description = "Maximum data size of rows that can be cached in " + + "SQL/Partition Cache. The default is 30MB.") public static int cache_result_max_data_size = 31457280; // 30M /** @@ -1569,7 +1508,7 @@ public class Config extends ConfigBase { @ConfField public static boolean enable_bdbje_debug_mode = false; - @ConfField(mutable = false, masterOnly = true, description = {"Whether to enable debug points, used in testing."}) + @ConfField(mutable = false, masterOnly = true, description = "Whether to enable debug points, used in testing.") public static boolean enable_debug_points = false; /** @@ -1602,7 +1541,7 @@ public class Config extends ConfigBase { * sets the time without read activity before sending a keepalive ping * the smaller the value, the sooner the channel is unavailable, but it will increase network io */ - @ConfField(description = {"The time without GRPC read activity before sending a keepalive ping"}) + @ConfField(description = "The time without GRPC read activity before sending a keepalive ping") public static int grpc_keep_alive_second = 10; /** @@ -1615,8 +1554,7 @@ public class Config extends ConfigBase { * This option should only be enabled when you are sure responses are small and the risk is acceptable. * Takes effect after FE restart. */ - @ConfField(description = {"是否为 BackendServiceClient 使用 gRPC directExecutor", - "Whether to use gRPC directExecutor for BackendServiceClient"}) + @ConfField(description = "Whether to use gRPC directExecutor for BackendServiceClient") public static boolean grpc_backend_client_use_direct_executor = false; /** @@ -1698,8 +1636,8 @@ public class Config extends ConfigBase { /** * Control the max num of tablets per backup job involved. */ - @ConfField(mutable = true, masterOnly = true, description = { - "Control the maximum number of tablets per backup job, to avoid OOM."}) + @ConfField(mutable = true, masterOnly = true, description = "Control the maximum number of tablets per backup " + + "job, to avoid OOM.") public static int max_backup_tablets_per_job = 300000; /** @@ -1711,8 +1649,8 @@ public class Config extends ConfigBase { /** * whether to ignore temp partitions when backup, and not report exception. */ - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to ignore temporary partitions during backup without reporting an exception."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to ignore temporary partitions during backup " + + "without reporting an exception.") public static boolean ignore_backup_tmp_partitions = false; /** @@ -1725,13 +1663,12 @@ public class Config extends ConfigBase { /** * Whether to enable cloud restore job. */ - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to enable cloud restore job."}, varType = VariableAnnotation.EXPERIMENTAL) + @ConfField(mutable = true, masterOnly = true, description = "Whether " + + "to enable " + "cloud " + "restore " + "job.", varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_cloud_restore_job = false; - @ConfField(mutable = true, masterOnly = true, description = { - "During the cloud restore job, the maximum number of tablets created per " - + "create-tablets RPC. Default is 256."}) + @ConfField(mutable = true, masterOnly = true, description = "During the cloud restore job, the maximum number of " + + "tablets created per create-tablets RPC. Default is " + "256.") public static int cloud_restore_create_tablet_batch_size = 256; /** @@ -1764,13 +1701,12 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int table_name_length_limit = 64; - @ConfField(mutable = true, description = {"Used to limit the length of column comments. " - + "If the existing column comment is too long, it will be truncated when displayed."}) + @ConfField(mutable = true, description = "Used to limit the length of column comments. If the existing column " + + "comment is too long, it will be truncated when displayed.") public static int column_comment_length_limit = -1; - @ConfField(mutable = true, description = { - "Default compression type for internal tables. Supported values: LZ4, LZ4F, LZ4HC, ZLIB, ZSTD, " - + "SNAPPY, NONE."}) + @ConfField(mutable = true, description = "Default compression type for internal tables. Supported values: LZ4, " + + "LZ4F, LZ4HC, ZLIB, ZSTD, SNAPPY, NONE.") public static String default_compression_type = "ZSTD"; /* @@ -1859,9 +1795,9 @@ public class Config extends ConfigBase { /* * the system automatically checks the time interval for statistics */ - @ConfField(mutable = true, masterOnly = true, description = { - "This parameter controls the time interval for automatic collection jobs to check the health of table " - + "statistics and trigger automatic collection."}) + @ConfField(mutable = true, masterOnly = true, description = "This parameter controls the time interval for " + + "automatic collection jobs to check the health of " + + "table statistics and trigger automatic collection.") public static int auto_check_statistics_in_minutes = 1; /** @@ -1886,7 +1822,7 @@ public class Config extends ConfigBase { * corresponding type of job * The value should be greater than 0, if it is 0 or <=0, set it to 5 */ - @ConfField(masterOnly = true, description = {"The number of threads used to dispatch timer jobs."}) + @ConfField(masterOnly = true, description = "The number of threads used to dispatch timer jobs.") public static int job_dispatch_timer_job_thread_num = 2; /** @@ -1896,63 +1832,56 @@ public class Config extends ConfigBase { * {@code @dispatch_timer_job_thread_num} * The value should be greater than 0, if it is 0 or <=0, set it to 1024 */ - @ConfField(masterOnly = true, description = {"The number of timer jobs that can be queued."}) + @ConfField(masterOnly = true, description = "The number of timer jobs that can be queued.") public static int job_dispatch_timer_job_queue_size = 1024; - @ConfField(masterOnly = true, description = { - "Maximum number of persisted tasks allowed per job. Tasks exceeding this limit will be discarded. " - + "If the value is less than 1, tasks will not be persisted."}) + @ConfField(masterOnly = true, description = "Maximum number of persisted tasks allowed per job. Tasks exceeding " + + "this limit will be discarded. If the value is less than 1, tasks " + "will not be persisted.") public static int max_persistence_task_count = 100; - @ConfField(masterOnly = true, description = { - "The size of the MTMV task's waiting queue. If the size is negative, 1024 will be used. If " - + "the size is not a power of two, the nearest power of two will be" - + " automatically selected."}) + @ConfField(masterOnly = true, description = "The size of the MTMV task's waiting queue. If the size is negative, " + + "1024 will be used. If the size is not a power of two, the nearest " + + "power of two will be automatically selected.") public static int mtmv_task_queue_size = 1024; - @ConfField(masterOnly = true, description = { - "The size of the Insert task's waiting queue. If the size is negative, 1024 will be used." - + " If the size is not a power of two, the nearest power of two will " - + "be automatically selected."}) + @ConfField(masterOnly = true, description = "The size of the Insert task's waiting queue. If the size is " + + "negative, 1024 will be used. If the size is not a power of two, the " + + "nearest power of two will be automatically selected.") public static int insert_task_queue_size = 1024; - @ConfField(masterOnly = true, description = { - "The size of the Dictionary loading task's waiting queue. If the size is negative, 1024 will be used." - + " If the size is not a power of two, the nearest power of two will " - + "be automatically selected."}) + @ConfField(masterOnly = true, description = "The size of the Dictionary loading task's waiting queue. If the size " + + "is negative, 1024 will be used. If the size is not a power of two, " + + "the nearest power of two will be automatically selected.") public static int dictionary_task_queue_size = 1024; - @ConfField(masterOnly = true, description = { - "The maximum time to retain a finished job before it is deleted. Unit: hour."}) + @ConfField(masterOnly = true, description = "The maximum time to retain a finished job before it is deleted. " + + "Unit: hour.") public static int finished_job_cleanup_threshold_time_hour = 24; - @ConfField(masterOnly = true, description = {"The number of threads used to consume Insert tasks, " - + "the value should be greater than 0, if it is <=0, default is 10."}) + @ConfField(masterOnly = true, description = "The number of threads used to consume Insert tasks, the value should " + + "be greater than 0, if it is <=0, default is 10.") public static int job_insert_task_consumer_thread_num = 10; - @ConfField(masterOnly = true, description = {"The number of threads used to consume MTMV tasks, " - + "the value should be greater than 0, if it is <=0, default is 10."}) + @ConfField(masterOnly = true, description = "The number of threads used to consume MTMV tasks, the value should " + + "be greater than 0, if it is <=0, default is 10.") public static int job_mtmv_task_consumer_thread_num = 10; - @ConfField(masterOnly = true, description = { - "The number of threads used to perform dictionary import and delete tasks. The value should be" - + " greater than 0; otherwise it defaults to 3."}) + @ConfField(masterOnly = true, description = "The number of threads used to perform dictionary import and delete " + + "tasks. The value should be greater than 0; otherwise it defaults to " + "3.") public static int job_dictionary_task_consumer_thread_num = 3; - @ConfField(masterOnly = true, description = {"The number of threads used to execute streaming tasks. " - + "The value should be greater than 0; if it is <=0, the default is 100."}) + @ConfField(masterOnly = true, description = "The number of threads used to execute streaming tasks. The value " + + "should be greater than 0; if it is <=0, the default is 100.") public static int job_streaming_task_exec_thread_num = 100; - @ConfField(masterOnly = true, description = {"The maximum number of streaming jobs. " - + "The value should be greater than 0; if it is <=0, the default is 1024."}) + @ConfField(masterOnly = true, description = "The maximum number of streaming jobs. The value should be greater " + + "than 0; if it is <=0, the default is 1024.") public static int max_streaming_job_num = 1024; - @ConfField(masterOnly = true, description = { - "The maximum number of tasks a streaming job can keep in memory. If the number exceeds the limit, " - + "old records will be discarded."}) + @ConfField(masterOnly = true, description = "The maximum number of tasks a streaming job can keep in memory. If " + + "the number exceeds the limit, old records will be discarded.") public static int max_streaming_task_show_count = 100; - @ConfField(masterOnly = true, mutable = true, description = { - "Max auto resume retry count for streaming jobs. " - + "After exceeding, the failure reason is rewritten to CANNOT_RESUME_ERR " - + "and the job requires manual intervention."}) + @ConfField(masterOnly = true, mutable = true, description = "Max auto resume retry count for streaming jobs. " + + "After exceeding, the failure reason is rewritten to " + "CANNOT_RESUME_ERR and the job requires manual " + + "intervention.") public static int streaming_job_max_auto_resume_count = 10; /* job test config */ @@ -1987,12 +1916,12 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static long query_queue_update_interval_ms = 5000; - @ConfField(mutable = true, description = {"When BE memory usage is higher than this value, queries may be queued. " - + "Default value is -1, meaning this feature is disabled. Decimal value range is from 0 to 1."}) + @ConfField(mutable = true, description = "When BE memory usage is higher than this value, queries may be queued. " + + "Default value is -1, meaning this feature is disabled. Decimal value " + "range is from 0 to 1.") public static double query_queue_by_be_used_memory = -1; - @ConfField(mutable = true, description = {"In the scenario of memory back-pressure, " - + "the time interval for periodically obtaining BE memory usage."}) + @ConfField(mutable = true, description = "In the scenario of memory back-pressure, the time interval for " + + "periodically obtaining BE memory usage.") public static long get_be_resource_usage_interval_ms = 10000; @ConfField(mutable = false, masterOnly = true) @@ -2055,20 +1984,19 @@ public class Config extends ConfigBase { * And the max number of compute node is controlled by min_backend_num_for_external_table. * If set to false, query on external table will assign to any node. */ - @ConfField(mutable = true, description = { - "If set to true, queries on external tables will prefer to be assigned to compute nodes. " - + "The maximum number of compute nodes is controlled by min_backend_num_for_external_table. " - + "If set to false, queries on external tables will be assigned to any node. " - + "If there are no compute nodes in the cluster, this config has no effect."}) + @ConfField(mutable = true, description = "If set to true, queries on external tables will prefer to be assigned " + + "to compute nodes. The maximum number of compute nodes is controlled by " + + "min_backend_num_for_external_table. If set to false, queries on " + + "external tables will be assigned to any node. If there are no compute " + + "nodes in the cluster, this config has no effect.") public static boolean prefer_compute_node_for_external_table = false; - @ConfField(mutable = true, description = {"Only takes effect when prefer_compute_node_for_external_table is true. " - + "If the compute node count is less than this value, " - + "queries on external tables will try to use some mix nodes as well, " - + "to let the total number of nodes reach this value. " - + "If the compute node count is larger than this value, " - + "queries on external tables will be assigned to compute nodes only. " - + "-1 means only use current compute nodes."}) + @ConfField(mutable = true, description = "Only takes effect when prefer_compute_node_for_external_table is true. " + + "If the compute node count is less than this value, queries on external " + + "tables will try to use some mix nodes as well, to let the total number " + + "of nodes reach this value. If the compute node count is larger than " + + "this value, queries on external tables will be assigned to compute " + + "nodes only. -1 means only use current compute nodes.") public static int min_backend_num_for_external_table = -1; /** @@ -2086,18 +2014,18 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = false) public static boolean disable_backend_black_list = false; - @ConfField(mutable = true, masterOnly = false, description = { - "If a backend is attempted to be added to the blacklist do_add_backend_black_list_threshold_count times " - + "within do_add_backend_black_list_threshold_seconds, it will be added to the blacklist."}) + @ConfField(mutable = true, masterOnly = false, description = "If a backend is attempted to be added to the " + + "blacklist do_add_backend_black_list_threshold_count " + "times within " + + "do_add_backend_black_list_threshold_seconds, it " + "will be added to the blacklist.") public static long do_add_backend_black_list_threshold_count = 10; - @ConfField(mutable = true, masterOnly = false, description = { - "If a backend is attempted to be added to the blacklist do_add_backend_black_list_threshold_count times " - + "within do_add_backend_black_list_threshold_seconds, it will be added to the blacklist."}) + @ConfField(mutable = true, masterOnly = false, description = "If a backend is attempted to be added to the " + + "blacklist do_add_backend_black_list_threshold_count " + "times within " + + "do_add_backend_black_list_threshold_seconds, it " + "will be added to the blacklist.") public static long do_add_backend_black_list_threshold_seconds = 30; - @ConfField(mutable = true, masterOnly = false, description = { - "A backend will stay in the blacklist for this duration after being added."}) + @ConfField(mutable = true, masterOnly = false, description = "A backend will stay in the blacklist for this " + + "duration after being added.") public static long stay_in_backend_black_list_threshold_seconds = 60; /** @@ -2160,48 +2088,47 @@ public class Config extends ConfigBase { * Max cache num of hive partition. * Decrease this value if FE's memory is small */ - @ConfField(description = {"Maximum cache number of partitions at table level in Hive Metastore."}) + @ConfField(description = "Maximum cache number of partitions at table level in Hive Metastore.") public static long max_hive_partition_cache_num = 100000; - @ConfField(description = {"Maximum cache number of Hudi/Iceberg tables."}) + @ConfField(description = "Maximum cache number of Hudi/Iceberg tables.") public static long max_external_table_cache_num = 1000; - @ConfField(description = {"Maximum cache number of database and table instances in external catalogs."}) + @ConfField(description = "Maximum cache number of database and table instances in external catalogs.") public static long max_meta_object_cache_num = 1000; - @ConfField(mutable = false, masterOnly = false, description = { - "Stripe count used by multi-key MetaCacheEntry instances such as external object caches."}) + @ConfField(mutable = false, masterOnly = false, + description = "Stripe count used by multi-key MetaCacheEntry instances such as external object caches.") public static int external_meta_cache_object_entry_lock_stripes = 256; - @ConfField(mutable = true, masterOnly = false, description = { - "Whether to synchronously refresh external database/table names when a name lookup misses in an " - + "existing cached snapshot. This option is enabled by default to preserve existing external " + @ConfField(mutable = true, masterOnly = false, + description = "Whether to synchronously refresh external database/table names when a name lookup misses " + + "in an existing cached snapshot. This option is enabled by default to preserve existing external " + "catalog visibility behavior. Disable it to avoid repeated remote name enumeration for " - + "non-existent objects."}) + + "non-existent objects.") public static boolean enable_external_meta_cache_name_miss_refresh = true; - @ConfField(description = {"Maximum cache number of Hive partitioned tables."}) + @ConfField(description = "Maximum cache number of Hive partitioned tables.") public static long max_hive_partition_table_cache_num = 10000; - @ConfField(mutable = false, masterOnly = false, description = { - "Max number of hive partition values to return while list partitions, -1 means no limitation."}) + @ConfField(mutable = false, masterOnly = false, description = "Max number of hive partition values to return " + + "while list partitions, -1 means no limitation.") public static short max_hive_list_partition_num = -1; - @ConfField(mutable = false, masterOnly = false, description = {"Max cache number of remote file system."}) + @ConfField(mutable = false, masterOnly = false, description = "Max cache number of remote file system.") public static long max_remote_file_system_cache_num = 100; - @ConfField(mutable = false, masterOnly = false, description = { - "Maximum cache number of external table row counts."}) + @ConfField(mutable = false, masterOnly = false, description = "Maximum cache number of external table row counts.") public static long max_external_table_row_count_cache_num = 100000; - @ConfField(description = {"Maximum cached file number for external table split file meta cache at query level."}) + @ConfField(description = "Maximum cached file number for external table split file meta cache at query level.") public static long max_external_table_split_file_meta_cache_num = 100000; /** * Maximum number of MaxCompute Storage API write block IDs that can be allocated in one write session. */ - @ConfField(mutable = false, masterOnly = true, description = { - "Maximum number of MaxCompute Storage API write block IDs that can be allocated in one write session."}) + @ConfField(mutable = false, masterOnly = true, description = "Maximum number of MaxCompute Storage API write " + + "block IDs that can be allocated in one write " + "session.") public static long max_compute_write_max_block_count = 20000L; /** @@ -2225,13 +2152,18 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static long max_external_schema_cache_num = 10000; - @ConfField(description = { - "The expiration time of a cache object after its last access. Used for external meta cache."}) + @ConfField(description = "The expiration time of a cache object after its last access. Used for external meta " + + "cache.") public static long external_cache_expire_time_seconds_after_access = 86400L; // 24 hours - @ConfField(description = {"The auto-refresh interval of the external meta cache."}) + @ConfField(description = "The auto-refresh interval of the external meta cache.") public static long external_cache_refresh_time_minutes = 10; // 10 mins + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. + @ConfField(mutable = true, masterOnly = false, + description = "Whether external meta cache uses manual miss load instead of Caffeine sync load.") + public static boolean enable_external_meta_cache_manual_miss_load = true; + /** * Github workflow test type, for setting some session variables * only for certain test type. E.g. only settting batch_size to small @@ -2261,9 +2193,8 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int max_same_name_catalog_trash_num = 3; - @ConfField(masterOnly = true, description = { - "The interval between catalog recycle bin clean tasks. " - + "Default is 30000 milliseconds (30 seconds)."}) + @ConfField(masterOnly = true, description = "The interval between catalog recycle bin clean tasks. Default is " + + "30000 milliseconds (30 seconds).") public static long catalog_recycle_bin_interval_ms = 30 * 1000; /** @@ -2308,18 +2239,16 @@ public class Config extends ConfigBase { @ConfField( mutable = true, callbackClassString = "org.apache.doris.common.cache.NereidsSqlCacheManager$UpdateConfig", - description = {"Currently defaults to 100. This config is used to control the number of " - + "SQL caches managed by NereidsSqlCacheManager."} - ) + description = "Currently defaults to 100. This config is used to control the number of SQL caches managed " + + "by NereidsSqlCacheManager.") public static int sql_cache_manage_num = 100; @ConfField( mutable = true, callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig", - description = {"Currently defaults to 100. This is used to control the number of ordered " - + "partition metadata caches in NereidsSortedPartitionsCacheManager, " - + "and to accelerate partition pruning."} - ) + description = "Currently defaults to 100. This is used to control the number of ordered partition " + + "metadata caches in NereidsSortedPartitionsCacheManager, and to accelerate partition " + + "pruning.") public static int cache_partition_meta_table_manage_num = 100; /** @@ -2328,9 +2257,8 @@ public class Config extends ConfigBase { @ConfField( mutable = true, callbackClassString = "org.apache.doris.nereids.stats.MemoryHboPlanStatisticsProvider$UpdateConfig", - description = {"Currently defaults to 100000. This config is used to control the number of " - + "HBO plan stats cache entries."} - ) + description = "Currently defaults to 100000. This config is used to control the number of HBO plan stats " + + "cache entries.") public static int hbo_plan_stats_cache_num = 100000; /** @@ -2338,9 +2266,8 @@ public class Config extends ConfigBase { */ @ConfField( mutable = true, - description = {"Currently defaults to 10. This config is used to control the number of " - + "recent runs entries in the HBO plan stats cache."} - ) + description = "Currently defaults to 10. This config is used to control the number of recent runs entries " + + "in the HBO plan stats cache.") public static int hbo_plan_stats_cache_recent_runs_entry_num = 10; /** @@ -2349,9 +2276,8 @@ public class Config extends ConfigBase { @ConfField( mutable = true, callbackClassString = "org.apache.doris.nereids.stats.HboPlanInfoProvider$UpdateConfig", - description = {"Currently defaults to 1000. This config is used to control the number of " - + "HBO plan info cache entries."} - ) + description = "Currently defaults to 1000. This config is used to control the number of HBO plan info " + + "cache entries.") public static int hbo_plan_info_cache_num = 1000; /** @@ -2479,9 +2405,8 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static boolean enable_round_robin_create_tablet = true; - @ConfField(mutable = true, masterOnly = true, description = { - "When creating tablets for a partition, always start from the first BE. " - + "Note: This method may cause BE imbalance."}) + @ConfField(mutable = true, masterOnly = true, description = "When creating tablets for a partition, always start " + + "from the first BE. Note: This method may cause BE " + "imbalance.") public static boolean create_tablet_round_robin_from_start = false; /** @@ -2541,33 +2466,33 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static boolean enable_query_hit_stats = false; - @ConfField(mutable = true, description = {"When set to true, if a query is unable to select a healthy replica, " - + "the detailed information of all replicas of the tablet, " - + "including the specific reason why they are unqueryable, will be printed out."}) + @ConfField(mutable = true, description = "When set to true, if a query is unable to select a healthy replica, the " + + "detailed information of all replicas of the tablet, including the " + + "specific reason why they are unqueryable, will be printed out.") public static boolean show_details_for_unaccessible_tablet = true; - @ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = { - "Whether to enable the binlog feature"}) + @ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = "Whether " + + "to " + "enable " + "the " + "binlog " + "feature") public static boolean enable_feature_binlog = false; - @ConfField(mutable = false, description = {"Whether to enable the binlog feature for databases/tables by default"}) + @ConfField(mutable = false, description = "Whether to enable the binlog feature for databases/tables by default") public static boolean force_enable_feature_binlog = false; - @ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = { - "Set the maximum byte length of a binlog message"}) + @ConfField(mutable = false, masterOnly = false, varType = VariableAnnotation.EXPERIMENTAL, description = "Set the " + + "maximum " + "byte " + "length " + "of a " + "binlog " + "message") public static int max_binlog_messsage_size = 1024 * 1024 * 1024; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to disable creating catalog with WITH RESOURCE statement."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to disable creating catalog with WITH " + + "RESOURCE statement.") public static boolean disallow_create_catalog_with_resource = true; - @ConfField(mutable = true, masterOnly = false, description = {"Sample size for hive row count estimation."}) + @ConfField(mutable = true, masterOnly = false, description = "Sample size for hive row count estimation.") public static int hive_stats_partition_sample_size = 30; - @ConfField(mutable = true, masterOnly = true, description = {"Whether to enable external Hive bucket tables"}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable external Hive bucket tables") public static boolean enable_create_hive_bucket_table = false; - @ConfField(mutable = true, masterOnly = true, description = {"Default Hive file format when creating tables."}) + @ConfField(mutable = true, masterOnly = true, description = "Default Hive file format when creating tables.") public static String hive_default_file_format = "orc"; @ConfField @@ -2576,42 +2501,34 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static long statistics_sql_mem_limit_in_bytes = 2L * 1024 * 1024 * 1024; - @ConfField(mutable = true, masterOnly = true, description = { - "Used to force the number of replicas of internal tables. If this config is greater than zero, " - + "the number of replicas specified by the user when creating the table will be ignored, " - + "and the value set by this parameter will be used. At the same time, the replica tags " - + "and other parameters specified in the CREATE TABLE statement will be ignored. " - + "This config does not affect operations including creating partitions " - + "and modifying table properties. " - + "This config is recommended to be used only in the test environment."}) + @ConfField(mutable = true, masterOnly = true, description = "Used to force the number of replicas of internal " + + "tables. If this config is greater than zero, the " + "number of replicas specified by the user when " + + "creating the table will be ignored, and the value " + "set by this parameter will be used. At the same " + + "time, the replica tags and other parameters " + "specified in the CREATE TABLE statement will be " + + "ignored. This config does not affect operations " + "including creating partitions and modifying table " + + "properties. This config is recommended to be used " + "only in the test environment.") public static int force_olap_table_replication_num = 0; - @ConfField(mutable = true, description = { - "Used to force set the replica allocation of internal tables. If this config is not empty, " - + "the replication_num and replication_allocation specified by the user when creating the table " - + "or partitions will be ignored, and the value set by this parameter will be used. " - + "This config affects operations including creating tables, creating partitions, and creating " - + "dynamic partitions. This config is recommended to be used only in the test environment."}) + @ConfField(mutable = true, description = "Used to force set the replica allocation of internal tables. If this " + + "config is not empty, the replication_num and replication_allocation " + + "specified by the user when creating the table or partitions will be " + + "ignored, and the value set by this parameter will be used. This config " + + "affects operations including creating tables, creating partitions, and " + + "creating dynamic partitions. This config is recommended to be used only " + "in the test environment.") public static String force_olap_table_replication_allocation = ""; @ConfField public static int auto_analyze_simultaneously_running_task_num = 1; - @ConfField(mutable = true, masterOnly = true, description = { - "统计信息收集时 string 列允许的最大字节长度。若列中存在长度超过该值的行，" - + "该列的统计信息将被跳过收集（task 仍标记为 FINISHED，在 SHOW ANALYZE 中显示跳过原因）。" - + "≤ 0 表示关闭此保护。默认 1024 (1KB)。" - + "注意：此保护只覆盖 FULL / LINEAR / DUJ1 统计收集路径（即 analyze 全表和 sample 的主 SQL）。" - + "当 enable_partition_analyze=true 时的 per-partition 路径（PARTITION_ANALYZE_TEMPLATE）" - + "出于正确性考虑不启用该保护，详见 BaseAnalysisTask 中的 NOTE。", - "Max byte length allowed for a string column when collecting statistics. " - + "If any row in a string column is longer than this value, the column's stats " - + "collection is skipped (the task is still marked FINISHED, with the skip reason " - + "shown in SHOW ANALYZE). A value <= 0 disables this protection. Default: 1024 (1KB). " - + "Note: this protection applies to the FULL / LINEAR / DUJ1 collection paths " - + "(i.e. the main SQL used by full-table and sample analyze). The per-partition path " - + "(PARTITION_ANALYZE_TEMPLATE, used when enable_partition_analyze=true) is intentionally " - + "not guarded for correctness reasons; see the NOTE in BaseAnalysisTask."}) + @ConfField(mutable = true, masterOnly = true, description = "Max byte length allowed for a string column when " + + "collecting statistics. If any row in a string column " + "is longer than this value, the column's stats " + + "collection is skipped (the task is still marked " + "FINISHED, with the skip reason shown in SHOW " + + "ANALYZE). A value <= 0 disables this protection. " + + "Default: 1024 (1KB). Note: this protection applies " + + "to the FULL / LINEAR / DUJ1 collection paths (i.e. " + + "the main SQL used by full-table and sample analyze). " + + "The per-partition path (PARTITION_ANALYZE_TEMPLATE, " + "used when enable_partition_analyze=true) is " + + "intentionally not guarded for correctness reasons; " + "see the NOTE in BaseAnalysisTask.") public static long statistics_max_string_column_length = 1024; @ConfField(mutable = false) @@ -2623,145 +2540,132 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static boolean force_sample_analyze = false; // avoid full analyze for performance reason - @ConfField(mutable = true, description = {"The maximum number of partitions allowed for an Export job"}) + @ConfField(mutable = true, description = "The maximum number of partitions allowed for an Export job") public static int maximum_number_of_export_partitions = 2000; - @ConfField(mutable = true, description = {"Whether to use MySQL's BIGINT type to return Doris's LARGEINT type"}) + @ConfField(mutable = true, description = "Whether to use MySQL's BIGINT type to return Doris's LARGEINT type") public static boolean use_mysql_bigint_for_largeint = false; @ConfField public static boolean forbid_running_alter_job = false; - @ConfField(description = {"Temporary config field. Will make all OLAP tables enable light schema change."}) + @ConfField(description = "Temporary config field. Will make all OLAP tables enable light schema change.") public static boolean enable_convert_light_weight_schema_change = false; - @ConfField(mutable = true, masterOnly = false, description = { - "When querying the information_schema.metadata_name_ids table, " - + "the timeout for obtaining all tables in one database."}) + @ConfField(mutable = true, masterOnly = false, description = "When querying the " + + "information_schema.metadata_name_ids table, the " + "timeout for obtaining all tables in one database.") public static long query_metadata_name_ids_timeout = 3; - @ConfField(mutable = true, masterOnly = true, description = {"Whether to disable LocalDeployManager drop node."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to disable LocalDeployManager drop node.") public static boolean disable_local_deploy_manager_drop_node = true; - @ConfField(mutable = true, description = { - "When file cache is enabled, the number of virtual nodes of each node in the consistent hash algorithm. " - + "The larger the value, the more uniform the distribution of the hash algorithm, " - + "but it will increase the memory overhead."}) + @ConfField(mutable = true, description = "When file cache is enabled, the number of virtual nodes of each node in " + + "the consistent hash algorithm. The larger the value, the more uniform " + + "the distribution of the hash algorithm, but it will increase the memory " + "overhead.") public static int split_assigner_virtual_node_number = 256; - @ConfField(mutable = true, description = {"Local node soft affinity optimization. Prefer local replication node."}) + @ConfField(mutable = true, description = "Local node soft affinity optimization. Prefer local replication node.") public static boolean split_assigner_optimized_local_scheduling = true; - @ConfField(mutable = true, description = { - "The random algorithm has the smallest number of candidates and will select the most idle node."}) + @ConfField(mutable = true, description = "The random algorithm has the smallest number of candidates and will " + + "select the most idle node.") public static int split_assigner_min_random_candidate_num = 2; - @ConfField(mutable = true, description = { - "The consistent hash algorithm has the smallest number of candidates and will select the most idle node."}) + @ConfField(mutable = true, description = "The consistent hash algorithm has the smallest number of candidates and " + + "will select the most idle node.") public static int split_assigner_min_consistent_hash_candidate_num = 2; - @ConfField(mutable = true, description = {"The maximum difference in the number of splits between nodes. " - + "If this number is exceeded, the splits will be redistributed."}) + @ConfField(mutable = true, description = "The maximum difference in the number of splits between nodes. If this " + + "number is exceeded, the splits will be redistributed.") public static int split_assigner_max_split_num_variance = 1; - @ConfField(description = {"Determines the number of persisted automatic analyze job execution status records."}) + @ConfField(description = "Determines the number of persisted automatic analyze job execution status records.") public static long analyze_record_limit = 20000; - @ConfField(mutable = true, masterOnly = true, description = {"Minimum number of buckets for auto bucketing."}) + @ConfField(mutable = true, masterOnly = true, description = "Minimum number of buckets for auto bucketing.") public static int autobucket_min_buckets = 3; - @ConfField(mutable = true, masterOnly = true, description = {"Maximum number of buckets for auto bucketing."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum number of buckets for auto bucketing.") public static int autobucket_max_buckets = 128; - @ConfField(mutable = true, masterOnly = true, description = { - "Maximum number of buckets allowed when creating a table or adding a partition. " - + "This config shares the same default value with autobucket_max_buckets for consistency. " - + "Behavior: " - + "1. For user-specified buckets (CREATE TABLE / ALTER TABLE ADD PARTITION): " - + "if bucket number exceeds this limit, the operation will be rejected with an error message. " - + "2. For auto-bucket feature (Dynamic Partition): " - + "bucket number will be capped at autobucket_max_buckets automatically. " - + "Set to 0 or negative value to disable this limit for user-specified buckets."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum number of buckets allowed when creating a " + + "table or adding a partition. This config shares the " + + "same default value with autobucket_max_buckets for " + + "consistency. Behavior: 1. For user-specified buckets " + + "(CREATE TABLE / ALTER TABLE ADD PARTITION): if " + + "bucket number exceeds this limit, the operation will " + "be rejected with an error message. 2. For " + + "auto-bucket feature (Dynamic Partition): bucket " + "number will be capped at autobucket_max_buckets " + + "automatically. Set to 0 or negative value to disable " + "this limit for user-specified buckets.") public static int max_bucket_num_per_partition = 768; - @ConfField(description = {"Maximum number of connections for the Arrow Flight Server per FE."}) + @ConfField(description = "Maximum number of connections for the Arrow Flight Server per FE.") public static int arrow_flight_max_connections = 4096; - @ConfField(mutable = true, masterOnly = true, description = { - "In auto bucketing, the number of buckets is estimated based on the partition size. " - + "For storage and computing integration, a partition size of 5GB is estimated as one bucket, " - + "but for cloud, a partition size of 10GB is estimated as one bucket. " - + "If the configuration is less than 0, the code will adaptively use a default of 5GB " - + "in non-cloud mode, and 10GB in cloud mode."}) + @ConfField(mutable = true, masterOnly = true, description = "In auto bucketing, the number of buckets is " + + "estimated based on the partition size. For storage " + + "and computing integration, a partition size of 5GB " + "is estimated as one bucket, but for cloud, a " + + "partition size of 10GB is estimated as one bucket. " + + "If the configuration is less than 0, the code will " + + "adaptively use a default of 5GB in non-cloud mode, " + "and 10GB in cloud mode.") public static int autobucket_partition_size_per_bucket_gb = -1; - @ConfField(mutable = true, masterOnly = true, description = { - "If the new partition bucket number calculated by auto bucketing exceeds this percentage " - + "of the previous partition's bucket number, " - + "it is considered an abnormal case and triggers an alert."}) + @ConfField(mutable = true, masterOnly = true, description = "If the new partition bucket number calculated by " + + "auto bucketing exceeds this percentage of the " + "previous partition's bucket number, it is considered " + + "an abnormal case and triggers an alert.") public static double autobucket_out_of_bounds_percent_threshold = 0.5; - @ConfField(description = { - "(Deprecated, replaced by arrow_flight_max_connection) The cache limit of all user tokens in " - + "Arrow Flight Server, which will be eliminated by LRU rules after exceeding the limit. " - + "Arrow Flight SQL is a stateless protocol; the connection is usually not actively disconnected. " - + "A bearer token evicted from the cache will unregister its ConnectContext."}) + @ConfField(description = "(Deprecated, replaced by arrow_flight_max_connection) The cache limit of all user " + + "tokens in Arrow Flight Server, which will be eliminated by LRU rules after exceeding " + + "the limit. Arrow Flight SQL is a stateless protocol; the connection is usually not " + + "actively disconnected. A bearer token evicted from the cache will unregister its " + "ConnectContext.") public static int arrow_flight_token_cache_size = 4096; - @ConfField(description = { - "The alive time of the user token in Arrow Flight Server (expire after write), in seconds. " - + "The default value is 86400, which is 1 day."}) + @ConfField(description = "The alive time of the user token in Arrow Flight Server (expire after write), in " + + "seconds. The default value is 86400, which is 1 day.") public static int arrow_flight_token_alive_time_second = 86400; - @ConfField(mutable = true, description = { - "To ensure compatibility with the MySQL ecosystem, Doris includes a built-in database called mysql. " - + "If this database conflicts with a user's own database, please modify this field to replace " - + "the name of the Doris built-in MySQL database with a different name."}) + @ConfField(mutable = true, description = "To ensure compatibility with the MySQL ecosystem, Doris includes a " + + "built-in database called mysql. If this database conflicts with a " + + "user's own database, please modify this field to replace the name of " + + "the Doris built-in MySQL database with a different name.") public static String mysqldb_replace_name = "mysql"; - @ConfField(description = {"Set the specific domain name that allows cross-domain access. " - + "By default, any domain name is allowed cross-domain access."}) + @ConfField(description = "Set the specific domain name that allows cross-domain access. By default, any domain " + + "name is allowed cross-domain access.") public static String access_control_allowed_origin_domain = "*"; - @ConfField(description = { - "Used to enable Java UDF. Default is true. If this configuration is false, creation and use of Java UDF is " - + "disabled. In some scenarios it may be necessary to disable this configuration to prevent " - + "command injection attacks."}) + @ConfField(description = "Used to enable Java UDF. Default is true. If this configuration is false, creation and " + + "use of Java UDF is disabled. In some scenarios it may be necessary to disable this " + + "configuration to prevent command injection attacks.") public static boolean enable_java_udf = true; - @ConfField(mutable = true, masterOnly = true, description = { - "When enabled, data can be processed using the globally created Java UDF function during import. " - + "The default setting is false."}) + @ConfField(mutable = true, masterOnly = true, description = "When enabled, data can be processed using the " + + "globally created Java UDF function during import. " + "The default setting is false.") public static boolean enable_udf_in_load = false; - @ConfField(description = { - "Used to enable Python UDF. Default is true. If this configuration is false, " - + "creation and use of Python UDF is disabled. " - + "In some scenarios it may be necessary to disable this configuration to prevent " - + "command injection attacks."}) + @ConfField(description = "Used to enable Python UDF. Default is true. If this configuration is false, creation " + + "and use of Python UDF is disabled. In some scenarios it may be necessary to disable " + + "this configuration to prevent command injection attacks.") public static boolean enable_python_udf = true; - @ConfField(description = {"Whether to ignore unknown modules in Image file. " - + "If true, metadata modules not in PersistMetaModules.MODULE_NAMES " - + "will be ignored and skipped. Default is false, if Image file contains unknown modules, " - + "Doris will throw exception. " - + "This parameter is mainly used in downgrade operation, " - + "old version can be compatible with new version Image file."}) + @ConfField(description = "Whether to ignore unknown modules in Image file. If true, metadata modules not in " + + "PersistMetaModules.MODULE_NAMES will be ignored and skipped. Default is false, if Image " + + "file contains unknown modules, Doris will throw exception. This parameter is mainly " + + "used in downgrade operation, old version can be compatible with new version Image file.") public static boolean ignore_unknown_metadata_module = false; - @ConfField(mutable = true, description = { - "The timeout for FE Follower/Observer synchronizing an image file from the FE Master. Can be adjusted " - + "based on the size of the image file in ${meta_dir}/image and the network environment between " - + "nodes. The default value is 300."}) + @ConfField(mutable = true, description = "The timeout for FE Follower/Observer synchronizing an image file from " + + "the FE Master. Can be adjusted based on the size of the image file in " + + "${meta_dir}/image and the network environment between nodes. The " + "default value is 300.") public static int sync_image_timeout_second = 300; - @ConfField(mutable = true, description = { - "The batch size (in bytes) when loading the binary content of a module from the " - + "image file into a byte array and deserializing it into a UTF-8 encoded string " - + "when FE starts. A value of -1 means reading the entire byte array at once and " - + "then deserializing it into a UTF-8 encoded string; any other value means reading " - + "a certain size (at least 16MB) of byte array in batches, deserializing each into a " - + "UTF-8 encoded string, and then merging them into a complete string. The default value is -1."}) + @ConfField(mutable = true, description = "The batch size (in bytes) when loading the binary content of a module " + + "from the image file into a byte array and deserializing it into a UTF-8 " + + "encoded string when FE starts. A value of -1 means reading the entire " + + "byte array at once and then deserializing it into a UTF-8 encoded " + + "string; any other value means reading a certain size (at least 16MB) of " + + "byte array in batches, deserializing each into a UTF-8 encoded string, " + + "and then merging them into a complete string. The default value is -1.") public static int metadata_text_read_max_batch_bytes = -1; @ConfField(mutable = true, masterOnly = true) @@ -2789,8 +2693,8 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static int query_audit_log_timeout_ms = 5000; - @ConfField(description = {"The operations of the users in this list will not be recorded in the audit log. " - + "Multiple users are separated by commas."}) + @ConfField(description = "The operations of the users in this list will not be recorded in the audit log. " + + "Multiple users are separated by commas.") public static String skip_audit_user_list = ""; @ConfField(mutable = true) @@ -2799,58 +2703,54 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int workload_group_max_num = 15; - @ConfField(description = {"The timeout threshold for checking the WAL queue on BE, in milliseconds."}) + @ConfField(description = "The timeout threshold for checking the WAL queue on BE, in milliseconds.") public static int check_wal_queue_timeout_threshold = 180000; // 3 min - @ConfField(mutable = true, masterOnly = true, description = { - "For auto-partitioned tables to prevent users from accidentally creating a large number of partitions, " - + "the number of partitions allowed per OLAP table is `max_auto_partition_num`. Default 20000."}) + @ConfField(mutable = true, masterOnly = true, description = "For auto-partitioned tables to prevent users from " + + "accidentally creating a large number of partitions, " + + "the number of partitions allowed per OLAP table is " + "`max_auto_partition_num`. Default 20000.") public static int max_auto_partition_num = 20000; - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum difference in the number of tablets of each BE in partition rebalance mode. " - + "If it is less than this value, it will be diagnosed as balanced."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum difference in the number of tablets of " + + "each BE in partition rebalance mode. If it is less " + + "than this value, it will be diagnosed as balanced.") public static int diagnose_balance_max_tablet_num_diff = 50; - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum ratio of the number of tablets in each BE in partition rebalance mode. " - + "If it is less than this value, it will be diagnosed as balanced."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum ratio of the number of tablets in each " + + "BE in partition rebalance mode. If it is less than " + "this value, it will be diagnosed as balanced.") public static double diagnose_balance_max_tablet_num_ratio = 1.1; - @ConfField(masterOnly = true, description = { - "Set root user initial 2-staged SHA-1 encrypted password, default as '', means no root password. " - + "Subsequent `set password` operations for root user will overwrite the initial root password. " - + "Example: If you want to configure a plaintext password `root@123`." - + "You can execute Doris SQL `select password('root@123')` to generate encrypted " - + "password `*A00C34073A26B40AB4307650BFB9309D6BFA6999`"}) + @ConfField(masterOnly = true, description = "Set root user initial 2-staged SHA-1 encrypted password, default as " + + "'', means no root password. Subsequent `set password` operations for " + + "root user will overwrite the initial root password. Example: If you " + + "want to configure a plaintext password `root@123`.You can execute " + + "Doris SQL `select password('root@123')` to generate encrypted " + + "password `*A00C34073A26B40AB4307650BFB9309D6BFA6999`") public static String initial_root_password = ""; - @ConfField(description = {"The path of the nereids trace file."}) + @ConfField(description = "The path of the nereids trace file.") public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace"; - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum number of snapshots assigned to an upload task during the backup process. " - + "The default value is 10."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum number of snapshots assigned to an " + + "upload task during the backup process. The default " + "value is 10.") public static int backup_upload_snapshot_batch_size = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum number of snapshots assigned to a download task during the restore process. " - + "The default value is 10."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum number of snapshots assigned to a " + + "download task during the restore process. The " + "default value is 10.") public static int restore_download_snapshot_batch_size = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "The maximum number of batched tasks per RPC assigned to each BE during the backup/restore process. " - + "The default value is 10000."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum number of batched tasks per RPC assigned " + + "to each BE during the backup/restore process. The " + "default value is 10000.") public static int backup_restore_batch_task_num_per_rpc = 10000; - @ConfField(mutable = true, masterOnly = true, description = {"The number of concurrent restore tasks per BE."}) + @ConfField(mutable = true, masterOnly = true, description = "The number of concurrent restore tasks per BE.") public static int restore_task_concurrency_per_be = 5000; - @ConfField(mutable = true, description = { - "The time after which a BE is considered unavailable if no heartbeat is received."}) + @ConfField(mutable = true, description = "The time after which a BE is considered unavailable if no heartbeat is " + + "received.") public static int agent_task_be_unavailable_heartbeat_timeout_second = 300; - @ConfField(description = {"Whether to enable the function of getting log files through the HTTP interface."}) + @ConfField(description = "Whether to enable the function of getting log files through the HTTP interface.") public static boolean enable_get_log_file_api = false; @ConfField(mutable = true) @@ -2858,18 +2758,16 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static boolean enable_collect_internal_query_profile = false; - @ConfField(mutable = false, masterOnly = false, description = { - "The maximum number of worker threads for the HTTP SQL submitter."}) + @ConfField(mutable = false, masterOnly = false, description = "The maximum number of worker threads for the HTTP " + + "SQL submitter.") public static int http_sql_submitter_max_worker_threads = 2; - @ConfField(mutable = true, masterOnly = true, description = { - "The threshold of load labels' number. After this number is exceeded, " - + "the labels of the completed import jobs or tasks will be deleted, " - + "and the deleted labels can be reused. " - + "When the value is -1, it indicates no threshold."}) + @ConfField(mutable = true, masterOnly = true, description = "The threshold of load labels' number. After this " + + "number is exceeded, the labels of the completed " + "import jobs or tasks will be deleted, and the " + + "deleted labels can be reused. When the value is -1, " + "it indicates no threshold.") public static int label_num_threshold = 2000; - @ConfField(description = {"Specify the default authentication class of internal catalog"}, + @ConfField(description = "Specify the default authentication class of internal catalog", options = {"default", "ranger-doris"}) public static String access_controller_type = "default"; @@ -2882,46 +2780,43 @@ public class Config extends ConfigBase { @ConfField public static boolean ignore_bdbje_log_checksum_read = false; - @ConfField(description = { - "Specifies the primary MySQL authenticator name, either a built-in authenticator " - + "or an authentication plugin name"}, + @ConfField(description = "Specifies the primary MySQL authenticator name, either a built-in authenticator or an " + + "authentication plugin name", options = {"default", "password", "ldap", ""}) public static String authentication_type = "default"; - @ConfField(mutable = true, description = { - "Specifies the authentication chain used after primary authentication failure, " - + "multiple integration names are comma-separated"}) + @ConfField(mutable = true, description = "Specifies the authentication chain used after primary authentication " + + "failure, multiple integration names are comma-separated") public static String authentication_chain = ""; // The dir the trino-connector catalog loads Trino's own plugins from, used verbatim. Keep the // default in sync with BE config trino_connector_plugin_dir: FE and BE load the same plugins and // an operator who leaves both untouched expects both to find them. - @ConfField(mutable = true, masterOnly = false, description = { - "Specify the default plugins loading path for the trino-connector catalog"}) + @ConfField(mutable = true, masterOnly = false, description = "Specify the default plugins loading path for the " + + "trino-connector catalog") public static String trino_connector_plugin_dir = EnvUtils.getDorisHome() + "/plugins/trino_plugins"; @ConfField(mutable = true) public static boolean fix_tablet_partition_id_eq_0 = false; - @ConfField(mutable = true, masterOnly = true, description = { - "Default storage format of inverted index, the default value is V3."}) + @ConfField(mutable = true, masterOnly = true, description = "Default storage format of inverted index, the " + + "default value is V3.") public static String inverted_index_storage_format = "V3"; - @ConfField(mutable = true, masterOnly = true, description = { - "Enable the 'delete predicate' for DELETE statements. If enabled, it will enhance the performance of " - + "DELETE statements, but partial column updates after a DELETE may result in erroneous data. " - + "If disabled, it will reduce the performance of DELETE statements to ensure accuracy."}) + @ConfField(mutable = true, masterOnly = true, description = "Enable the 'delete predicate' for DELETE statements. " + + "If enabled, it will enhance the performance of " + "DELETE statements, but partial column updates after " + + "a DELETE may result in erroneous data. If disabled, " + + "it will reduce the performance of DELETE statements " + "to ensure accuracy.") public static boolean enable_mow_light_delete = false; - @ConfField(description = {"Whether to enable proxy protocol"}) + @ConfField(description = "Whether to enable proxy protocol") public static boolean enable_proxy_protocol = false; - @ConfField(description = { - "Profile async collect expire time. After the query is completed, if the profile is not collected within " - + "the time specified by this parameter, the uncompleted profile will be abandoned."}) + @ConfField(description = "Profile async collect expire time. After the query is completed, if the profile is not " + + "collected within the time specified by this parameter, the uncompleted profile will be " + "abandoned.") public static int profile_async_collect_expire_time_secs = 5; - @ConfField(description = {"Used to control the interval time of ProfileManager for profile garbage collection."}) + @ConfField(description = "Used to control the interval time of ProfileManager for profile garbage collection.") public static int profile_manager_gc_interval_seconds = 1; // Used to check compatibility when upgrading. @ConfField @@ -2932,17 +2827,17 @@ public class Config extends ConfigBase { public static boolean checkpoint_after_check_compatibility = false; // Advance the next id before transferring to the master. - @ConfField(description = {"Whether to advance the ID generator after becoming Master to ensure that the id " - + "generator will not be rolled back even when metadata is rolled back."}) + @ConfField(description = "Whether to advance the ID generator after becoming Master to ensure that the id " + + "generator will not be rolled back even when metadata is rolled back.") public static boolean enable_advance_next_id = true; // The count threshold to do manual GC when doing checkpoint but not enough memory. // Set zero to disable it. - @ConfField(description = {"The threshold to do manual GC when doing checkpoint but not enough memory"}) + @ConfField(description = "The threshold to do manual GC when doing checkpoint but not enough memory") public static int checkpoint_manual_gc_threshold = 0; - @ConfField(mutable = true, description = { - "Whether to log the request content before each request starts, specifically the query statements."}) + @ConfField(mutable = true, description = "Whether to log the request content before each request starts, " + + "specifically the query statements.") public static boolean enable_print_request_before_execution = false; @ConfField @@ -2960,54 +2855,51 @@ public class Config extends ConfigBase { public static long spilled_profile_storage_limit_bytes = 1 * 1024 * 1024 * 1024; // 1GB // Profile will be spilled to storage after query has finished for this time. - @ConfField(mutable = true, description = { - "Profile will be spilled to storage after the query has been finished for this duration."}) + @ConfField(mutable = true, description = "Profile will be spilled to storage after the query has been finished " + + "for this duration.") public static int profile_waiting_time_for_spill_seconds = 10; // Enable profile archive feature. When enabled, profiles exceeding storage limits // will be archived to compressed ZIP files instead of being directly deleted. - @ConfField(mutable = true, description = { - "Enable profile archive feature. When enabled, profiles exceeding storage limits " - + "will be archived to compressed ZIP files instead of being directly deleted."}) + @ConfField(mutable = true, description = "Enable profile archive feature. When enabled, profiles exceeding " + + "storage limits will be archived to compressed ZIP files instead of " + "being directly deleted.") public static boolean enable_profile_archive = true; // Number of profiles to include in each archive ZIP file. // Recommended value: 1000 - @ConfField(mutable = true, description = {"Number of profiles per archive ZIP file. Recommended: 1000"}) + @ConfField(mutable = true, description = "Number of profiles per archive ZIP file. Recommended: 1000") public static int profile_archive_batch_size = 1000; // Storage path for archived profiles. // If empty, defaults to ${spilled_profile_storage_path}/archive - @ConfField(description = { - "Storage path for archived profiles. Defaults to ${spilled_profile_storage_path}/archive if empty."}) + @ConfField(description = "Storage path for archived profiles. Defaults to ${spilled_profile_storage_path}/archive " + + "if empty.") public static String profile_archive_path = ""; // Retention period for archive files in seconds. // -1: keep forever // 0: disable archiving (equivalent to enable_profile_archive = false) // >0: delete archives older than specified seconds (e.g., 604800 = 30 days) - @ConfField(mutable = true, description = { - "Retention period for archive files in seconds. -1 for unlimited, 0 to disable archiving."}) + @ConfField(mutable = true, description = "Retention period for archive files in seconds. -1 for unlimited, 0 to " + + "disable archiving.") public static int profile_archive_retention_seconds = 28800; // 8 hours // Maximum waiting time for pending archive files in seconds. // If the oldest file in pending directory exceeds this time, archive will be forced // even if the batch size is not reached. - @ConfField(mutable = true, description = {"Maximum waiting time for pending archive files in seconds. " - + "Forces archive even if the batch is not full."}) + @ConfField(mutable = true, description = "Maximum waiting time for pending archive files in seconds. Forces " + + "archive even if the batch is not full.") public static int profile_archive_pending_timeout_seconds = 3600; // 1 hours - @ConfField(mutable = true, description = {"Whether to abort transactions by checking coordinator BE heartbeat."}) + @ConfField(mutable = true, description = "Whether to abort transactions by checking coordinator BE heartbeat.") public static boolean enable_abort_txn_by_checking_coordinator_be = true; - @ConfField(mutable = true, description = { - "Whether to abort transactions by checking conflict transactions in schema change " - + "or cloud upgrade checks."}) + @ConfField(mutable = true, description = "Whether to abort transactions by checking conflict transactions in " + + "schema change or cloud upgrade checks.") public static boolean enable_abort_txn_by_checking_conflict_txn = true; - @ConfField(mutable = true, description = { - "Columns that have not been collected within the specified interval will trigger automatic analyze. " - + "0 means not trigger."}) + @ConfField(mutable = true, description = "Columns that have not been collected within the specified interval will " + + "trigger automatic analyze. 0 means not trigger.") public static long auto_analyze_interval_seconds = 86400; // 24 hours. // A internal config to control whether to enable the checkpoint. @@ -3016,32 +2908,32 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static boolean enable_checkpoint = true; - @ConfField(description = {"The default directory for storing hadoop conf configuration files."}) + @ConfField(description = "The default directory for storing hadoop conf configuration files.") public static String hadoop_config_dir = EnvUtils.getDorisHome() + "/plugins/hadoop_conf/"; - @ConfField(mutable = true, masterOnly = true, description = {"Timeout for dictionary-related RPCs."}) + @ConfField(mutable = true, masterOnly = true, description = "Timeout for dictionary-related RPCs.") public static int dictionary_rpc_timeout_seconds = 5; - @ConfField(mutable = true, masterOnly = true, description = { - "Interval at which the dictionary triggers a data expiration check, in seconds."}) + @ConfField(mutable = true, masterOnly = true, description = "Interval at which the dictionary triggers a data " + + "expiration check, in seconds.") public static int dictionary_auto_refresh_interval_seconds = 5; - @ConfField(mutable = false, masterOnly = false, description = { - "Whether to enable the experimental Table Stream functionality" }, + @ConfField(mutable = false, masterOnly = false, description = "Whether to enable the experimental Table Stream " + + "functionality", varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_table_stream = false; - @ConfField(mutable = true, masterOnly = true, description = { - "The interval at which FE cleans stale partition offset state from table streams, in seconds."}, + @ConfField(mutable = true, masterOnly = true, description = "The interval at which FE cleans stale partition " + + "offset state from table streams, in seconds.", varType = VariableAnnotation.EXPERIMENTAL) public static int table_stream_partition_offset_cleanup_interval_second = 3600; //========================================================================== // begin of cloud config //========================================================================== - @ConfField(description = {"Whether to enable the FE log file deletion policy based on size, " - + "where logs exceeding the specified size are deleted. " - + "It is disabled by default and follows a time-based deletion policy."}, + @ConfField(description = "Whether to enable the FE log file deletion policy based on size, where logs exceeding " + + "the specified size are deleted. It is disabled by default and follows a time-based " + + "deletion policy.", options = {"age", "size"}) public static String log_rollover_strategy = "age"; @@ -3203,66 +3095,59 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, masterOnly = true) public static double cloud_balance_tablet_percent_per_run = 0.05; - @ConfField(mutable = true, masterOnly = true, description = { - "Specify the scaling and warming methods for all compute groups in cloud mode. " - + "without_warmup: Directly modify shard mapping, first read from S3, " - + "fastest rebalance but largest fluctuation; " - + "async_warmup: Asynchronous warmup, best-effort cache pulling, " - + "faster rebalance but possible cache miss; " - + "sync_warmup: Synchronous warmup, ensure cache migration completion, " - + "slower rebalance but no cache miss; " - + "peer_read_async_warmup: Directly modify shard mapping, first read from peer BE, " - + "fastest rebalance but may affect other BEs in the same compute group's performance. " - + "Note: This is a global FE configuration. " - + "You can also use SQL (ALTER COMPUTE GROUP cg PROPERTIES) " - + "to set balance type at compute group level. " - + "Compute group level configuration has higher priority."}, + @ConfField(mutable = true, masterOnly = true, description = "Specify the scaling and warming methods for all " + + "compute groups in cloud mode. without_warmup: " + "Directly modify shard mapping, first read from S3, " + + "fastest rebalance but largest fluctuation; " + "async_warmup: Asynchronous warmup, best-effort cache " + + "pulling, faster rebalance but possible cache miss; " + "sync_warmup: Synchronous warmup, ensure cache " + + "migration completion, slower rebalance but no cache " + + "miss; peer_read_async_warmup: Directly modify shard " + + "mapping, first read from peer BE, fastest rebalance " + + "but may affect other BEs in the same compute group's " + "performance. Note: This is a global FE " + + "configuration. You can also use SQL (ALTER COMPUTE " + + "GROUP cg PROPERTIES) to set balance type at compute " + + "group level. Compute group level configuration has " + "higher priority.", options = {"without_warmup", "async_warmup", "sync_warmup", "peer_read_async_warmup"}) public static String cloud_warm_up_for_rebalance_type = "async_warmup"; - @ConfField(mutable = true, masterOnly = true, description = {"The maximum number of tablets per host " - + "when batching warm-up requests during tablet rebalancing in " - + "compute-storage separation mode. Default is 10."}) + @ConfField(mutable = true, masterOnly = true, description = "The maximum number of tablets per host when batching " + + "warm-up requests during tablet rebalancing in " + "compute-storage separation mode. Default is 10.") public static int cloud_warm_up_batch_size = 10; - @ConfField(mutable = true, masterOnly = true, description = {"Maximum wait time in milliseconds before a " - + "pending warm-up batch is flushed. Default is 50ms."}) + @ConfField(mutable = true, masterOnly = true, description = "Maximum wait time in milliseconds before a pending " + + "warm-up batch is flushed. Default is 50ms.") public static int cloud_warm_up_batch_flush_interval_ms = 50; - @ConfField(mutable = true, masterOnly = true, description = { - "Thread pool size for asynchronous warm-up RPC dispatch during tablet " - + "rebalancing in compute-storage separation mode. Default is 4."}) + @ConfField(mutable = true, masterOnly = true, description = "Thread pool size for asynchronous warm-up RPC " + + "dispatch during tablet rebalancing in " + "compute-storage separation mode. Default is 4.") public static int cloud_warm_up_rpc_async_pool_size = 4; - @ConfField(masterOnly = true, description = {"When tablets are being balanced in compute-storage separation mode, " - + "whether to enable the active tablet priority scheduling strategy. Default is true."}) + @ConfField(masterOnly = true, description = "When tablets are being balanced in compute-storage separation mode, " + + "whether to enable the active tablet priority scheduling strategy. " + "Default is true.") public static boolean enable_cloud_active_tablet_priority_scheduling = true; - @ConfField(masterOnly = true, description = { - "Whether to enable active tablet sliding window access statistics feature. Default is true."}) + @ConfField(masterOnly = true, description = "Whether to enable active tablet sliding window access statistics " + + "feature. Default is true.") public static boolean enable_active_tablet_sliding_window_access_stats = true; - @ConfField(mutable = true, masterOnly = true, description = { - "Time window size in seconds for active tablet sliding window access statistics. " - + "Default is 3600 seconds (1 hour)."}) + @ConfField(mutable = true, masterOnly = true, description = "Time window size in seconds for active tablet " + + "sliding window access statistics. Default is 3600 " + "seconds (1 hour).") public static long active_tablet_sliding_window_time_window_second = 3600L; - @ConfField(mutable = true, masterOnly = true, description = { - "When active tablet priority scheduling is enabled: partition-level scheduling processes TopN active " - + "partitions first, then other active partitions, " - + "then inactive partitions, and internal databases last. " - + "Default is 10000. <=0 disables TopN segmentation."}) + @ConfField(mutable = true, masterOnly = true, description = "When active tablet priority scheduling is enabled: " + + "partition-level scheduling processes TopN active " + + "partitions first, then other active partitions, then " + + "inactive partitions, and internal databases last. " + + "Default is 10000. <=0 disables TopN segmentation.") public static int cloud_active_partition_scheduling_topn = 10000; - @ConfField(mutable = true, masterOnly = true, description = { - "Refresh interval in seconds for the active-tablet snapshot when active priority scheduling is enabled. " - + "Default 60 seconds. Reuses the same active-tablet set within the interval."}) + @ConfField(mutable = true, masterOnly = true, description = "Refresh interval in seconds for the active-tablet " + + "snapshot when active priority scheduling is enabled. " + + "Default 60 seconds. Reuses the same active-tablet " + "set within the interval.") public static long cloud_active_tablet_ids_refresh_interval_second = 60L; - @ConfField(mutable = true, masterOnly = true, description = { - "When active priority scheduling is enabled and the active phase remains unbalanced for N consecutive " - + "rounds, force one inactive phase round to avoid long-term starvation. " - + "Default 10. <=0 disables this forced mechanism."}) + @ConfField(mutable = true, masterOnly = true, description = "When active priority scheduling is enabled and the " + + "active phase remains unbalanced for N consecutive " + "rounds, force one inactive phase round to avoid " + + "long-term starvation. Default 10. <=0 disables this " + "forced mechanism.") public static int cloud_active_unbalanced_force_inactive_after_rounds = 10; @ConfField(mutable = true, masterOnly = false) @@ -3271,36 +3156,34 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true) public static int mow_calculate_delete_bitmap_retry_times = 10; - @ConfField(description = { - "The allowlist for S3 load endpoints. If it is empty, no allowlist will be set. " - + "For example: s3_load_endpoint_white_list=a,b,c. " - + "This can only be set in fe.conf and takes effect after a restart; " - + "it is intentionally not modifiable at runtime via ADMIN SET FRONTEND CONFIG."}) + @ConfField(description = "The allowlist for S3 load endpoints. If it is empty, no allowlist will be set. For " + + "example: s3_load_endpoint_white_list=a,b,c. This can only be set in fe.conf and takes " + + "effect after a restart; it is intentionally not modifiable at runtime via ADMIN SET " + + "FRONTEND CONFIG.") public static String[] s3_load_endpoint_white_list = {}; - @ConfField(description = { - "The default scheme for S3 endpoints without an explicit scheme. Valid values are http and https."}, + @ConfField(description = "The default scheme for S3 endpoints without an explicit scheme. Valid values are http " + + "and https.", options = {"http", "https"}) public static String s3_client_http_scheme = "https"; - @ConfField(mutable = true, description = { - "For deterministic S3 paths (without wildcards like *, ?), use HEAD requests instead of " - + "ListObjects to avoid requiring ListBucket permission. Brace patterns {1,2,3} and " - + "non-negated bracket patterns [abc] are expanded to concrete paths. This is useful when only " - + "GetObject permission is granted. Set to false to fall back to the original listing behavior."}) + @ConfField(mutable = true, description = "For deterministic S3 paths (without wildcards like *, ?), use HEAD " + + "requests instead of ListObjects to avoid requiring ListBucket " + + "permission. Brace patterns {1,2,3} and non-negated bracket patterns " + + "[abc] are expanded to concrete paths. This is useful when only " + + "GetObject permission is granted. Set to false to fall back to the " + "original listing behavior.") public static boolean s3_skip_list_for_deterministic_path = true; - @ConfField(mutable = true, description = { - "Maximum number of expanded paths when using HEAD requests instead of ListObjects. " - + "If the expanded path count exceeds this limit, falls back to ListObjects. " - + "This prevents patterns like {1..100}/{1..100} from triggering too many HEAD requests."}) + @ConfField(mutable = true, description = "Maximum number of expanded paths when using HEAD requests instead of " + + "ListObjects. If the expanded path count exceeds this limit, falls back " + + "to ListObjects. This prevents patterns like {1..100}/{1..100} from " + + "triggering too many HEAD requests.") public static int s3_head_request_max_paths = 100; - @ConfField(mutable = true, description = { - "The host suffix whitelist for Azure endpoints (both blob and dfs), separated by commas. " - + "The default value is .blob.core.windows.net,.dfs.core.windows.net," - + ".blob.core.chinacloudapi.cn,.dfs.core.chinacloudapi.cn," - + ".blob.core.usgovcloudapi.net,.dfs.core.usgovcloudapi.net," - + ".blob.core.cloudapi.de,.dfs.core.cloudapi.de."}) + @ConfField(mutable = true, description = "The host suffix whitelist for Azure endpoints (both blob and dfs), " + + "separated by commas. The default value is " + + ".blob.core.windows.net,.dfs.core.windows.net,.blob.core.chinacloudapi.cn" + + ",.dfs.core.chinacloudapi.cn,.blob.core.usgovcloudapi.net,.dfs.core.usgov" + + "cloudapi.net,.blob.core.cloudapi.de,.dfs.core.cloudapi.de.") public static String[] azure_blob_host_suffixes = { ".blob.core.windows.net", ".dfs.core.windows.net", @@ -3312,14 +3195,13 @@ public static int metaServiceRpcRetryTimes() { ".dfs.core.cloudapi.de" }; - @ConfField(description = { - "The allowlist for JDBC driver URLs. If it is empty, no allowlist will be set. " - + "For example: jdbc_driver_url_white_list=a,b,c. " - + "This can only be set in fe.conf and takes effect after a restart; " - + "it is intentionally not modifiable at runtime via ADMIN SET FRONTEND CONFIG."}) + @ConfField(description = "The allowlist for JDBC driver URLs. If it is empty, no allowlist will be set. For " + + "example: jdbc_driver_url_white_list=a,b,c. This can only be set in fe.conf and takes " + + "effect after a restart; it is intentionally not modifiable at runtime via ADMIN SET " + + "FRONTEND CONFIG.") public static String[] jdbc_driver_url_white_list = {}; - @ConfField(description = {"The maximum length of label in Stream Load is limited."}) + @ConfField(description = "The maximum length of label in Stream Load is limited.") public static int label_regex_length = 128; @ConfField(mutable = true, masterOnly = true) @@ -3337,19 +3219,16 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, masterOnly = true) public static long cloud_warm_up_job_max_bytes_per_batch = 21474836480L; // 20GB - @ConfField(mutable = true, masterOnly = true, description = { - "zh-CN: 定期刷新 table-level warmup 任务匹配的 table ID 集合的时间间隔（毫秒）", - "en: Interval in milliseconds to refresh matched table IDs for table-level warmup jobs"}) + @ConfField(mutable = true, masterOnly = true, description = "en: Interval in milliseconds to refresh matched " + + "table IDs for table-level warmup jobs") public static long cloud_warm_up_table_filter_refresh_interval_ms = 60000; // 60 seconds - @ConfField(mutable = true, masterOnly = true, description = { - "zh-CN: 定期从 BE 拉取主动增量预热 SyncStats 并缓存到 FE job 的时间间隔（毫秒）", - "en: Interval in milliseconds to collect event-driven warmup SyncStats from BEs and cache it in FE jobs"}) + @ConfField(mutable = true, masterOnly = true, description = "en: Interval in milliseconds to collect event-driven " + + "warmup SyncStats from BEs and cache it in FE jobs") public static long cloud_warm_up_sync_stats_refresh_interval_ms = 15000; // 15 seconds - @ConfField(mutable = true, masterOnly = true, description = { - "zh-CN: SHOW WARM UP JOB 和 FE 日志中 MatchedTables 最多展示的表数量", - "en: Maximum number of MatchedTables entries displayed in SHOW WARM UP JOB and FE logs"}) + @ConfField(mutable = true, masterOnly = true, description = "en: Maximum number of MatchedTables entries " + + "displayed in SHOW WARM UP JOB and FE logs") public static int cloud_warm_up_matched_tables_display_limit = 100; @ConfField(mutable = true, masterOnly = true) @@ -3378,177 +3257,158 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true) public static int audit_event_log_queue_size = 250000; - @ConfField(description = {"Maximum size of the lineage event queue. Events will be discarded when exceeded."}) + @ConfField(description = "Maximum size of the lineage event queue. Events will be discarded when exceeded.") public static int lineage_event_queue_size = 50000; - @ConfField(mutable = true, description = {"Stream load route policy. Available options are " - + "public-private/public/private/direct/random-be and empty string."}) + @ConfField(mutable = true, description = "Stream load route policy. Available options are " + + "public-private/public/private/direct/random-be and empty string.") public static String streamload_redirect_policy = ""; - @ConfField(mutable = true, description = { - "Stream Load redirect 场景下，FE 在返回 307 后额外丢弃请求体的最大字节数。" - + "0 表示关闭该兼容逻辑，正数表示最大丢弃字节数。", - "The maximum number of request body bytes FE drains after returning 307 for Stream Load redirects. " - + "0 disables the compatibility logic, and a positive value sets the byte limit."}) + @ConfField(mutable = true, description = "The maximum number of request body bytes FE drains after returning 307 " + + "for Stream Load redirects. 0 disables the compatibility logic, and a " + + "positive value sets the byte limit.") // Enable a generous bounded drain window by default to preserve FE redirect compatibility on Jetty 12. public static long stream_load_redirect_bounded_drain_max_bytes = 1024L * 1024 * 1024; - @ConfField(mutable = true, description = { - "Stream Load redirect 场景下，FE 在检测到请求体暂时无可读数据后继续等待的最大空闲时长，单位毫秒。" - + "0 表示不额外等待，用于给慢客户端或分段到达的数据保留一个有限的缓冲窗口。", - "The maximum idle wait time in milliseconds after FE detects no readable request body bytes " - + "during Stream Load redirect drain. 0 disables the extra idle wait, while a positive value " - + "keeps a bounded grace window for slow clients or delayed request body chunks."}) + @ConfField(mutable = true, description = "The maximum idle wait time in milliseconds after FE detects no readable " + + "request body bytes during Stream Load redirect drain. 0 disables the " + + "extra idle wait, while a positive value keeps a bounded grace window " + + "for slow clients or delayed request body chunks.") // Keep a small grace period for delayed body chunks after FE has already written the redirect. public static int stream_load_redirect_bounded_drain_max_idle_time_ms = 1000; - @ConfField(mutable = true, description = { - "Whether to enable group commit streamload BE forward feature in cloud mode. " - + "Solves the issue where LB random forwarding breaks group commit batching " - + "by implementing BE-level forwarding to ensure same-table requests reach the same BE node."}) + @ConfField(mutable = true, description = "Whether to enable group commit streamload BE forward feature in cloud " + + "mode. Solves the issue where LB random forwarding breaks group commit " + + "batching by implementing BE-level forwarding to ensure same-table " + "requests reach the same BE node.") public static boolean enable_group_commit_streamload_be_forward = false; - @ConfField(description = {"When creating a table in cloud mode, check if recycler keys remain. Default is true."}) + @ConfField(description = "When creating a table in cloud mode, check if recycler keys remain. Default is true.") public static boolean check_create_table_recycle_key_remained = true; - @ConfField(mutable = true, description = { - "Lock expiration time for FE requesting a lock from meta service in cloud mode. Default is 60s."}) + @ConfField(mutable = true, description = "Lock expiration time for FE requesting a lock from meta service in " + + "cloud mode. Default is 60s.") public static int delete_bitmap_lock_expiration_seconds = 60; - @ConfField(mutable = true, description = { - "Timeout for calculate delete bitmap task in cloud mode. Default is 60s."}) + @ConfField(mutable = true, description = "Timeout for calculate delete bitmap task in cloud mode. Default is 60s.") public static int calculate_delete_bitmap_task_timeout_seconds = 60; - @ConfField(mutable = true, description = { - "Timeout for calculate delete bitmap task during transaction load in cloud mode. Default is 300s."}) + @ConfField(mutable = true, description = "Timeout for calculate delete bitmap task during transaction load in " + + "cloud mode. Default is 300s.") public static int calculate_delete_bitmap_task_timeout_seconds_for_transaction_load = 300; - @ConfField(mutable = true, description = {"Lock wait timeout during commit phase in cloud mode. Default is 5s."}) + @ConfField(mutable = true, description = "Lock wait timeout during commit phase in cloud mode. Default is 5s.") public static int try_commit_lock_timeout_seconds = 5; - @ConfField(mutable = true, description = {"Whether to enable commit lock for all tables during transaction commit. " - + "If true, commit lock will be applied to all tables. " - + "If false, commit lock will only be applied to Merge-On-Write tables. " - + "Default value is true."}) + @ConfField(mutable = true, description = "Whether to enable commit lock for all tables during transaction commit. " + + "If true, commit lock will be applied to all tables. If false, commit " + + "lock will only be applied to Merge-On-Write tables. Default value is " + "true.") public static boolean enable_commit_lock_for_all_tables = true; - @ConfField(mutable = true, description = { - "Whether to enable lazy commit for large transactions in cloud mode. Default is true."}) + @ConfField(mutable = true, description = "Whether to enable lazy commit for large transactions in cloud mode. " + + "Default is true.") public static boolean enable_cloud_txn_lazy_commit = true; @ConfField(mutable = true, masterOnly = true, - description = { - "Whether to immediately reassign tablets to a new BE when the assigned BE is abnormal " - + "in cloud mode. Default is false."}) + description = "Whether to immediately reassign tablets to a new BE when the assigned BE is abnormal in " + + "cloud mode. Default is false.") public static boolean enable_immediate_be_assign = false; @ConfField(mutable = true, masterOnly = false, - description = { - "Time in seconds after a BE goes down before its tablets are permanently reassigned " - + "to other BEs in cloud mode."}) + description = "Time in seconds after a BE goes down before its tablets are permanently reassigned to " + + "other BEs in cloud mode.") public static int rehash_tablet_after_be_dead_seconds = 3600; @ConfField(mutable = false, masterOnly = true, - description = { - "Whether to use rendezvous hashing for colocate bucket placement in cloud mode. " - + "If false, use the legacy modulo placement. Restart-only."}) + description = "Whether to use rendezvous hashing for colocate bucket placement in cloud mode. If false, " + + "use the legacy modulo placement. Restart-only.") public static boolean enable_cloud_colocate_consistent_hash = true; - @ConfField(mutable = true, description = { - "Whether to enable the automatic start-stop feature in cloud model, default is true."}) + @ConfField(mutable = true, description = "Whether to enable the automatic start-stop feature in cloud model, " + + "default is true.") public static boolean enable_auto_start_for_cloud_cluster = true; - @ConfField(mutable = true, description = { - "The automatic start-stop wait time for cluster wake-up backoff retry count in the cloud " - + "model is set to 300 times, which is approximately 5 minutes by default."}) + @ConfField(mutable = true, description = "The automatic start-stop wait time for cluster wake-up backoff retry " + + "count in the cloud model is set to 300 times, which is approximately 5 " + "minutes by default.") public static int auto_start_wait_to_resume_times = 300; - @ConfField(description = { - "Maximal concurrent num of master FE sync tablet stats task to observers and followers in cloud mode."}) + @ConfField(description = "Maximal concurrent num of master FE sync tablet stats task to observers and followers " + + "in cloud mode.") public static int cloud_sync_tablet_stats_task_threads_num = 4; - @ConfField(mutable = true, description = {"Version of getting tablet stats in cloud mode. " - + "Version 1: get all tablets; Version 2: get active and interval expired tablets"}) + @ConfField(mutable = true, description = "Version of getting tablet stats in cloud mode. Version 1: get all " + + "tablets; Version 2: get active and interval expired tablets") public static int cloud_get_tablet_stats_version = 2; - @ConfField(description = {"Maximum concurrent number of get tablet stat jobs."}) + @ConfField(description = "Maximum concurrent number of get tablet stat jobs.") public static int max_get_tablet_stat_task_threads_num = 4; - @ConfField(description = { - "Cloud table and partition version syncer interval. All frontends will perform the checking."}) + @ConfField(description = "Cloud table and partition version syncer interval. All frontends will perform the " + + "checking.") public static int cloud_version_syncer_interval_second = 20; - @ConfField(mutable = true, description = { - "Whether to enable the function of syncing table and partition version in cloud mode."}) + @ConfField(mutable = true, description = "Whether to enable the function of syncing table and partition version " + + "in cloud mode.") public static boolean cloud_enable_version_syncer = true; - @ConfField(description = {"Concurrent number of get version tasks."}) + @ConfField(description = "Concurrent number of get version tasks.") public static int cloud_get_version_task_threads_num = 4; - @ConfField(description = {"Maximum concurrent number of sync version tasks between Master FE and other FEs."}) + @ConfField(description = "Maximum concurrent number of sync version tasks between Master FE and other FEs.") public static int cloud_sync_version_task_threads_num = 4; - @ConfField(mutable = true, description = {"Maximum table or partition batch size for get version tasks."}) + @ConfField(mutable = true, description = "Maximum table or partition batch size for get version tasks.") public static int cloud_get_version_task_batch_size = 2000; - @ConfField(mutable = true, description = { - "Whether to enable retry when a schema change job fails, default is true."}) + @ConfField(mutable = true, description = "Whether to enable retry when a schema change job fails, default is true.") public static boolean enable_schema_change_retry = true; - @ConfField(mutable = true, description = {"Max retry times when a schema change job fails, default is 3."}) + @ConfField(mutable = true, description = "Max retry times when a schema change job fails, default is 3.") public static int schema_change_max_retry_time = 3; - @ConfField(mutable = true, description = {"Whether to enable the use of ShowCacheHotSpotStmt, default is false."}) + @ConfField(mutable = true, description = "Whether to enable the use of ShowCacheHotSpotStmt, default is false.") public static boolean enable_show_file_cache_hotspot_stmt = false; - @ConfField(mutable = true, description = { - "Request timeout for FE connecting to meta service in cloud mode, default is 30000ms."}) + @ConfField(mutable = true, description = "Request timeout for FE connecting to meta service in cloud mode, " + + "default is 30000ms.") public static int meta_service_brpc_timeout_ms = 30000; - @ConfField(mutable = true, description = { - "Connection timeout for FE connecting to meta service in cloud mode. Default is 500ms."}) + @ConfField(mutable = true, description = "Connection timeout for FE connecting to meta service in cloud mode. " + + "Default is 500ms.") public static int meta_service_brpc_connect_timeout_ms = 500; - @ConfField(mutable = true, description = { - "In cloud mode, the retry count when the FE request to meta service times out. Default is 1."}) + @ConfField(mutable = true, description = "In cloud mode, the retry count when the FE request to meta service " + + "times out. Default is 1.") public static int meta_service_rpc_timeout_retry_times = 1; - @ConfField(mutable = true, description = { - "Whether to enable QPS rate limit for RPC requests to meta service."}) + @ConfField(mutable = true, description = "Whether to enable QPS rate limit for RPC requests to meta service.") public static boolean meta_service_rpc_rate_limit_enabled = false; - @ConfField(mutable = true, description = { - "Default QPS limit for each method (requests per second) in each cpu core, " - + "non-positive value (<= 0) means no limit"}) + @ConfField(mutable = true, description = "Default QPS limit for each method (requests per second) in each cpu " + + "core, non-positive value (<= 0) means no limit") public static int meta_service_rpc_rate_limit_default_qps_per_core = 50; @ConfField(mutable = true, callback = MetaServiceRpcRateLimitConfigValidator.QpsConfigHandler.class, - description = { - "QPS limit config per rpc method to meta service in per cpu core, " - + "format: method1:qps1;method2:qps2, " - + "e.g.: getPartitionVersion:100;getTableVersion:100;getTabletStats:50, " - + "non-positive value (<= 0) means no limit"}) + description = "QPS limit config per rpc method to meta service in per cpu core, format: " + + "method1:qps1;method2:qps2, e.g.: " + + "getPartitionVersion:100;getTableVersion:100;getTabletStats:50, non-positive value (<= 0) " + + "means no limit") public static String meta_service_rpc_rate_limit_qps_per_core_config = "getPartitionVersion:500;getTableVersion:500;getTabletStats:50;beginTxn:50"; @ConfField(mutable = true, callback = MetaServiceRpcRateLimitConfigValidator.PositiveIntConfigHandler.class, - description = { - "Burst window for meta service RPC rate limit in seconds. " - + "The long-term average QPS is unchanged, while calls can burst within this window."}) + description = "Burst window for meta service RPC rate limit in seconds. The long-term average QPS is " + + "unchanged, while calls can burst within this window.") public static int meta_service_rpc_rate_limit_burst_seconds = 2; @ConfField(mutable = true, callback = MetaServiceRpcRateLimitConfigValidator.NonNegativeLongConfigHandler.class, - description = { - "Max wait time in milliseconds when meta service RPC is rate limited, " - + "zero means fail fast."}) + description = "Max wait time in milliseconds when meta service RPC is rate limited, zero means fail fast.") public static long meta_service_rpc_rate_limit_wait_timeout_ms = 1000; - @ConfField(mutable = true, description = { - "In cloud mode, the auto start and stop ignores the databases used by internal jobs, " - + "such as those used for statistics. " - + "For example: auto_start_ignore_db_names=__internal_schema, information_schema"}) + @ConfField(mutable = true, description = "In cloud mode, the auto start and stop ignores the databases used by " + + "internal jobs, such as those used for statistics. For example: " + + "auto_start_ignore_db_names=__internal_schema, information_schema") public static String[] auto_start_ignore_resume_db_names = {"__internal_schema", "information_schema"}; @ConfField(mutable = true, masterOnly = true) @@ -3563,38 +3423,34 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, masterOnly = true) public static long mow_get_ms_lock_retry_backoff_interval = 80; - @ConfField(mutable = false, masterOnly = true, description = { - "TSO service update interval in milliseconds. Default is 50, which means the TSO service " - + "will perform timestamp update checks every 50 milliseconds."}) + @ConfField(mutable = false, masterOnly = true, description = "TSO service update interval in milliseconds. " + + "Default is 50, which means the TSO service will " + "perform timestamp update checks every 50 " + + "milliseconds.") public static int tso_service_update_interval_ms = 50; - @ConfField(mutable = true, masterOnly = true, description = { - "TSO service max retry count. Default is 3, which means the TSO service will retry 3 times " - + "to update the global timestamp."}) + @ConfField(mutable = true, masterOnly = true, description = "TSO service max retry count. Default is 3, which " + + "means the TSO service will retry 3 times to update " + "the global timestamp.") public static int tso_max_update_retry_count = 3; - @ConfField(mutable = true, masterOnly = true, description = { - "TSO get max retry count. Default is 10, which means the TSO service will retry 10 times " - + "to generate TSO."}) + @ConfField(mutable = true, masterOnly = true, description = "TSO get max retry count. Default is 10, which means " + + "the TSO service will retry 10 times to generate TSO.") public static int tso_max_get_retry_count = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "TSO service time window in milliseconds. Default is 5000, which means the TSO service " - + "will apply for a TSO time window of 5000ms from BDBJE once."}) + @ConfField(mutable = true, masterOnly = true, description = "TSO service time window in milliseconds. Default is " + + "5000, which means the TSO service will apply for a " + "TSO time window of 5000ms from BDBJE once.") public static int tso_service_window_duration_ms = 5000; - @ConfField(mutable = true, masterOnly = true, description = { - "Max tolerated clock backward threshold during TSO calibration in milliseconds. " - + "Exceeding this threshold will fail enabling TSO. Default is 30 minutes."}) + @ConfField(mutable = true, masterOnly = true, description = "Max tolerated clock backward threshold during TSO " + + "calibration in milliseconds. Exceeding this " + "threshold will fail enabling TSO. Default is 30 " + + "minutes.") public static long tso_clock_backward_startup_threshold_ms = 30L * 60 * 1000; - @ConfField(mutable = true, description = { - "TSO service time offset in milliseconds. Only for test. Default is 0, which means the TSO service " - + "timestamp offset is 0 milliseconds."}) + @ConfField(mutable = true, description = "TSO service time offset in milliseconds. Only for test. Default is 0, " + + "which means the TSO service timestamp offset is 0 milliseconds.") public static int tso_time_offset_debug_mode = 0; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to forward TSO 1ms when logical counter is nearly full. Default is true."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to forward TSO 1ms when logical counter is " + + "nearly full. Default is true.") public static boolean enable_tso_forward_when_counter_full = true; @ConfField(mutable = true, masterOnly = true) @@ -3608,119 +3464,111 @@ public static int metaServiceRpcRetryTimes() { //========================================================================== //========================================================================== // start of lock config - @ConfField(description = {"Whether to enable deadlock detection."}) + @ConfField(description = "Whether to enable deadlock detection.") public static boolean enable_deadlock_detection = true; - @ConfField(description = {"Deadlock detection interval time, in minutes."}) + @ConfField(description = "Deadlock detection interval time, in minutes.") public static long deadlock_detection_interval_minute = 5; - @ConfField(mutable = true, description = {"Maximum lock hold time. Logs a warning if exceeded."}) + @ConfField(mutable = true, description = "Maximum lock hold time. Logs a warning if exceeded.") public static long max_lock_hold_threshold_seconds = 10; - @ConfField(mutable = true, description = {"Whether metadata synchronization is enabled in safe mode."}) + @ConfField(mutable = true, description = "Whether metadata synchronization is enabled in safe mode.") public static boolean meta_helper_security_mode = false; - @ConfField(description = {"Interval for checking if a resource is ready."}) + @ConfField(description = "Interval for checking if a resource is ready.") public static long resource_not_ready_sleep_seconds = 5; - @ConfField(mutable = true, description = { - "When set to true, if a query cannot select a healthy replica, " - + "detailed information of all replicas of the tablet will be printed."}) + @ConfField(mutable = true, description = "When set to true, if a query cannot select a healthy replica, detailed " + + "information of all replicas of the tablet will be printed.") public static boolean sql_block_rule_ignore_admin = false; - @ConfField(description = {"Authentication plugin root directories. Use a comma-separated list to configure " - + "multiple roots."}) + @ConfField(description = "Authentication plugin root directories. Use a comma-separated list to configure " + + "multiple roots.") public static String authentication_plugins_dir = EnvUtils.getDorisHome() + "/plugins/authentication"; - @ConfField(description = {"Authorization plugin root directories. Use a comma-separated list to configure " - + "multiple roots."}) + @ConfField(description = "Authorization plugin root directories. Use a comma-separated list to configure multiple " + + "roots.") public static String authorization_plugins_dir = EnvUtils.getDorisHome() + "/plugins/authorization"; - @ConfField(description = {"Security plugin directory."}) + @ConfField(description = "Security plugin directory.") public static String security_plugins_dir = EnvUtils.getDorisHome() + "/plugins/security"; - @ConfField(description = {"Directory containing filesystem provider plugin subdirectories. " - + "Each subdirectory is one storage backend (e.g., s3/, hdfs/, azure/). " - + "If empty, only classpath-based built-in providers are used (test/dev mode)."}) + @ConfField(description = "Directory containing filesystem provider plugin subdirectories. Each subdirectory is " + + "one storage backend (e.g., s3/, hdfs/, azure/). If empty, only classpath-based built-in " + + "providers are used (test/dev mode).") public static String filesystem_plugin_root = EnvUtils.getDorisHome() + "/plugins/filesystem"; - @ConfField(description = {"Directory containing connector provider plugin subdirectories. " - + "Each subdirectory is one connector (e.g., es/, jdbc/, iceberg/). " - + "If empty, only classpath-based built-in providers are used (test/dev mode)."}) + @ConfField(description = "Directory containing connector provider plugin subdirectories. Each subdirectory is one " + + "connector (e.g., es/, jdbc/, iceberg/). If empty, only classpath-based built-in " + + "providers are used (test/dev mode).") public static String connector_plugin_root = EnvUtils.getDorisHome() + "/plugins/connector"; - @ConfField(description = {"Authorization plugin configuration file path. Must be under DORIS_HOME. " - + "Default is conf/authorization.conf."}) + @ConfField(description = "Authorization plugin configuration file path. Must be under DORIS_HOME. Default is " + + "conf/authorization.conf.") public static String authorization_config_file_path = "/conf/authorization.conf"; - @ConfField(description = {"Authentication plugin configuration file path. Must be under DORIS_HOME. " - + "Default is conf/authentication.conf."}) + @ConfField(description = "Authentication plugin configuration file path. Must be under DORIS_HOME. Default is " + + "conf/authentication.conf.") public static String authentication_config_file_path = "/conf/authentication.conf"; - @ConfField(description = {"For testing purposes, all queries are forcibly forwarded to the master to verify " - + "the behavior of forwarding queries."}) + @ConfField(description = "For testing purposes, all queries are forcibly forwarded to the master to verify the " + + "behavior of forwarding queries.") public static boolean force_forward_all_queries = false; - @ConfField(description = { - "For disabling certain SQL queries, the configuration item is a list of simple class names of AST " - + "(for example CreateRepositoryStmt, CreatePolicyCommand), separated by commas."}) + @ConfField(description = "For disabling certain SQL queries, the configuration item is a list of simple class " + + "names of AST (for example CreateRepositoryStmt, CreatePolicyCommand), separated by " + "commas.") public static String block_sql_ast_names = ""; public static long meta_service_rpc_reconnect_interval_ms = 100; public static long meta_service_rpc_retry_cnt = 10; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to allow the use of inverted index v1 for variant."}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to allow the use of inverted index v1 for " + + "variant.") public static boolean enable_inverted_index_v1_for_variant = false; - @ConfField(mutable = true, description = {"Prometheus output table dimension metric count limit."}) + @ConfField(mutable = true, description = "Prometheus output table dimension metric count limit.") public static int prom_output_table_metrics_limit = 10000; @ConfField(mutable = true, masterOnly = true) public static long create_partition_wait_seconds = 300; - @ConfField(mutable = true, description = { - "The ID of the master key in KMS, used for generating and encrypting data keys"}) + @ConfField(mutable = true, description = "The ID of the master key in KMS, used for generating and encrypting " + + "data keys") public static String doris_tde_key_id = ""; - @ConfField(mutable = true, description = {"The endpoint of the KMS service, should match the region of the key"}) + @ConfField(mutable = true, description = "The endpoint of the KMS service, should match the region of the key") public static String doris_tde_key_endpoint = ""; - @ConfField(mutable = true, description = {"The region where the KMS key is located, used for SDK configuration"}) + @ConfField(mutable = true, description = "The region where the KMS key is located, used for SDK configuration") public static String doris_tde_key_region = ""; - @ConfField(mutable = true, description = { - "The key provider for TDE (Transparent Data Encryption), currently supports aws_kms"}) + @ConfField(mutable = true, description = "The key provider for TDE (Transparent Data Encryption), currently " + + "supports aws_kms") public static String doris_tde_key_provider = ""; - @ConfField(mutable = true, description = { - "The encryption algorithm used for data. Default is AES256; may be set to empty later for KMS to decide."}) + @ConfField(mutable = true, description = "The encryption algorithm used for data. Default is AES256; may be set " + + "to empty later for KMS to decide.") public static String doris_tde_algorithm = "PLAINTEXT"; - @ConfField(mutable = true, description = { - "The time interval for automatic rotation of the master key in data encryption, in milliseconds." - + "The default interval is one month."}) + @ConfField(mutable = true, description = "The time interval for automatic rotation of the master key in data " + + "encryption, in milliseconds.The default interval is one month.") public static long doris_tde_rotate_master_key_interval_ms = 30 * 24 * 3600 * 1000L; - @ConfField(mutable = true, description = { - "The interval at which data encryption checks whether to rotate the master key, in milliseconds. " - + "The default interval is five minutes."}) + @ConfField(mutable = true, description = "The interval at which data encryption checks whether to rotate the " + + "master key, in milliseconds. The default interval is five minutes.") public static long doris_tde_check_rotate_master_key_interval_ms = 5 * 60 * 1000L; - @ConfField(mutable = true, description = { - "The maximum length of the first row error message when data quality error occurs, default is 256 bytes"}) + @ConfField(mutable = true, description = "The maximum length of the first row error message when data quality " + + "error occurs, default is 256 bytes") public static int first_error_msg_max_length = 256; - @ConfField(mutable = false, description = { - "Whether to enable file cache admission control(Blocklist and Allowlist)" - }) + @ConfField(mutable = false, description = "Whether to enable file cache admission control(Blocklist and Allowlist)") public static boolean enable_file_cache_admission_control = false; - @ConfField(mutable = false, description = { - "Directory path for storing admission rules JSON files" - }) + @ConfField(mutable = false, description = "Directory path for storing admission rules JSON files") public static String file_cache_admission_control_json_dir = ""; @ConfField @@ -3734,10 +3582,9 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true) public static long cloud_auto_snapshot_min_interval_seconds = 3600; - @ConfField(mutable = true, description = {"The minimum privilege required for cluster snapshot operations. " - + "Valid values: 'root' (only root user can execute)" - + " or 'admin' (users with ADMIN privilege can execute). " - + "Default is 'root'."}) + @ConfField(mutable = true, description = "The minimum privilege required for cluster snapshot operations. Valid " + + "values: 'root' (only root user can execute) or 'admin' (users with " + + "ADMIN privilege can execute). Default is 'root'.") public static String cluster_snapshot_min_privilege = "root"; @ConfField(mutable = true) @@ -3750,46 +3597,38 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true) public static String aws_credentials_provider_version = "v2"; - @ConfField(mutable = true, description = { - "The soft upper limit of FILE_CACHE percent that a single query of a user can use (range: 1 to 100).", - "100 indicates that the full FILE_CACHE capacity can be used."}) + @ConfField(mutable = true, description = "The soft upper limit of FILE_CACHE percent that a single query of a " + + "user can use (range: 1 to 100). 100 indicates that the full FILE_CACHE " + "capacity can be used.") public static int file_cache_query_limit_max_percent = 100; - @ConfField(description = { - "The thread pool size used by the AWS SDK to schedule asynchronous retries, timeout tasks, " - + "and other background operations. Shared globally."}) + @ConfField(description = "The thread pool size used by the AWS SDK to schedule asynchronous retries, timeout " + + "tasks, and other background operations. Shared globally.") public static int aws_sdk_async_scheduler_thread_pool_size = 20; - @ConfField(description = { - "Agent tasks health check interval. Default is five minutes. " - + "No health check when less than or equal to 0."}) + @ConfField(description = "Agent tasks health check interval. Default is five minutes. No health check when less " + + "than or equal to 0.") public static long agent_task_health_check_intervals_ms = 5 * 60 * 1000L; // 5 min - @ConfField(description = { - "Whether to skip the FE internal catalog privilege check in catalog-level privilege validation. " - + "This only applies to SHOW/SELECT on external catalogs with a custom access controller. " - + "Internal catalogs, catalogs without a custom access controller, and other privileges such " - + "as CREATE/LOAD/ALTER are still validated by the default logic."}) + @ConfField(description = "Whether to skip the FE internal catalog privilege check in catalog-level privilege " + + "validation. This only applies to SHOW/SELECT on external catalogs with a custom access " + + "controller. Internal catalogs, catalogs without a custom access controller, and other " + + "privileges such as CREATE/LOAD/ALTER are still validated by the default logic.") public static boolean skip_catalog_priv_check = false; - @ConfField(mutable = true, description = { - "In compute-storage separation mode, whether to obtain partition version information in batches when " - + "calculating the delete bitmap. Enabled by default."}) + @ConfField(mutable = true, description = "In compute-storage separation mode, whether to obtain partition version " + + "information in batches when calculating the delete bitmap. Enabled by " + "default.") public static boolean calc_delete_bitmap_get_versions_in_batch = true; - @ConfField(mutable = true, description = { - "In compute-storage separation mode, whether to wait for pending transactions to complete before " - + "obtaining partition version information when calculating the delete bitmap. Enabled " - + "by default."}) + @ConfField(mutable = true, description = "In compute-storage separation mode, whether to wait for pending " + + "transactions to complete before obtaining partition version information " + + "when calculating the delete bitmap. Enabled by default.") public static boolean calc_delete_bitmap_get_versions_waiting_for_pending_txns = true; - @ConfField(mutable = true, masterOnly = true, description = { - "Whether to enable adaptive random bucket load. When enabled, each BE computes its own local " - + "bucket set (buckets whose primary replica it hosts) from the tablet location info " - + "sent by FE, and rotates across those buckets once per-tablet write volume exceeds " - + "the threshold (default 200 MB). This reduces import memory pressure and improves " - + "throughput for random-distribution tables. Covers all load types uniformly.", - "是否启用自适应随机桶导入。开启后每个 BE 根据 FE 下发的 tablet 位置信息自行计算本地桶集合" - + "（持有主副本的桶），并在单个 tablet 写入量超过阈值（默认 200 MB）后在本地桶之间轮转。" - + "可降低导入内存压力并提升随机分桶表的吞吐量，覆盖所有导入类型。"}) + @ConfField(mutable = true, masterOnly = true, description = "Whether to enable adaptive random bucket load. When " + + "enabled, each BE computes its own local bucket set " + + "(buckets whose primary replica it hosts) from the " + + "tablet location info sent by FE, and rotates across " + + "those buckets once per-tablet write volume exceeds " + + "the threshold (default 200 MB). This reduces import " + "memory pressure and improves throughput for " + + "random-distribution tables. Covers all load types " + "uniformly.") public static boolean enable_adaptive_random_bucket_load = true; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index e58fea913d7b48..436a8a34d33366 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -65,10 +65,7 @@ public class ConfigBase { String callbackClassString() default ""; // description for this config item. - // There should be 2 elements in the array. - // The first element is the description in Chinese. - // The second element is the description in English. - String[] description() default {"待补充", "TODO"}; + String description() default "TODO"; // Enum options for this config item, if it has. String[] options() default {}; diff --git a/fe/fe-common/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-common/src/main/java/org/apache/doris/qe/GlobalVariable.java index d2ba60eb13e78e..46eceb0ecaf1ef 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-common/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -165,14 +165,11 @@ private static String resolveSystemTimeZone() { public static long validatePasswordPolicy = 0; @VarAttrDef.VarAttr(name = VALIDATE_PASSWORD_DICTIONARY_FILE, flag = VarAttrDef.GLOBAL, - description = {"密码验证字典文件路径。文件为纯文本格式，每行一个词。" - + "当 validate_password_policy 为 STRONG(2) 时，密码中不能包含字典中的任何词（不区分大小写）。" - + "如果为空，则使用内置字典。", - "Path to the password validation dictionary file. " - + "The file should be plain text with one word per line. " - + "When validate_password_policy is STRONG(2), " - + "the password cannot contain any word from the dictionary " - + "(case-insensitive). If empty, a built-in dictionary will be used."}) + description = "Path to the password validation dictionary file. " + + "The file should be plain text with one word per line. " + + "When validate_password_policy is STRONG(2), " + + "the password cannot contain any word from the dictionary " + + "(case-insensitive). If empty, a built-in dictionary will be used.") public static volatile String validatePasswordDictionaryFile = ""; // If set to true, the db name of TABLE_SCHEMA column in tables in information_schema @@ -197,94 +194,69 @@ private static String resolveSystemTimeZone() { public static int auditPluginMaxSqlLength = 2097152; @VarAttrDef.VarAttr(name = AUDIT_PLUGIN_MAX_INSERT_STMT_LENGTH, flag = VarAttrDef.GLOBAL, - description = {"专门用于限制 INSERT 语句的长度。如果该值大于 AUDIT_PLUGIN_MAX_SQL_LENGTH，" - + "则使用 AUDIT_PLUGIN_MAX_SQL_LENGTH 的值。" - + "如果 INSERT 语句超过该长度，将会被截断。", - "This is specifically used to limit the length of INSERT statements. " - + "If this value is greater than AUDIT_PLUGIN_MAX_SQL_LENGTH, " - + "it will use the value of AUDIT_PLUGIN_MAX_SQL_LENGTH. " - + "If an INSERT statement exceeds this length, it will be truncated."}) + description = "This is specifically used to limit the length of INSERT statements. " + + "If this value is greater than AUDIT_PLUGIN_MAX_SQL_LENGTH, " + + "it will use the value of AUDIT_PLUGIN_MAX_SQL_LENGTH. " + + "If an INSERT statement exceeds this length, it will be truncated.") public static int auditPluginMaxInsertStmtLength = Integer.MAX_VALUE; @VarAttrDef.VarAttr(name = AUDIT_PLUGIN_LOAD_TIMEOUT, flag = VarAttrDef.GLOBAL) public static int auditPluginLoadTimeoutS = 600; @VarAttrDef.VarAttr(name = ENABLE_GET_ROW_COUNT_FROM_FILE_LIST, flag = VarAttrDef.GLOBAL, - description = { - "针对外表，是否允许根据文件列表估算表行数。获取文件列表可能是一个耗时的操作，" - + "如果不需要估算表行数或者对性能有影响，可以关闭该功能。", - "For external tables, whether to enable getting row count from file list. " - + "Getting file list may be a time-consuming operation. " - + "If you don't need to estimate the number of rows in the table " - + "or it affects performance, you can disable this feature."}) + description = "For external tables, whether to enable getting row count from file list. " + + "Getting file list may be a time-consuming operation. " + + "If you don't need to estimate the number of rows in the table " + + "or it affects performance, you can disable this feature.") public static boolean enable_get_row_count_from_file_list = true; @VarAttrDef.VarAttr(name = READ_ONLY, flag = VarAttrDef.GLOBAL, - description = {"仅用于兼容 MySQL 生态，暂无实际意义", - "Only for compatibility with MySQL ecosystem, no practical meaning"}) + description = "Only for compatibility with MySQL ecosystem, no practical meaning") public static boolean read_only = true; @VarAttrDef.VarAttr(name = SUPER_READ_ONLY, flag = VarAttrDef.GLOBAL, - description = {"仅用于兼容 MySQL 生态，暂无实际意义", - "Only for compatibility with MySQL ecosystem, no practical meaning"}) + description = "Only for compatibility with MySQL ecosystem, no practical meaning") public static boolean super_read_only = true; @VarAttrDef.VarAttr(name = PARTITION_ANALYZE_BATCH_SIZE, flag = VarAttrDef.GLOBAL, - description = { - "批量收集分区信息的分区数", - "Number of partitions to collect in one batch."}) + description = "Number of partitions to collect in one batch.") public static int partitionAnalyzeBatchSize = 10; @VarAttrDef.VarAttr(name = HUGE_PARTITION_LOWER_BOUND_ROWS, flag = VarAttrDef.GLOBAL, - description = { - "行数超过该值的分区将跳过自动分区收集", - "This defines the lower size bound for large partitions, which will skip auto partition analyze."}) + description = "This defines the lower size bound for large partitions, " + + "which will skip auto partition analyze.") public static long hugePartitionLowerBoundRows = 100000000L; @VarAttrDef.VarAttr(name = ENABLE_FETCH_ICEBERG_STATS, flag = VarAttrDef.GLOBAL, - description = { - "当 HMS catalog 中的 Iceberg 表没有统计信息时，是否通过 Iceberg Api 获取统计信息", - "Enable fetch stats for HMS Iceberg table when it's not analyzed."}) + description = "Enable fetch stats for HMS Iceberg table when it's not analyzed.") public static boolean enableFetchIcebergStats = false; @VarAttrDef.VarAttr(name = ENABLE_ANSI_QUERY_ORGANIZATION_BEHAVIOR, flag = VarAttrDef.GLOBAL, - description = { - "控制 query organization 的行为。当设置为 true 时使用 ANSI 的 query organization 行为，即作用于整个语句。" - + "当设置为 false 时，使用 Doris 历史版本的行为，" - + "即 order by 默认只作用于 set operation 的最后一个 operand。", - "Controls the behavior of query organization. When set to true, uses the ANSI query" - + " organization behavior, which applies to the entire statement. When set to false," - + " uses the behavior of Doris's historical versions, where order by by default only" - + " applies to the last operand of the set operation."}) + description = "Controls the behavior of query organization. When set to true, uses the ANSI query" + + " organization behavior, which applies to the entire statement. When set to false," + + " uses the behavior of Doris's historical versions, where order by by default only" + + " applies to the last operand of the set operation.") public static boolean enable_ansi_query_organization_behavior = true; @VarAttrDef.VarAttr(name = ENABLE_NEW_TYPE_COERCION_BEHAVIOR, flag = VarAttrDef.GLOBAL, - description = { - "控制隐式类型转换的行为，当设置为 true 时，使用新的行为。新行为更为合理。类型优先级从高到低为时间相关类型 > " - + "数值类型 > 复杂类型 / JSON 类型 / IP 类型 > 字符串类型 > VARIANT 类型。当两个或多个不同类型的表达式" - + "进行比较时，强制类型转换优先向高优先级类型转换。转换时尽可能保留精度，如：" - + "当转换为时间相关类型时，当无法确定精度时，优先使用 6 位精度的 DATETIME 类型。" - + "当转换为数值类型时，当无法确定精度时，优先使用 DECIMAL 类型。", - "Controls the behavior of implicit type conversion. When set to true, the new behavior is used," - + " which is more reasonable. The type priority, from highest to lowest, is: time-related" - + " types > numeric types > complex types / JSON types / IP types > string types" - + " > VARIANT types. When comparing two or more expressions of different types, " - + "type coercion preferentially converts values toward the type with higher priority. " - + "Precision is preserved as much as possible during conversion. For example, " - + "when converting to a time-related type and precision cannot be determined, " - + "the DATETIME type with 6-digit precision is preferred. When converting to" - + " a numeric type and precision cannot be determined, the DECIMAL type is preferred."}) + description = "Controls the behavior of implicit type conversion. When set to true, " + + "the new behavior is used," + + " which is more reasonable. The type priority, from highest to lowest, is: time-related" + + " types > numeric types > complex types / JSON types / IP types > string types" + + " > VARIANT types. When comparing two or more expressions of different types, " + + "type coercion preferentially converts values toward the type with higher priority. " + + "Precision is preserved as much as possible during conversion. For example, " + + "when converting to a time-related type and precision cannot be determined, " + + "the DATETIME type with 6-digit precision is preferred. When converting to" + + " a numeric type and precision cannot be determined, the DECIMAL type is preferred.") public static boolean enableNewTypeCoercionBehavior = true; @VarAttrDef.VarAttr(name = ENABLE_NESTED_NAMESPACE, flag = VarAttrDef.GLOBAL, - description = { - "是否允许访问 `ns1.ns2` 这种类型的 database。当前仅适用于 External Catalog 中映射 Database 并访问。" - + "不支持创建。", - "Whether to allow accessing databases of the form `ns1.ns2`. " - + "Currently, this only applies to mapping databases in " - + "External Catalogs and accessing them. " - + "Creation is not supported."}) + description = "Whether to allow accessing databases of the form `ns1.ns2`. " + + "Currently, this only applies to mapping databases in " + + "External Catalogs and accessing them. " + + "Creation is not supported.") public static boolean enableNestedNamespace = false; // Don't allow creating instance. diff --git a/fe/fe-common/src/main/java/org/apache/doris/qe/VarAttrDef.java b/fe/fe-common/src/main/java/org/apache/doris/qe/VarAttrDef.java index 25de254eee01c6..a5d4eda439169c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/qe/VarAttrDef.java +++ b/fe/fe-common/src/main/java/org/apache/doris/qe/VarAttrDef.java @@ -68,11 +68,8 @@ public class VarAttrDef { VariableAnnotation varType() default VariableAnnotation.NONE; - // description for this config item. - // There should be 2 elements in the array. - // The first element is the description in Chinese. - // The second element is the description in English. - String[] description() default {"待补充", "TODO"}; + // description for this config item + String description() default "TODO"; // Enum options for this config item, if it has. String[] options() default {}; diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java index c59cb44c66d65e..09572c618c7ba3 100644 --- a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java +++ b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java @@ -21,6 +21,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -99,6 +100,18 @@ public void testSetEmptyArray() throws ConfigException { Assert.assertEquals("array length should be 0", 0, Config.mysql_compat_var_whitelist.length); } + @Test + public void testConfFieldDescriptionsAreEnglishStrings() throws Exception { + for (Field field : Config.class.getFields()) { + ConfigBase.ConfField confField = field.getAnnotation(ConfigBase.ConfField.class); + if (confField == null) { + continue; + } + Assert.assertFalse("Chinese description found in config: " + field.getName(), + confField.description().matches(".*[\\u4e00-\\u9fff].*")); + } + } + // File-path and jdbc-driver security configs must only be settable in fe.conf (ops), never at runtime // via ADMIN SET FRONTEND CONFIG. setMutableConfig is exactly that runtime entrypoint, so it must reject them. @Test diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 1eba8466a56912..cc144759f24455 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -914,13 +914,12 @@ public String toString() { public static final String HOT_VALUE_COLLECT_COUNT = "hot_value_collect_count"; @VarAttrDef.VarAttr(name = HOT_VALUE_COLLECT_COUNT, needForward = true, - description = {"列统计信息收集时，收集占比排名前 HOT_VALUE_COLLECT_COUNT 的值作为 hot value", - "When collecting column statistics, collect the top values ranked by their " - + "proportion as hot values, up to HOT_VALUE_COLLECT_COUNT."}) + description = "When collecting column statistics, collect the top values ranked by their " + + "proportion as hot values, up to HOT_VALUE_COLLECT_COUNT.") public int hotValueCollectCount = 10; // Select the values that account for at least 10% of the column @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_WAND_QUERY, - description = {"是否开启倒排索引WAND查询优化", "Whether to enable inverted index WAND query optimization"}) + description = "Whether to enable inverted index WAND query optimization") public boolean enableInvertedIndexWandQuery = true; public void setHotValueCollectCount(int count) { @@ -942,11 +941,10 @@ public static int getHotValueCollectCount() { public static final String SKEW_VALUE_THRESHOLD = "skew_value_threshold"; @VarAttrDef.VarAttr(name = SKEW_VALUE_THRESHOLD, needForward = true, - description = {"当列中某个特定值的出现次数大于等于（rowCount/ndv）× skewValueThreshold 时，该值即被视为热点值", - "When the occurrence of a value in a column is greater than " - + "skewValueThreshold tmies of average occurences " - + "(occurrences >= skewValueThreshold * rowCount / ndv), " - + "the value is regarded as hot value"}) + description = "When the occurrence of a value in a column is greater than " + + "skewValueThreshold tmies of average occurences " + + "(occurrences >= skewValueThreshold * rowCount / ndv), " + + "the value is regarded as hot value") private double skewValueThreshold = 10; public void setSkewValueThreshold(int threshold) { @@ -963,8 +961,7 @@ public static double getSkewValueThreshold() { public static final String HOT_VALUE_THRESHOLD = "hot_value_threshold"; @VarAttrDef.VarAttr(name = HOT_VALUE_THRESHOLD, needForward = true, - description = {"hot value 在列中出现的最小比例", - "The minimum ratio of occurrences of a hot value in a column"}) + description = "The minimum ratio of occurrences of a hot value in a column") private double hotValueThreshold = 0.10d; public void setHotValueThreshold(double threshold) { @@ -1057,10 +1054,8 @@ public static double getHotValueThreshold() { public boolean enableStats = true; @VarAttrDef.VarAttr(name = ENABLE_LOW_CONFIDENCE_EQ_JOIN_REMAINING_CONDITION_DECAY, needForward = true, - description = { - "是否在低置信度等值 join 只有 untrust 条件时，基于最小 ratio 对剩余条件继续做衰减", - "Whether to continue decaying remaining low-confidence equality join conditions after " - + "applying the minimum ratio when all equality predicates are untrustworthy" }) + description = "Whether to continue decaying remaining low-confidence equality join conditions after " + + "applying the minimum ratio when all equality predicates are untrustworthy") public boolean enableLowConfidenceEqJoinRemainingConditionDecay = true; // session origin value @@ -1073,25 +1068,20 @@ public static double getHotValueThreshold() { public boolean expandRuntimeFilterByInnerJoin = true; @VarAttrDef.VarAttr(name = ENABLE_DECOUPLED_RUNTIME_FILTER, - description = {"启用解耦 Runtime Filter：允许 RF 的生产者和条件来源分属不同 join 节点", - "Enable decoupled runtime filter: allow RF producer and predicate source " - + "to be on different join nodes"}) + description = "Enable decoupled runtime filter: allow RF producer and predicate source " + + "to be on different join nodes") public boolean enableDecoupledRuntimeFilter = true; @VarAttrDef.VarAttr(name = DECOUPLED_RF_NDV_RATIO_THRESHOLD, - description = {"解耦 RF 的 NDV 比值阈值。当 probe_ndv/build_ndv < 该值时，" - + "优先使用解耦 RF 并删除标准 RF；否则保留标准 RF，解耦 RF 设为非阻塞", - "NDV ratio threshold for decoupled RF. When probe_ndv/build_ndv < threshold, " + description = "NDV ratio threshold for decoupled RF. When probe_ndv/build_ndv < threshold, " + "prefer decoupled RF and remove standard RF; otherwise keep standard RF " - + "and make decoupled RF non-blocking"}) + + "and make decoupled RF non-blocking") public double decoupledRfNdvRatioThreshold = 0.5; @VarAttrDef.VarAttr(name = MIN_DECOUPLED_RF_TARGET_ROWS, - description = {"解耦 RF 目标扫描节点的最小行数。当目标扫描行数低于此阈值时，" - + "跳过生成解耦 RF（因为小表扫描太快，RF 来不及生效）", - "Minimum row count for the target scan of a decoupled RF. " + description = "Minimum row count for the target scan of a decoupled RF. " + "Skip generating decoupled RF when the target scan has fewer rows " - + "(small scans complete too quickly for the RF to arrive in time)"}) + + "(small scans complete too quickly for the RF to arrive in time)") public long minDecoupledRfTargetRows = 5_000_000; @VarAttrDef.VarAttr(name = "enable_aggregate_cse", needForward = true) @@ -1101,26 +1091,24 @@ public static double getHotValueThreshold() { // When false (default), the optimizer rule PushDownVirtualColumnsIntoOlapScan will not apply. @VarAttrDef.VarAttr(name = "enable_virtual_slot_for_cse", needForward = true, varType = VariableAnnotation.EXPERIMENTAL, - description = {"是否启用将公共子表达式作为虚拟列下推到 OlapScan（实验特性）", - "Enable pushing common sub-expressions as virtual columns into OlapScan (experimental)"}) + description = "Enable pushing common sub-expressions as virtual columns into OlapScan (experimental)") public boolean experimentalEnableVirtualSlotForCse = false; @VarAttrDef.VarAttr(name = ENABLE_NEW_SHUFFLE_HASH_METHOD) public boolean enableNewShffleHashMethod = true; @VarAttrDef.VarAttr(name = JDBC_CLICKHOUSE_QUERY_FINAL, needForward = true, - description = {"是否在查询 ClickHouse JDBC 外部表时，对查询 SQL 添加 FINAL 关键字。", - "Whether to add the FINAL keyword to the query SQL when querying ClickHouse JDBC external tables."}) + description = "Whether to add the FINAL keyword to the query SQL when querying ClickHouse JDBC external " + + "tables.") public boolean jdbcClickhouseQueryFinal = false; @VarAttrDef.VarAttr(name = ENABLE_JDBC_ORACLE_NULL_PREDICATE_PUSH_DOWN, needForward = true, - description = {"是否允许将 NULL 谓词下推到 Oracle JDBC 外部表。", - "Whether to allow NULL predicates to be pushed down to Oracle JDBC external tables."}) + description = "Whether to allow NULL predicates to be pushed down to Oracle JDBC external tables.") public boolean enableJdbcOracleNullPredicatePushDown = false; @VarAttrDef.VarAttr(name = ENABLE_JDBC_CAST_PREDICATE_PUSH_DOWN, needForward = true, - description = {"是否允许将带有 CAST 表达式的谓词下推到 JDBC 外部表。", - "Whether to allow predicates with CAST expressions to be pushed down to JDBC external tables."}) + description = "Whether to allow predicates with CAST expressions to be pushed down to JDBC external tables." + ) public boolean enableJdbcCastPredicatePushDown = true; @VarAttrDef.VarAttr(name = INSERT_VISIBLE_TIMEOUT_MS, needForward = true) @@ -1129,21 +1117,19 @@ public static double getHotValueThreshold() { // Control whether publish timeout keeps the committed response or returns an explicit error. @VarAttrDef.VarAttr(name = INSERT_VISIBLE_TIMEOUT_RETURN_MODE, needForward = true, checker = "checkInsertVisibleTimeoutReturnMode", setter = "setInsertVisibleTimeoutReturnMode", - description = {"控制普通内表 INSERT 在 publish timeout 时返回给客户端的状态。", - "Controls the status returned to the client when a normal internal-table INSERT times out " - + "while waiting for publish visibility."}, + description = "Controls the status returned to the client when a normal internal-table INSERT times out " + + "while waiting for publish visibility.", options = {INSERT_VISIBLE_TIMEOUT_RETURN_MODE_COMMITTED, INSERT_VISIBLE_TIMEOUT_RETURN_MODE_ERROR}) public String insertVisibleTimeoutReturnMode = INSERT_VISIBLE_TIMEOUT_RETURN_MODE_COMMITTED; @VarAttrDef.VarAttr(name = ENABLE_EVENTUAL_CONSISTENT_CHANGE, needForward = true, - description = {"是否允许在 CHANGES/快照类时间查询中使用最终一致语义（不等待事务发布）。开启后可能返回不包含最新 commit 的结果。", - "Whether to allow eventual consistent semantics for time-based CHANGES/snapshot queries. " - + "If true, query may return results without waiting committed txns to be visible."}) + description = "Whether to allow eventual consistent semantics for time-based CHANGES/snapshot queries. " + + "If true, query may return results without waiting committed txns to be visible.") public boolean enableEventualConsistentChange = false; @VarAttrDef.VarAttr(name = CHANGE_VISIBLE_TIMEOUT_MS, needForward = true, - description = {"时间范围 CHANGES/快照查询等待 COMMITTED 事务发布为 VISIBLE 的最长时间（毫秒）。", - "Max time in ms to wait committed txns become visible for time-based CHANGES/snapshot queries."}) + description = "Max time in ms to wait committed txns become visible for time-based CHANGES/snapshot " + + "queries.") public long changeVisibleTimeoutMs = DEFAULT_CHANGE_VISIBLE_TIMEOUT_MS; // max memory used on every backend. Default value to 100G. @@ -1155,46 +1141,41 @@ public static double getHotValueThreshold() { public boolean enableAdaptiveScan = true; @VarAttrDef.VarAttr(name = SCAN_QUEUE_MEM_LIMIT, needForward = true, - description = {"每个 Scan Instance 的 block queue 能够保存多少字节的 block", - "How many bytes of block can be saved in the block queue of each Scan Instance"}) + description = "How many bytes of block can be saved in the block queue of each Scan Instance") // 100MB public long maxScanQueueMemByte = 2147483648L / 20; - @VarAttrDef.VarAttr(name = MAX_SCANNERS_CONCURRENCY, needForward = true, description = { - "ScanNode 扫描数据的最大并发，默认为 4", "The max threads to read data of ScanNode, default 4"}) + @VarAttrDef.VarAttr(name = MAX_SCANNERS_CONCURRENCY, needForward = true, description = "The max threads to read " + + "data of ScanNode, default 4") public int maxScannersConcurrency = 4; - @VarAttrDef.VarAttr(name = MAX_FILE_SCANNERS_CONCURRENCY, needForward = true, description = { - "FileScanNode 扫描数据的最大并发，默认为 16", "The max threads to read data of FileScanNode, default 16"}) + @VarAttrDef.VarAttr(name = MAX_FILE_SCANNERS_CONCURRENCY, needForward = true, description = "The max threads to " + + "read data of FileScanNode, default 16") public int maxFileScannersConcurrency = 16; - @VarAttrDef.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true, fuzzy = true, description = { - "开启后 FileScanNode 会在支持的查询场景使用 FileScannerV2，默认开启", - "When enabled, FileScanNode uses FileScannerV2 for supported query scans. Enabled by default."}) + @VarAttrDef.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true, fuzzy = true, description = "When enabled, " + + "FileScanNode uses FileScannerV2 for supported query scans. Enabled by default.") public boolean enableFileScannerV2 = true; @VarAttrDef.VarAttr(name = LOCAL_EXCHANGE_FREE_BLOCKS_LIMIT) public int localExchangeFreeBlocksLimit = 4; - @VarAttrDef.VarAttr(name = MIN_SCANNERS_CONCURRENCY, needForward = true, description = { - "Scanner 的最小并发度，默认为 1", "The min concurrency of Scanner, default 1" - }) + @VarAttrDef.VarAttr(name = MIN_SCANNERS_CONCURRENCY, needForward = true, description = "The min concurrency of " + + "Scanner, default 1") public int minScannersConcurrency = 1; - @VarAttrDef.VarAttr(name = MIN_FILE_SCANNERS_CONCURRENCY, needForward = true, description = { - "外表Scanner 的最小并发度，默认为 1", "The min concurrency of Remote Scanner, default 1" - }) + @VarAttrDef.VarAttr(name = MIN_FILE_SCANNERS_CONCURRENCY, needForward = true, description = "The min concurrency " + + "of Remote Scanner, default 1") public int minFileScannersConcurrency = 1; - @VarAttrDef.VarAttr(name = MIN_SCAN_SCHEDULER_CONCURRENCY, needForward = true, description = { - "ScanScheduler 的最小并发度，默认值 0 表示使用 Scan 线程池线程数量的两倍", "The min concurrency of ScanScheduler, " - + "default 0 means use twice the number of Scan thread pool threads" - }, varType = VariableAnnotation.DEPRECATED) + @VarAttrDef.VarAttr(name = MIN_SCAN_SCHEDULER_CONCURRENCY, needForward = true, description = "The min concurrency " + + "of ScanScheduler, " + + "default 0 means use twice the number of Scan thread pool threads", + varType = VariableAnnotation.DEPRECATED) public int minScanSchedulerConcurrency = 0; - @VarAttrDef.VarAttr(name = CTE_MAX_RECURSION_DEPTH, needForward = true, description = { - "CTE递归的最大深度，默认值100", - "The maximum depth of CTE recursion. Default is 100" }) + @VarAttrDef.VarAttr(name = CTE_MAX_RECURSION_DEPTH, needForward = true, description = "The maximum depth of CTE " + + "recursion. Default is 100") public int cteMaxRecursionDepth = 100; // By default, the number of Limit items after OrderBy is changed from 65535 items @@ -1258,16 +1239,13 @@ public static double getHotValueThreshold() { @VarAttrDef.VarAttr(name = WORKLOAD_VARIABLE, needForward = true) public String workloadGroup = ""; - @VarAttrDef.VarAttr(name = BYPASS_WORKLOAD_GROUP, needForward = true, description = { - "查询是否绕开 WorkloadGroup 的限制，目前仅支持绕开查询排队的逻辑", - "whether bypass workload group's limitation, currently only support bypass query queue"}) + @VarAttrDef.VarAttr(name = BYPASS_WORKLOAD_GROUP, needForward = true, description = "whether bypass workload " + + "group's limitation, currently only support bypass query queue") public boolean bypassWorkloadGroup = false; @VarAttrDef.VarAttr(name = QUERY_SLOT_COUNT, needForward = true, checker = "checkQuerySlotCount", - description = { - "每个查询占用的 slot 的数量，workload group 的 query slot 的总数等于设置的最大并发数", - "Number of slots occupied by each query, the total number of query slots " - + "of the workload group equals the maximum number of concurrent requests"}) + description = "Number of slots occupied by each query, the total number of query slots " + + "of the workload group equals the maximum number of concurrent requests") public int wgQuerySlotCount = 1; public void checkQuerySlotCount(String slotCnt) { @@ -1390,10 +1368,8 @@ public void checkQuerySlotCount(String slotCnt) { // Valid range: [1MB, 512MB]. Default 8MB. @VarAttrDef.VarAttr(name = PREFERRED_BLOCK_SIZE_BYTES, needForward = true, checker = "checkPreferredBlockSizeBytes", - description = {"目标输出 Block 字节数上限，自适应 batch size 功能使用。" - + "范围 [1MB, 512MB]，默认 8MB", - "Target output block size in bytes for adaptive batch size. " - + "Range [1MB, 512MB]. Default 8MB."}) + description = "Target output block size in bytes for adaptive batch size. " + + "Range [1MB, 512MB]. Default 8MB.") public long preferredBlockSizeBytes = 8388608L; // 8MB @VarAttrDef.VarAttr(name = DISABLE_STREAMING_PREAGGREGATIONS, fuzzy = true) @@ -1466,45 +1442,38 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = IGNORE_SPLIT_TYPE, checker = "checkIgnoreSplitType", options = {"NONE", "IGNORE_JNI", "IGNORE_NATIVE", "IGNORE_PAIMON_CPP"}, - description = {"忽略指定类型的 split", "Ignore splits of the specified type"}) + description = "Ignore splits of the specified type") public String ignoreSplitType = IgnoreSplitType.NONE.toString(); public static final String USE_CONSISTENT_HASHING_FOR_EXTERNAL_SCAN = "use_consistent_hash_for_external_scan"; @VarAttrDef.VarAttr(name = USE_CONSISTENT_HASHING_FOR_EXTERNAL_SCAN, - description = {"对外表采用一致性 hash 的方式做 split 的分发", - "Use consistent hashing to split the appearance for external scan"}) + description = "Use consistent hashing to split the appearance for external scan") public boolean useConsistentHashForExternalScan = false; @VarAttrDef.VarAttr(name = PROFILE_LEVEL, fuzzy = false, setter = "setProfileLevel", checker = "checkProfileLevel", - description = { "查询 profile 的级别，1 表示只收集 MergedProfile 级别的 Counter，2 表示打印详细信息，" - + "3 表示打开一些可能导致性能回退的 Counter", "The level of query profile, " - + "1 means only collect Counter of MergedProfile, 2 means print detailed information," - + " 3 means open some Counters that may cause performance degradation"}) + description = "The level of query profile, " + + "1 means only collect Counter of MergedProfile, 2 means print detailed information," + + " 3 means open some Counters that may cause performance degradation") public int profileLevel = 2; @VarAttrDef.VarAttr(name = MAX_INSTANCE_NUM) public int maxInstanceNum = 64; - @VarAttrDef.VarAttr(name = DML_PLAN_RETRY_TIMES, needForward = true, description = { - "写入规划的最大重试次数。为了避免死锁，写入规划时采用了分阶段加锁。当在两次加锁中间，表结构发生变更时，会尝试重新规划。" - + "此变量限制重新规划的最大尝试次数。", - "Maximum retry attempts for write planning. To avoid deadlocks, " - + "phased locking is adopted during write planning. " - + "When changes occur to the table structure between two locking phases, " - + "re-planning will be attempted. " - + "This variable limits the maximum number of retry attempts for re-planning." - }) + @VarAttrDef.VarAttr(name = DML_PLAN_RETRY_TIMES, needForward = true, description = "Maximum retry attempts for " + + "write planning. To avoid deadlocks, " + + "phased locking is adopted during write planning. " + + "When changes occur to the table structure between two locking phases, " + + "re-planning will be attempted. " + + "This variable limits the maximum number of retry attempts for re-planning.") public int dmlPlanRetryTimes = 3; @VarAttrDef.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true) public boolean enableInsertStrict = true; - @VarAttrDef.VarAttr(name = ENABLE_INSERT_VALUE_AUTO_CAST, needForward = true, description = { - "INSERT VALUE 语句是否自动类型转换。当前只针对长字符串自动截短。默认开。", - "INSERT VALUE statement whether to automatically type cast. Only use for truncate long string. " - + "ON by default." - }) + @VarAttrDef.VarAttr(name = ENABLE_INSERT_VALUE_AUTO_CAST, needForward = true, description = "INSERT VALUE " + + "statement whether to automatically type cast. Only use for truncate long string. " + + "ON by default.") public boolean enableInsertValueAutoCast = true; @VarAttrDef.VarAttr(name = INSERT_MAX_FILTER_RATIO, needForward = true) @@ -1516,30 +1485,21 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr( name = ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS, fuzzy = true, - description = { - "是否允许使用二分查找算法去过滤分区。默认开。", - "Whether to allow use binary search algorithm to filter partitions. ON by default." - } + description = "Whether to allow use binary search algorithm to filter partitions. ON by default." ) public boolean enableBinarySearchFilteringPartitions = true; @VarAttrDef.VarAttr( name = CACHE_SORTED_PARTITION_INTERVAL_SECOND, fuzzy = false, - description = { - "表数据更新后，多少秒之内不能使用二分查找分区裁剪", - "After updating table data, within how many seconds can " - + "binary search partitioning and pruning not be used." - } + description = "After updating table data, within how many seconds can " + + "binary search partitioning and pruning not be used." ) public int cacheSortedPartitionIntervalSecond = 10; @VarAttrDef.VarAttr(name = SKIP_PRUNE_PREDICATE, fuzzy = true, - description = { - "是否跳过“在分区裁剪后删除恒真谓词”的优化。默认为 OFF（即执行此优化）。", - "Skips the removal of always-true predicates after partition pruning. " - + "Defaults to OFF (optimization is active)." - } + description = "Skips the removal of always-true predicates after partition pruning. " + + "Defaults to OFF (optimization is active)." ) public boolean skipPrunePredicate = false; @@ -1572,13 +1532,7 @@ public enum IgnoreSplitType { // once it graduates. @VarAttrDef.VarAttr(name = ENABLE_QUERY_CACHE_INCREMENTAL, varType = VariableAnnotation.EXPERIMENTAL, needForward = true, - description = {"是否允许 BE 以增量合并的方式复用过期的 Query Cache 条目：只扫描缓存版本之后的" - + "增量 rowset，并与缓存的聚合中间结果一起交给上游合并。仅对聚合直压扫描且不做 finalize " - + "的缓存点生效，且选中索引须为追加写：DUP_KEYS 表，或增量窗口内未改写既有主键的写时合并" - + "（merge-on-write）UNIQUE_KEYS 表（BE 按 tablet 检查 delete bitmap）；增量不可捕获（如" - + "已被 compaction 合并）、含 DELETE 谓词或改写了历史行时自动回退全量重算。需与 " - + "enable_query_cache 同时开启。", - "Whether BE may reuse a stale query cache entry by incremental merge: scan only" + description = "Whether BE may reuse a stale query cache entry by incremental merge: scan only" + " the delta rowsets since the cached version and emit them together with the" + " cached partial aggregation blocks for the upstream merge. Only takes effect" + " when the cache point is a non-finalize aggregation directly over the scan" @@ -1586,7 +1540,7 @@ public enum IgnoreSplitType { + " UNIQUE_KEYS whose delta did not rewrite pre-existing keys (BE checks the" + " delete bitmap per tablet); falls back to a full recompute whenever the" + " delta cannot be captured (e.g. compacted away), contains delete predicates" - + " or rewrites history rows. Requires enable_query_cache."}) + + " or rewrites history rows. Requires enable_query_cache.") public boolean enableQueryCacheIncremental = false; // Forwarded for the same reason as enable_query_cache: the master builds @@ -1633,11 +1587,9 @@ public enum IgnoreSplitType { ) public boolean showHiddenColumns = false; - @VarAttrDef.VarAttr(name = ALLOW_PARTITION_COLUMN_NULLABLE, description = { - "是否允许 NULLABLE 列作为 PARTITION 列。开启后，RANGE PARTITION 允许 NULLABLE PARTITION 列" - + "（LIST PARTITION 当前不支持）。默认开。", - "Whether to allow NULLABLE columns as PARTITION columns. When ON, RANGE PARTITION allows " - + "NULLABLE PARTITION columns (LIST PARTITION is not supported currently). ON by default." }) + @VarAttrDef.VarAttr(name = ALLOW_PARTITION_COLUMN_NULLABLE, description = "Whether to allow NULLABLE columns as " + + "PARTITION columns. When ON, RANGE PARTITION allows " + + "NULLABLE PARTITION columns (LIST PARTITION is not supported currently). ON by default.") public boolean allowPartitionColumnNullable = true; @VarAttrDef.VarAttr(name = DELETE_WITHOUT_PARTITION, needForward = true) @@ -1650,8 +1602,7 @@ public enum IgnoreSplitType { public boolean enableNereidsDML = true; @VarAttrDef.VarAttr(name = ENABLE_NEREIDS_DML_WITH_PIPELINE, - varType = VariableAnnotation.REMOVED, description = { "在新优化器中，使用 pipeline 引擎执行 DML", - "execute DML with pipeline engine in Nereids" }) + varType = VariableAnnotation.REMOVED, description = "execute DML with pipeline engine in Nereids") public boolean enableNereidsDmlWithPipeline = true; @VarAttrDef.VarAttr(name = ENABLE_STRICT_CONSISTENCY_DML, needForward = true) @@ -1678,8 +1629,8 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = OPTIMIZE_INDEX_SCAN_PARALLELISM, needForward = true, - description = {"优化索引扫描时的 Scan 并行度，该优化目前只对 ann topn 查询生效", - "Optimize the Scan parallelism when indexing, this optimization only works for ann topn queries."}) + description = "Optimize the Scan parallelism when indexing, this optimization only works for ann topn " + + "queries.") private boolean optimizeIndexScanParallelism = true; @VarAttrDef.VarAttr(name = PARALLEL_SCAN_MAX_SCANNERS_COUNT, fuzzy = true, @@ -1700,41 +1651,33 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr( name = ENABLE_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"是否在 pipelineX 引擎上开启 local shuffle 优化", - "Whether to enable local shuffle on pipelineX engine."}, needForward = true) + description = "Whether to enable local shuffle on pipelineX engine.", needForward = true) private boolean enableLocalShuffle = true; @VarAttrDef.VarAttr( name = ENABLE_LOCAL_SHUFFLE_PLANNER, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"是否在FE规划Local Shuffle", - "Whether to plan local shuffle in frontend"}, needForward = true) + description = "Whether to plan local shuffle in frontend", needForward = true) private boolean enableLocalShufflePlanner = true; @VarAttrDef.VarAttr( name = FORCE_TO_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"是否在 pipelineX 引擎上强制开启 local shuffle 优化", - "Whether to force to local shuffle on pipelineX engine."}) + description = "Whether to force to local shuffle on pipelineX engine.") private boolean forceToLocalShuffle = false; @VarAttrDef.VarAttr( name = LOCAL_SHUFFLE_BUCKET_UPGRADE_RATIO, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"FE规划Local Shuffle时, 当池化bucket join所在fragment的每BE实例数大于" - + "每BE有数据分桶数的该倍数时, 将join两侧的桶分布本地重分发为hash分布以突破桶数并发上限。" - + "必须大于1才生效; 小于等于1(含0和负数)时关闭该优化", - "When FE plans local shuffle and a pooled bucket join fragment has more instances" + description = "When FE plans local shuffle and a pooled bucket join fragment has more instances" + " per BE than (buckets-with-data per BE) * this ratio, re-distribute both join" + " sides with local hash instead of bucket hash so join parallelism is no longer" + " capped at bucket count. Only takes effect when > 1; values <= 1 (including 0" - + " and negatives) disable the upgrade."}, needForward = true) + + " and negatives) disable the upgrade.", needForward = true) private double localShuffleBucketUpgradeRatio = 1.5; @VarAttrDef.VarAttr( name = BUCKET_SHUFFLE_DOWNGRADE_RATIO, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"当一侧基表总桶数小于总实例数的该倍数时, 放弃bucket shuffle join降级为shuffle join。" - + "小于等于0时永不降级。默认0.8保持原有行为", - "Downgrade bucket shuffle join to shuffle join when the base table side's total" + description = "Downgrade bucket shuffle join to shuffle join when the base table side's total" + " bucket count is less than total instance count times this ratio. Values <= 0" - + " never downgrade. Default 0.8 keeps the original behavior."}, needForward = true) + + " never downgrade. Default 0.8 keeps the original behavior.", needForward = true) private double bucketShuffleDowngradeRatio = 0.8; @VarAttrDef.VarAttr(name = ENABLE_LOCAL_MERGE_SORT) @@ -1763,14 +1706,12 @@ public enum IgnoreSplitType { public int parallelPrepareThreshold = 32; @VarAttrDef.VarAttr(name = READ_HIVE_JSON_IN_ONE_COLUMN, - description = {"在读取 hive json 的时候，由于存在一些不支持的 json 格式，我们默认会报错。为了让用户使用体验更好，" - + "当该变量为 true 的时候，将一整行 json 读取到第一列中，用户可以自行选择对一整行 json 进行处理，例如 JSON_PARSE。" - + "需要表的第一列的数据类型为 string.", - "When reading hive json, we will report an error by default because there are some unsupported " + description = "When reading hive json, we will report an error by default because there are some " + + "unsupported " + "json formats. In order to provide users with a better experience, when this variable is true," + "a whole line of json is read into the first column. Users can choose to process a whole line" + "of json, such as JSON_PARSE. The data type of the first column of the table needs to" - + "be string."}) + + "be string.") private boolean readHiveJsonInOneColumn = false; @VarAttrDef.VarAttr(name = ENABLE_COST_BASED_JOIN_REORDER) @@ -1784,16 +1725,16 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = ENABLE_REWRITE_ELEMENT_AT_TO_SLOT, fuzzy = true) private boolean enableRewriteElementAtToSlot = true; - @VarAttrDef.VarAttr(name = FORCE_SORT_ALGORITHM, needForward = true, description = { "强制指定 SortNode 的排序算法", - "Force the sort algorithm of SortNode to be specified" }) + @VarAttrDef.VarAttr(name = FORCE_SORT_ALGORITHM, needForward = true, description = "Force the sort algorithm of " + + "SortNode to be specified") public String forceSortAlgorithm = ""; @VarAttrDef.VarAttr(name = FULL_SORT_MAX_BUFFERED_BYTES, needForward = true, setter = "setFullSortMaxBufferedBytes") public long fullSortMaxBufferedBytes = 64L * 1024L * 1024L; - @VarAttrDef.VarAttr(name = "ignore_runtime_filter_error", needForward = true, description = { "在 rf 遇到错误的时候忽略该 rf", - "Ignore the rf when it encounters an error" }) + @VarAttrDef.VarAttr(name = "ignore_runtime_filter_error", + needForward = true, description = "Ignore the rf when it encounters an error") public boolean ignoreRuntimeFilterError = false; @VarAttrDef.VarAttr(name = RUNTIME_FILTER_MODE, needForward = true) @@ -1825,11 +1766,9 @@ public enum IgnoreSplitType { private boolean enableSyncRuntimeFilterSize = true; @VarAttrDef.VarAttr(name = RUNTIME_FILTER_BROADCAST_JOIN_PRODUCER_NUM, needForward = true, - description = {"控制 Nereids 分布式规划中每个 broadcast join runtime filter 的生产 BE 数量。" - + "设置为小于等于 0 时不限制。Legacy Coordinator 路径保持原行为。", - "Controls the number of producer BEs for each broadcast join runtime filter in " + description = "Controls the number of producer BEs for each broadcast join runtime filter in " + "the Nereids distributed planner. Values less than or equal to 0 disable the limit. " - + "The legacy Coordinator path keeps the existing behavior."}) + + "The legacy Coordinator path keeps the existing behavior.") private int runtimeFilterBroadcastJoinProducerNum = 3; @VarAttrDef.VarAttr(name = RUNTIME_FILTER_TREE_PUBLISH_MAX_SEND_BYTES, needForward = true, fuzzy = true, @@ -1843,16 +1782,14 @@ public enum IgnoreSplitType { private boolean enableParallelResultSink = true; @VarAttrDef.VarAttr(name = "sort_phase_num", fuzzy = true, needForward = true, - description = {"如设置为 1，则只生成 1 阶段 sort，设置为 2，则只生成 2 阶段 sort，设置其它值，优化器根据代价选择 sort 类型", - "set the number of sort phases 1 or 2. if set other value, let cbo decide the sort type"}) + description = "set the number of sort phases 1 or 2. if set other value, let cbo decide the sort type") public int sortPhaseNum = 0; @VarAttrDef.VarAttr(name = HIVE_TEXT_COMPRESSION, fuzzy = true, needForward = true) private String hiveTextCompression = "plain"; @VarAttrDef.VarAttr(name = READ_CSV_EMPTY_LINE_AS_NULL, needForward = true, - description = {"在读取 csv 文件时是否读取 csv 的空行为 null", - "Determine whether to read empty rows in CSV files as NULL when reading CSV files."}) + description = "Determine whether to read empty rows in CSV files as NULL when reading CSV files.") public boolean readCsvEmptyLineAsNull = false; @VarAttrDef.VarAttr(name = USE_RF_DEFAULT) @@ -1871,16 +1808,14 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = "enable_topn_expr_pullup", needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"是否将TopN下方Project中的非平凡表达式上拉至TopN之上，" - + "以扩大延迟物化范围", - "Whether to pull up non-trivial expressions from Project below TopN, " - + "to expand lazy materialization scope"}) + description = "Whether to pull up non-trivial expressions from Project below TopN, " + + "to expand lazy materialization scope") public boolean enableTopnExprPullup = true; @VarAttrDef.VarAttr(name = ENABLE_PRUNE_NESTED_COLUMN, needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, - description = {"是否裁剪 map/struct 类型", "Whether to prune the type of map/struct"} + description = "Whether to prune the type of map/struct" ) public boolean enablePruneNestedColumns = true; @@ -1993,20 +1928,16 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { @VarAttrDef.VarAttr(name = ENABLE_PARTITION_TOPN) private boolean enablePartitionTopN = true; - @VarAttrDef.VarAttr(name = PARTITION_TOPN_MAX_PARTITIONS, needForward = true, description = { - "这个阈值决定了 partition_topn 计算时的最大分区数量，超过这个阈值后且输入总行数少于预估总量，剩余的数据将直接透传给下一个算子", - "This threshold determines how many partitions will be allocated for window function get topn." - + " if this threshold is exceeded and input rows less than the estimated total rows, the remaining" - + " data will be pass through to other node directly." - }) + @VarAttrDef.VarAttr(name = PARTITION_TOPN_MAX_PARTITIONS, needForward = true, description = "This threshold " + + "determines how many partitions will be allocated for window function get topn." + + " if this threshold is exceeded and input rows less than the estimated total rows, the remaining" + + " data will be pass through to other node directly.") private int partitionTopNMaxPartitions = 1024; - @VarAttrDef.VarAttr(name = PARTITION_TOPN_PER_PARTITION_ROWS, needForward = true, description = { - "这个数值用于 partition_topn 预估每个分区的行数，用来计算所有分区的预估数据总量，决定是否能透传下一个算子", - "This value is used for partition_topn to estimate the number of rows in each partition, to calculate " + @VarAttrDef.VarAttr(name = PARTITION_TOPN_PER_PARTITION_ROWS, needForward = true, description = "This value is " + + "used for partition_topn to estimate the number of rows in each partition, to calculate " + " the estimated total amount of data for all partitions, and to determine whether the next operator " - + " can be passed transparently." - }) + + " can be passed transparently.") private int partitionTopNPerPartitionRows = 1000; @VarAttrDef.VarAttr(name = GLOBAL_PARTITION_TOPN_THRESHOLD) @@ -2031,18 +1962,17 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { private boolean checkOverflowForDecimal = true; @VarAttrDef.VarAttr(name = DECIMAL_OVERFLOW_SCALE, needForward = true, affectQueryResultInPlan = true, - description = { - "当 decimal 数值计算结果精度溢出时，计算结果最多可保留的小数位数", "When the precision of the result of" + description = "When the precision of the result of" + " a decimal numerical calculation overflows," - + "the maximum number of decimal scale that the result can be retained"} + + "the maximum number of decimal scale that the result can be retained" ) public int decimalOverflowScale = 6; @VarAttrDef.VarAttr(name = ENABLE_DPHYP_OPTIMIZER) public boolean enableDPHypOptimizer = false; - @VarAttrDef.VarAttr(name = SHORT_CIRCUIT_EVALUATION, fuzzy = true, description = { "是否启用短路求值", - "Whether to enable short-circuit evaluation" }) + @VarAttrDef.VarAttr(name = SHORT_CIRCUIT_EVALUATION, fuzzy = true, description = "Whether to enable short-circuit " + + "evaluation") public boolean shortCircuitEvaluation = false; /** @@ -2054,9 +1984,8 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { private int nthOptimizedPlan = 1; @VarAttrDef.VarAttr(name = REQUIRED_GROUP_IDS, - description = {"指定优化器必须选择包含这些 Group ID 的物理计划（逗号分隔的整数列表）", - "Force the optimizer to choose a physical plan containing these Group IDs " - + "(comma-separated integer list)"}) + description = "Force the optimizer to choose a physical plan containing these Group IDs " + + "(comma-separated integer list)") public String requiredGroupIds = ""; public boolean isEnableLeftZigZag() { @@ -2122,11 +2051,12 @@ public boolean isEnableHboNonStrictMatchingMode() { private boolean enableNereidsPlanner = true; @VarAttrDef.VarAttr(name = ENABLE_PRELOAD_EXTERNAL_METADATA, - needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, description = { - "是否在获取内表规划期读锁前预加载 Hive/Hudi/Iceberg/Paimon/JDBC 外表元数据", - "Whether to preload Hive/Hudi/Iceberg/Paimon/JDBC external table metadata before internal table " - + "plan-time read locks are acquired" - }) + needForward = true, + fuzzy = false, + varType = VariableAnnotation.EXPERIMENTAL, + description = "Whether to preload Hive/Hudi/Iceberg/Paimon/JDBC external table metadata before internal " + + "table " + + "plan-time read locks are acquired") private boolean enablePreloadExternalMetadata = false; @VarAttrDef.VarAttr(name = DISABLE_NEREIDS_RULES, needForward = true) @@ -2135,12 +2065,9 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr(name = ENABLE_NEREIDS_RULES, needForward = true) public String enableNereidsRules = ""; - @VarAttrDef.VarAttr(name = ENABLE_VISITOR_REWRITER_DEPTH_THRESHOLD, needForward = true, description = { - "当查询计划的深度小于或等于这个阈值时，使用 visitor rewriter 去加速改写，否则使用 stack rewriter 去改写，" - + "防止 StackOverflowError", - "When the depth of the query plan is less than or equal to this threshold, use visitor rewriter to " - + "speed up rewriting, otherwise use stack rewriter to rewrite to prevent StackOverflowError" - }) + @VarAttrDef.VarAttr(name = ENABLE_VISITOR_REWRITER_DEPTH_THRESHOLD, needForward = true, description = "When the " + + "depth of the query plan is less than or equal to this threshold, use visitor rewriter to " + + "speed up rewriting, otherwise use stack rewriter to rewrite to prevent StackOverflowError") public int enableVisitorRewriterDepthThreshold = 100; @VarAttrDef.VarAttr(name = DISABLE_NEREIDS_EXPRESSION_RULES, needForward = true, @@ -2153,13 +2080,11 @@ public boolean isEnableHboNonStrictMatchingMode() { public double filterCostFactor = 0.0001; @VarAttrDef.VarAttr(name = ENABLE_NEREIDS_DISTRIBUTE_PLANNER, needForward = true, - fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, description = { - "使用新的 nereids 的分布式规划器的开关，这个分布式规划器可以规划出一些更高效的查询计划，比如在某些情况下，" - + "可以把左表 shuffle 到右表去做 bucket shuffle join", - "The switch to use new DistributedPlanner of nereids, this planner can planning some " - + "more efficient query plans, e.g. in certain situations, shuffle left side to " - + "right side to do bucket shuffle join" - } + fuzzy = false, + varType = VariableAnnotation.EXPERIMENTAL, + description = "The switch to use new DistributedPlanner of nereids, this planner can planning some " + + "more efficient query plans, e.g. in certain situations, shuffle left side to " + + "right side to do bucket shuffle join" ) private boolean enableNereidsDistributePlanner = true; @@ -2198,11 +2123,9 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr( name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE, - description = {"控制支持该变量的 scanner 是否启用运行时过滤器分区裁剪。" - + "File Scanner V2 始终启用安全的分区裁剪。默认为 true。", - "Controls runtime-filter partition pruning in scanners that honor this variable. " - + "File Scanner V2 always enables safe partition pruning. " - + "The default value is true."}, + description = "Controls runtime-filter partition pruning in scanners that honor this variable. " + + "File Scanner V2 always enables safe partition pruning. " + + "The default value is true.", needForward = true, fuzzy = true) public boolean enableRuntimeFilterPartitionPrune = true; @@ -2221,10 +2144,7 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr( name = ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES, fuzzy = true, - description = { - "跳过大部分的优化规则，快速分析 insert into values 语句", - "Skip most optimization rules and quickly analyze insert into values statements" - } + description = "Skip most optimization rules and quickly analyze insert into values statements" ) private boolean enableFastAnalyzeInsertIntoValues = true; @@ -2232,17 +2152,15 @@ public boolean isEnableHboNonStrictMatchingMode() { public boolean enableFunctionPushdown = false; @VarAttrDef.VarAttr(name = ENABLE_EXT_FUNC_PRED_PUSHDOWN, needForward = true, - description = {"启用外部表（如通过 ODBC 或 JDBC 访问的表）查询中谓词的函数下推", - "Enable function pushdown for predicates in queries to external tables " - + "(such as tables accessed via ODBC or JDBC)"}) + description = "Enable function pushdown for predicates in queries to external tables " + + "(such as tables accessed via ODBC or JDBC)") public boolean enableExtFuncPredPushdown = true; @VarAttrDef.VarAttr(name = FORBID_UNKNOWN_COLUMN_STATS) public boolean forbidUnknownColStats = false; @VarAttrDef.VarAttr(name = ENABLE_SEGMENT_LIMIT_PUSHDOWN, fuzzy = true, needForward = true, - description = {"是否启用 SegmentIterator 层 LIMIT 下推。", - "Set whether to push down LIMIT into SegmentIterator."}) + description = "Set whether to push down LIMIT into SegmentIterator.") public boolean enableSegmentLimitPushdown = true; @VarAttrDef.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = false, flag = VarAttrDef.INVISIBLE, @@ -2334,10 +2252,7 @@ public boolean isEnableHboNonStrictMatchingMode() { name = USE_ONE_PHASE_AGG_FOR_GROUP_CONCAT_WITH_ORDER, needForward = true, fuzzy = true, - description = { - "允许使用一阶段聚合来执行带有 order 的 group_concat 函数", - "Enable to use one stage aggregation to execute the group_concat function with order" - } + description = "Enable to use one stage aggregation to execute the group_concat function with order" ) public boolean useOnePhaseAggForGroupConcatWithOrder = false; @@ -2345,10 +2260,8 @@ public boolean isEnableHboNonStrictMatchingMode() { // 1. read related rowids along with necessary column data // 2. spawn fetch RPC to other nodes to get related data by sorted rowids @VarAttrDef.VarAttr(name = ENABLE_TWO_PHASE_READ_OPT, fuzzy = true, varType = VariableAnnotation.REMOVED, - description = {"由topn_lazy_materialization_threshold 替代，" - + "当topn_lazy_materialization_threshold=-1时关闭两阶段读优化", - "Replaced by topn_lazy_materialization_threshold. The two-stage read optimization " - + "is disabled when topn_lazy_materialization_threshold = -1."}) + description = "Replaced by topn_lazy_materialization_threshold. The two-stage read optimization " + + "is disabled when topn_lazy_materialization_threshold = -1.") public boolean enableTwoPhaseReadOpt = true; @VarAttrDef.VarAttr(name = TOPN_OPT_LIMIT_THRESHOLD) public long topnOptLimitThreshold = 1024; @@ -2357,13 +2270,12 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr(name = ENABLE_SNAPSHOT_POINT_QUERY) public boolean enableSnapshotPointQuery = true; - @VarAttrDef.VarAttr(name = ENABLE_SERVER_SIDE_PREPARED_STATEMENT, needForward = true, description = { - "是否启用开启服务端 prepared statement", "Set whether to enable server side prepared statement."}) + @VarAttrDef.VarAttr(name = ENABLE_SERVER_SIDE_PREPARED_STATEMENT, needForward = true, description = "Set whether " + + "to enable server side prepared statement.") public boolean enableServeSidePreparedStatement = true; @VarAttrDef.VarAttr(name = MAX_PREPARED_STMT_COUNT, flag = VarAttrDef.GLOBAL, - needForward = true, description = { - "服务端 prepared statement 最大个数", "the maximum prepared statements server holds."}) + needForward = true, description = "the maximum prepared statements server holds.") public int maxPreparedStmtCount = 100000; @VarAttrDef.VarAttr(name = ENABLE_GROUP_COMMIT_FULL_PREPARE) @@ -2380,66 +2292,58 @@ public boolean isEnableHboNonStrictMatchingMode() { public boolean disableFileCache = false; @VarAttrDef.VarAttr(name = ENABLE_TOPN_LAZY_MAT_PHASE2_NO_WRITE_FILE_CACHE, needForward = true, - description = { - "开启后，TopN 延迟物化第二阶段读取在 file cache miss 时直接读远端且不写回 file cache。", - "When enabled, TopN lazy materialization phase-2 reads go remote-only on " - + "file-cache miss and do not write the missed range back to file cache." - }) + description = "When enabled, TopN lazy materialization phase-2 reads go remote-only on " + + "file-cache miss and do not write the missed range back to file cache.") public boolean enableTopnLazyMatPhase2NoWriteFileCache = false; // Whether enable block file cache. Only take effect when BE config item enable_file_cache is true. - @VarAttrDef.VarAttr(name = ENABLE_FILE_CACHE, needForward = true, description = { - "是否启用 file cache。该变量只有在 be.conf 中 enable_file_cache=true 时才有效，" - + "如果 be.conf 中 enable_file_cache=false，该 BE 节点的 file cache 处于禁用状态。", - "Set wether to use file cache. This variable takes effect only if the BE config enable_file_cache=true. " - + "The cache is not used when BE config enable_file_cache=false."}) + @VarAttrDef.VarAttr(name = ENABLE_FILE_CACHE, needForward = true, description = "Set wether to use file cache. " + + "This variable takes effect only if the BE config enable_file_cache=true. " + + "The cache is not used when BE config enable_file_cache=false.") public boolean enableFileCache = false; // Specify base path for file cache, or chose a random path. - @VarAttrDef.VarAttr(name = FILE_CACHE_BASE_PATH, needForward = true, description = { - "指定 block file cache 在 BE 上的存储路径，默认 'random'，随机选择 BE 配置的存储路径。", - "Specify the storage path of the block file cache on BE, default 'random', " - + "and randomly select the storage path configured by BE."}) + @VarAttrDef.VarAttr(name = FILE_CACHE_BASE_PATH, needForward = true, description = "Specify the storage path of " + + "the block file cache on BE, default 'random', " + + "and randomly select the storage path configured by BE.") public String fileCacheBasePath = "random"; // Whether enable query with inverted index. - @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_QUERY, needForward = true, description = { - "是否启用 inverted index query。", "Set whether to use inverted index query."}) + @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_QUERY, needForward = true, description = "Set whether to use " + + "inverted index query.") public boolean enableInvertedIndexQuery = true; // Whether enable pushdown count agg to scan node when using inverted index match. - @VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_COUNT_ON_INDEX, needForward = true, description = { - "是否启用 count_on_index pushdown。", "Set whether to pushdown count_on_index."}) + @VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_COUNT_ON_INDEX, needForward = true, description = "Set whether to " + + "pushdown count_on_index.") public boolean enablePushDownCountOnIndex = true; // Whether enable no need read data opt in segment_iterator. - @VarAttrDef.VarAttr(name = ENABLE_NO_NEED_READ_DATA_OPT, needForward = true, description = { - "是否启用 no_need_read_data opt。", "Set whether to enable no_need_read_data opt."}) + @VarAttrDef.VarAttr(name = ENABLE_NO_NEED_READ_DATA_OPT, needForward = true, description = "Set whether to enable " + + "no_need_read_data opt.") public boolean enableNoNeedReadDataOpt = true; // Whether enable pushdown minmax to scan node of unique table. - @VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE, needForward = true, description = { - "是否启用 pushdown minmax on unique table。", "Set whether to pushdown minmax on unique table."}) + @VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE, needForward = true, description = "Set whether to " + + "pushdown minmax on unique table.") public boolean enablePushDownMinMaxOnUnique = false; // Whether enable push down string type minmax to scan node. - @VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_STRING_MINMAX, needForward = true, description = { - "是否启用 string 类型 min max 下推。", "Set whether to enable push down string type minmax."}) + @VarAttrDef.VarAttr(name = ENABLE_PUSHDOWN_STRING_MINMAX, needForward = true, description = "Set whether to enable " + + "push down string type minmax.") public boolean enablePushDownStringMinMax = false; // Comma-separated list of MOR tables to enable value predicate pushdown. - @VarAttrDef.VarAttr(name = ENABLE_MOR_VALUE_PREDICATE_PUSHDOWN_TABLES, needForward = true, description = { - "指定启用MOR表value列谓词下推的表列表，格式：db1.tbl1,db2.tbl2 或 * 表示所有MOR表。", - "Comma-separated list of MOR tables to enable value predicate pushdown. " - + "Format: db1.tbl1,db2.tbl2 or * for all MOR tables."}) + @VarAttrDef.VarAttr(name = ENABLE_MOR_VALUE_PREDICATE_PUSHDOWN_TABLES, needForward = true, description = "Comma-sep" + + "arated list of MOR tables to enable value predicate pushdown. " + + "Format: db1.tbl1,db2.tbl2 or * for all MOR tables.") public String enableMorValuePredicatePushdownTables = ""; // Comma-separated list of MOR tables to read as DUP (skip merge, skip delete sign filter). @VarAttrDef.VarAttr(name = READ_MOR_AS_DUP_TABLES, needForward = true, - affectQueryResultInPlan = true, description = { - "指定以DUP模式读取MOR表的表列表（跳过合并和删除标记过滤），格式：db1.tbl1,db2.tbl2 或 * 表示所有MOR表。", - "Comma-separated list of MOR tables to read as DUP (skip merge, skip delete sign filter). " - + "Format: db1.tbl1,db2.tbl2 or * for all MOR tables."}) + affectQueryResultInPlan = true, + description = "Comma-separated list of MOR tables to read as DUP (skip merge, skip delete sign filter). " + + "Format: db1.tbl1,db2.tbl2 or * for all MOR tables.") public String readMorAsDupTables = ""; @VarAttrDef.VarAttr(name = MAX_TABLE_COUNT_USE_CASCADES_JOIN_REORDER, needForward = true) @@ -2461,11 +2365,9 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr(name = MEMO_LOGICAL_ROW_COUNT_AGGREGATION_POLICY, needForward = true, checker = "checkMemoLogicalRowCountAggregationPolicy", setter = "setMemoLogicalRowCountAggregationPolicy", - options = {"trust_join_count", "average", "median", "min" }, description = { - "控制 MemoStatsAndCostRecomputer 在多个逻辑候选统计之间如何聚合 group row count。" - + "支持 trust_join_count, average、median、min。", - "Controls how MemoStatsAndCostRecomputer aggregates group row count across multiple logical " - + "statistics candidates. Supported values: trust_join_count, average, median, min." }, + options = {"trust_join_count", "average", "median", "min" }, + description = "Controls how MemoStatsAndCostRecomputer aggregates group row count across multiple logical " + + "statistics candidates. Supported values: trust_join_count, average, median, min.", affectQueryResultInPlan = true) public String memoLogicalRowCountAggregationPolicy = "median"; @@ -2477,35 +2379,23 @@ public boolean isEnableHboNonStrictMatchingMode() { public int dphyperLimit = 2600; @VarAttrDef.VarAttr(name = "eager_aggregation_mode", needForward = true, - description = {"0: 根据统计信息决定是使用eager aggregation，" - + "1: 强制使用 eager aggregation，" - + "-1: 禁止使用 eager aggregation", - "0: Determine eager aggregation by statistics, " - + "1: force eager aggregation, " - + "-1: Prohibit eager aggregation "} + description = "0: Determine eager aggregation by statistics, " + + "1: force eager aggregation, " + + "-1: Prohibit eager aggregation " ) private int eagerAggregationMode = 0; @VarAttrDef.VarAttr(name = "force_eager_agg_hint", needForward = true, setter = "setForceEagerAggHint", - description = { - "用于测试/调试 eager aggregation 下推的匹配 hint。" - + "格式：`:=`，" - + "多个条目以分号分隔。例如：" - + "`sum:t1.a=push; sum:t2.a=nopush; count:*=push`。" - + "注意：hint 按聚合函数匹配，但生效粒度是当前候选下推分支/子树，而不是单个聚合函数独立生效；" - + "同一分支中只要有任一匹配项为 `nopush`，该分支本次不下推；" - + "否则只要有任一匹配项为 `push`，该分支本次可被强制下推，" - + "同分支内其他聚合函数会跟随这一决定。", - "Test/debug hint for eager aggregation push-down. " - + "Format: `:=`, " - + "with multiple entries separated by `;`. " - + "Example: `sum:t1.a=push; sum:t2.a=nopush; count:*=push`. " - + "Note: entries are matched per aggregate-function key, but the effect " - + "is applied at the current candidate push-down branch/subtree rather " - + "than to one function independently. If any matched entry in the branch " - + "is `nopush`, push-down is disabled for that branch; otherwise, if any " - + "matched entry is `push`, push-down may be forced for that branch, and " - + "the other aggregates in the same branch follow that branch-level decision."}) + description = "Test/debug hint for eager aggregation push-down. " + + "Format: `:=`, " + + "with multiple entries separated by `;`. " + + "Example: `sum:t1.a=push; sum:t2.a=nopush; count:*=push`. " + + "Note: entries are matched per aggregate-function key, but the effect " + + "is applied at the current candidate push-down branch/subtree rather " + + "than to one function independently. If any matched entry in the branch " + + "is `nopush`, push-down is disabled for that branch; otherwise, if any " + + "matched entry is `push`, push-down may be forced for that branch, and " + + "the other aggregates in the same branch follow that branch-level decision.") public String forceEagerAggHint = ""; private Map forceEagerAggHintMap = ImmutableMap.of(); @@ -2542,16 +2432,14 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = ENABLE_PAGE_CACHE, - description = {"控制是否启用 page cache。默认为 true。", - "Controls whether to use page cache. " - + "The default value is true."}, + description = "Controls whether to use page cache. " + + "The default value is true.", needForward = true) public boolean enablePageCache = true; @VarAttrDef.VarAttr( name = ENABLE_PARQUET_FILE_PAGE_CACHE, - description = {"控制是否启用 Parquet file page cache。默认为 true。", - "Controls whether to use Parquet file page cache. The default is true."}, + description = "Controls whether to use Parquet file page cache. The default is true.", needForward = true) public boolean enableParquetFilePageCache = true; @@ -2572,38 +2460,31 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = MAX_INITIAL_FILE_SPLIT_SIZE, - description = {"对于每个 table scan，最大文件分片初始大小。" - + "初始化使用 MAX_INITIAL_FILE_SPLIT_SIZE，一旦超过了 MAX_INITIAL_FILE_SPLIT_NUM，则使用 MAX_FILE_SPLIT_SIZE。", - "For each table scan, The maximum initial file split size. " - + "Initialize using MAX_INITIAL_FILE_SPLIT_SIZE," - + " and once MAX_INITIAL_FILE_SPLIT_NUM is exceeded, use MAX_FILE_SPLIT_SIZE instead."}, + description = "For each table scan, The maximum initial file split size. " + + "Initialize using MAX_INITIAL_FILE_SPLIT_SIZE," + + " and once MAX_INITIAL_FILE_SPLIT_NUM is exceeded, use MAX_FILE_SPLIT_SIZE instead.", needForward = true) public long maxInitialSplitSize = 32L * 1024L * 1024L; @VarAttrDef.VarAttr( name = MAX_FILE_SPLIT_SIZE, - description = {"对于每个 table scan，最大文件分片大小。" - + "初始化使用 MAX_INITIAL_FILE_SPLIT_SIZE，一旦超过了 MAX_INITIAL_FILE_SPLIT_NUM，则使用 MAX_FILE_SPLIT_SIZE。", - "For each table scan, the maximum initial file split size. " - + "Initialize using MAX_INITIAL_FILE_SPLIT_SIZE," - + " and once MAX_INITIAL_FILE_SPLIT_NUM is exceeded, use MAX_FILE_SPLIT_SIZE instead."}, + description = "For each table scan, the maximum initial file split size. " + + "Initialize using MAX_INITIAL_FILE_SPLIT_SIZE," + + " and once MAX_INITIAL_FILE_SPLIT_NUM is exceeded, use MAX_FILE_SPLIT_SIZE instead.", needForward = true) public long maxSplitSize = 64L * 1024L * 1024L; @VarAttrDef.VarAttr( name = MAX_INITIAL_FILE_SPLIT_NUM, - description = {"对于每个 table scan，最大文件分片初始数目。" - + "初始化使用 MAX_INITIAL_FILE_SPLIT_SIZE，一旦超过了 MAX_INITIAL_FILE_SPLIT_NUM，则使用 MAX_FILE_SPLIT_SIZE。", - "For each table scan, the maximum initial file split number. " - + "Initialize using MAX_INITIAL_FILE_SPLIT_SIZE," - + " and once MAX_INITIAL_FILE_SPLIT_NUM is exceeded, use MAX_FILE_SPLIT_SIZE instead."}, + description = "For each table scan, the maximum initial file split number. " + + "Initialize using MAX_INITIAL_FILE_SPLIT_SIZE," + + " and once MAX_INITIAL_FILE_SPLIT_NUM is exceeded, use MAX_FILE_SPLIT_SIZE instead.", needForward = true) public int maxInitialSplitNum = 200; @VarAttrDef.VarAttr( name = MAX_FILE_SPLIT_NUM, - description = {"在非 batch 模式下，每个 table scan 最大允许的 split 数量，防止产生过多 split 导致 OOM。", - "In non-batch mode, the maximum number of splits allowed per table scan to avoid OOM."}, + description = "In non-batch mode, the maximum number of splits allowed per table scan to avoid OOM.", needForward = true) public int maxFileSplitNum = 100000; @@ -2615,55 +2496,48 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = NUM_PARTITIONS_IN_BATCH_MODE, fuzzy = true, - description = {"如果分区数量超过阈值，BE 将通过 batch 方式获取 scan ranges。作用于 Hive、Hudi、MaxCompute 表。", - "If the number of partitions exceeds the threshold, scan ranges will be got through batch mode."}, + description = "If the number of partitions exceeds the threshold, scan ranges will be got through batch " + + "mode.", needForward = true) public int numPartitionsInBatchMode = 1024; @VarAttrDef.VarAttr( name = NUM_FILES_IN_BATCH_MODE, fuzzy = true, - description = {"如果文件数量超过阈值，BE 将通过 batch 方式获取 scan ranges", - "If the number of files exceeds the threshold, scan ranges will be got through batch mode."}, + description = "If the number of files exceeds the threshold, scan ranges will be got through batch mode.", needForward = true) public int numFilesInBatchMode = 1024; @VarAttrDef.VarAttr( name = FETCH_SPLITS_MAX_WAIT_TIME, - description = {"batch 方式中 BE 获取 splits 的最大等待时间", - "The max wait time of getting splits in batch mode."}, + description = "The max wait time of getting splits in batch mode.", needForward = true) public long fetchSplitsMaxWaitTime = 1000; @VarAttrDef.VarAttr( name = ENABLE_PARQUET_LAZY_MAT, fuzzy = true, - description = {"控制 parquet reader 是否启用延迟物化技术。默认为 true。", - "Controls whether to use lazy materialization technology in parquet reader. " - + "The default value is true."}, + description = "Controls whether to use lazy materialization technology in parquet reader. " + + "The default value is true.", needForward = true) public boolean enableParquetLazyMat = true; @VarAttrDef.VarAttr( name = ENABLE_ORC_LAZY_MAT, fuzzy = true, - description = {"控制 orc reader 是否启用延迟物化技术。默认为 true。", - "Controls whether to use lazy materialization technology in orc reader. " - + "The default value is true."}, + description = "Controls whether to use lazy materialization technology in orc reader. " + + "The default value is true.", needForward = true) public boolean enableOrcLazyMat = true; @VarAttrDef.VarAttr( name = ORC_TINY_STRIPE_THRESHOLD_BYTES, fuzzy = true, - description = {"在 orc 文件中如果一个 stripe 的字节大小小于`orc_tiny_stripe_threshold`," - + "我们认为该 stripe 为 tiny stripe。对于多个连续的 tiny stripe 我们会进行读取优化，即一次性读多个 tiny stripe." - + "如果你不想使用该优化，可以将该值设置为 0。默认为 8M。", - "In an orc file, if the byte size of a stripe is less than `orc_tiny_stripe_threshold`," - + "we consider the stripe to be a tiny stripe. For multiple consecutive tiny stripes," - + "we will perform read optimization, that is, read multiple tiny stripes at a time." - + "If you do not want to use this optimization, you can set this value to 0." - + "The default is 8M."}, + description = "In an orc file, if the byte size of a stripe is less than `orc_tiny_stripe_threshold`," + + "we consider the stripe to be a tiny stripe. For multiple consecutive tiny stripes," + + "we will perform read optimization, that is, read multiple tiny stripes at a time." + + "If you do not want to use this optimization, you can set this value to 0." + + "The default is 8M.", needForward = true, setter = "setOrcTinyStripeThresholdBytes") public long orcTinyStripeThresholdBytes = 8L * 1024L * 1024L; @@ -2671,12 +2545,10 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = ORC_ONCE_MAX_READ_BYTES, fuzzy = true, - description = {"在使用 tiny stripe 读取优化的时候，会对多个 tiny stripe 合并成一次 IO，" - + "该参数用来控制每次 IO 请求的最大字节大小。你不应该将值设置的小于`orc_tiny_stripe_threshold`。默认为 8M。", - "When using tiny stripe read optimization, multiple tiny stripes will be merged into one IO." - + "This parameter is used to control the maximum byte size of each IO request." - + "You should not set the value less than `orc_tiny_stripe_threshold`." - + "The default is 8M."}, + description = "When using tiny stripe read optimization, multiple tiny stripes will be merged into one IO." + + "This parameter is used to control the maximum byte size of each IO request." + + "You should not set the value less than `orc_tiny_stripe_threshold`." + + "The default is 8M.", needForward = true, setter = "setOrcOnceMaxReadBytes") public long orcOnceMaxReadBytes = 8L * 1024L * 1024L; @@ -2684,11 +2556,9 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = ORC_MAX_MERGE_DISTANCE_BYTES, fuzzy = true, - description = {"在使用 tiny stripe 读取优化的时候，由于 tiny stripe 并不一定连续。" - + "当两个 tiny stripe 之间距离大于该参数时，我们不会将其合并成一次 IO。默认为 1M。", - "When using tiny stripe read optimization, since tiny stripes are not necessarily continuous," - + "when the distance between two tiny stripes is greater than this parameter," - + "we will not merge them into one IO. The default value is 1M."}, + description = "When using tiny stripe read optimization, since tiny stripes are not necessarily continuous," + + "when the distance between two tiny stripes is greater than this parameter," + + "we will not merge them into one IO. The default value is 1M.", needForward = true, setter = "setOrcMaxMergeDistanceBytes") public long orcMaxMergeDistanceBytes = 1024L * 1024L; @@ -2696,53 +2566,46 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = ENABLE_PARQUET_FILTER_BY_MIN_MAX, fuzzy = true, - description = {"控制 parquet reader 是否启用 min-max 值过滤。默认为 true。", - "Controls whether to filter by min-max values in parquet reader. " - + "The default value is true."}, + description = "Controls whether to filter by min-max values in parquet reader. " + + "The default value is true.", needForward = true) public boolean enableParquetFilterByMinMax = true; @VarAttrDef.VarAttr( name = ENABLE_PARQUET_FILTER_BY_BLOOM_FILTER, fuzzy = true, - description = {"控制 parquet reader 是否启用 bloom filter 过滤。默认为 true。", - "Controls whether to filter by bloom filter in parquet reader. " - + "The default value is true."}, + description = "Controls whether to filter by bloom filter in parquet reader. " + + "The default value is true.", needForward = true) public boolean enableParquetFilterByBloomFilter = true; @VarAttrDef.VarAttr( name = ENABLE_ORC_FILTER_BY_MIN_MAX, - description = {"控制 orc reader 是否启用 min-max 值过滤。默认为 true。", - "Controls whether to filter by min-max values in orc reader. " - + "The default value is true."}, + description = "Controls whether to filter by min-max values in orc reader. " + + "The default value is true.", needForward = true) public boolean enableOrcFilterByMinMax = true; @VarAttrDef.VarAttr( name = ENABLE_EXPR_ZONEMAP_FILTER, fuzzy = true, - description = {"控制支持该变量的 scanner 是否启用表达式 ZoneMap 过滤。" - + "File Scanner V2 始终启用安全的表达式 ZoneMap 过滤。默认为 true。", - "Controls expression ZoneMap filtering in scanners that honor this variable. " - + "File Scanner V2 always enables safe expression ZoneMap filtering. " - + "The default value is true."}, + description = "Controls expression ZoneMap filtering in scanners that honor this variable. " + + "File Scanner V2 always enables safe expression ZoneMap filtering. " + + "The default value is true.", needForward = true) public boolean enableExprZonemapFilter = true; @VarAttrDef.VarAttr( name = CHECK_ORC_INIT_SARGS_SUCCESS, - description = {"是否检查 orc init sargs 是否成功。默认为 false。", - "Whether to check whether orc init sargs is successful. " - + "The default value is false."}, + description = "Whether to check whether orc init sargs is successful. " + + "The default value is false.", needForward = true) public boolean checkOrcInitSargsSuccess = false; @VarAttrDef.VarAttr( name = EXTERNAL_TABLE_ANALYZE_PART_NUM, - description = {"收集外表统计信息行数时选取的采样分区数，默认 -1 表示全部分区", - "Number of sample partition for collecting external table line number, " - + "default -1 means all partitions"}, + description = "Number of sample partition for collecting external table line number, " + + "default -1 means all partitions", needForward = false) public int externalTableAnalyzePartNum = -1; @@ -2755,37 +2618,31 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr(name = ENABLE_ORDERED_SCAN_RANGE_LOCATIONS) public boolean enableOrderedScanRangeLocations = false; - @VarAttrDef.VarAttr(name = CTE_INLINE_MODE, description = { - "CTE内联模式。<0:禁用; =0:仅当CTE体含UNION ALL且filter可消除部分分支时内联; >=1:CBO比较物化与内联", - "CTE inline mode. <0: disable; =0: only inline when CTE body contains UNION ALL " - + "and consumer filters can eliminate some union branches; " - + ">=1: both materialized and inlined alternatives are added to Memo for CBO." }) + @VarAttrDef.VarAttr(name = CTE_INLINE_MODE, description = "CTE inline mode. <0: disable; =0: only inline when CTE " + + "body contains UNION ALL " + + "and consumer filters can eliminate some union branches; " + + ">=1: both materialized and inlined alternatives are added to Memo for CBO.") public int cteInlineMode = 0; @VarAttrDef.VarAttr(name = ENABLE_ANALYZE_COMPLEX_TYPE_COLUMN) public boolean enableAnalyzeComplexTypeColumn = false; - @VarAttrDef.VarAttr(name = ENABLE_STRONG_CONSISTENCY, description = {"用以开启强一致读。Doris 默认支持同一个会话内的" - + "强一致性，即同一个会话内对数据的变更操作是实时可见的。如需要会话间的强一致读，则需将此变量设置为 true。", - "Used to enable strong consistent reading. By default, Doris supports strong consistency " - + "within the same session, that is, changes to data within the same session are visible in " - + "real time. If you want strong consistent reads between sessions, set this variable to true. " - }) + @VarAttrDef.VarAttr(name = ENABLE_STRONG_CONSISTENCY, description = "Used to enable strong consistent reading. By " + + "default, Doris supports strong consistency " + + "within the same session, that is, changes to data within the same session are visible in " + + "real time. If you want strong consistent reads between sessions, set this variable to true. ") public boolean enableStrongConsistencyRead = false; @VarAttrDef.VarAttr(name = PARALLEL_SYNC_ANALYZE_TASK_NUM) public int parallelSyncAnalyzeTaskNum = 2; @VarAttrDef.VarAttr(name = TRUNCATE_CHAR_OR_VARCHAR_COLUMNS, - description = {"是否按照表的 schema 来截断 char 或者 varchar 列。默认为 false。\n" - + "因为外表会存在表的 schema 中 char 或者 varchar 列的最大长度和底层 parquet 或者 orc 文件中的 schema 不一致" - + "的情况。此时开启改选项，会按照表的 schema 中的最大长度进行截断。", - "Whether to truncate char or varchar columns according to the table's schema. " - + "The default is true.\n" + description = "Whether to truncate char or varchar columns according to the table's schema. " + + "The default is true.\n" + "Because the maximum length of the char or varchar column in the schema of the table" - + " is inconsistent with the schema in the underlying parquet or orc file." + + " is inconsistent with the schema in the underlying parquet or orc file." + " At this time, if the option is turned on, it will be truncated according to the maximum length" - + " in the schema of the table."}, + + " in the schema of the table.", needForward = true) public boolean truncateCharOrVarcharColumns = false; @@ -2802,49 +2659,42 @@ public Map getForceEagerAggHintMap() { public boolean enablePreparedStmtAuditLog = false; @VarAttrDef.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD, - description = {"在 match_all 中求取多个倒排索引的交集时，如果最大的倒排索引中的总数是最小倒排索引中的总数的整数倍，" - + "则使用跳表来优化交集操作。", - "When intersecting multiple inverted indexes in match_all," + description = "When intersecting multiple inverted indexes in match_all," + " if the maximum total count of the largest inverted index" + " is a multiple of the minimum total count of the smallest inverted index," - + " use a skiplist to optimize the intersection."}) + + " use a skiplist to optimize the intersection.") public int invertedIndexConjunctionOptThreshold = 1000; @VarAttrDef.VarAttr(name = INVERTED_INDEX_MAX_EXPANSIONS, affectQueryResultInExecution = true, - description = {"这个参数用来限制查询时扩展的词项（terms）的数量，以此来控制查询的性能", - "This parameter is used to limit the number of term expansions during a query," - + " thereby controlling query performance"}) + description = "This parameter is used to limit the number of term expansions during a query," + + " thereby controlling query performance") public int invertedIndexMaxExpansions = 50; @VarAttrDef.VarAttr(name = INVERTED_INDEX_SKIP_THRESHOLD, - description = {"在倒排索引中如果预估命中量占比总量超过百分比阈值，则跳过索引直接进行匹配。", - "In the inverted index," - + " if the estimated hit ratio exceeds the percentage threshold of the total amount, " - + " then skip the index and proceed directly to matching."}) + description = "In the inverted index," + + " if the estimated hit ratio exceeds the percentage threshold of the total amount, " + + " then skip the index and proceed directly to matching.") public int invertedIndexSkipThreshold = 50; @VarAttrDef.VarAttr(name = INVERTED_INDEX_COMPATIBLE_READ, - description = {"兼容读取倒排索引，用于在 x86 和 arm 集群之间读取旧版本索引文件。", - "Compatible read for inverted index between x86 and arm, " - + "used to read old version index file from x86 in arm cluster" - + "or read old version index file from arm in x86 cluster"}) + description = "Compatible read for inverted index between x86 and arm, " + + "used to read old version index file from x86 in arm cluster" + + "or read old version index file from arm in x86 cluster") public boolean invertedIndexCompatibleRead = false; @VarAttrDef.VarAttr(name = SQL_DIALECT, needForward = true, checker = "checkSqlDialect", - description = {"解析 sql 使用的方言", "The dialect used to parse sql."}, + description = "The dialect used to parse sql.", affectQueryResultInPlan = true ) public String sqlDialect = "doris"; @VarAttrDef.VarAttr(name = RETRY_ORIGIN_SQL_ON_CONVERT_FAIL, needForward = true, - description = {"当转换后的 SQL 解析失败时，是否重试原始 SQL", - "Enable retrying original SQL when converted SQL parsing fails."}) + description = "Enable retrying original SQL when converted SQL parsing fails.") public boolean retryOriginSqlOnConvertFail = false; @VarAttrDef.VarAttr(name = SERDE_DIALECT, needForward = true, checker = "checkSerdeDialect", - description = {"返回给 MySQL 客户端时各数据类型的输出格式方言", - "The output format dialect of each data type returned to the MySQL client."}, + description = "The output format dialect of each data type returned to the MySQL client.", options = {"doris", "presto", "trino"}, affectQueryResultInPlan = true, affectQueryResultInExecution = true ) @@ -2853,232 +2703,195 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = true) public boolean enableUniqueKeyPartialUpdate = false; - @VarAttrDef.VarAttr(name = PARTIAL_UPDATE_NEW_KEY_BEHAVIOR, needForward = true, description = { - "用于设置部分列更新中对于新插入的行的行为", - "Used to set the behavior for newly inserted rows in partial update." - }, checker = "checkPartialUpdateNewKeyBehavior", options = {"APPEND", "ERROR"}) + @VarAttrDef.VarAttr(name = PARTIAL_UPDATE_NEW_KEY_BEHAVIOR, needForward = true, description = "Used to set the " + + "behavior for newly inserted rows in partial update.", + checker = "checkPartialUpdateNewKeyBehavior", options = {"APPEND", "ERROR"}) public String partialUpdateNewKeyPolicy = "APPEND"; @VarAttrDef.VarAttr(name = ENABLE_AUTO_ANALYZE, - description = {"该参数控制是否开启自动收集", "Set false to disable auto analyze"}, + description = "Set false to disable auto analyze", flag = VarAttrDef.GLOBAL) public volatile boolean enableAutoAnalyze = true; @VarAttrDef.VarAttr(name = FORCE_SAMPLE_ANALYZE, needForward = true, - description = {"是否将 full analyze 自动转换成 sample analyze", "Set true to force sample analyze"}, + description = "Set true to force sample analyze", flag = VarAttrDef.GLOBAL) public boolean forceSampleAnalyze = Config.force_sample_analyze; @VarAttrDef.VarAttr(name = ENABLE_AUTO_ANALYZE_INTERNAL_CATALOG, - description = {"临时参数，收否自动收集所有内表", "Temp variable, enable to auto collect all OlapTable."}, + description = "Temp variable, enable to auto collect all OlapTable.", flag = VarAttrDef.GLOBAL) public boolean enableAutoAnalyzeInternalCatalog = true; @VarAttrDef.VarAttr(name = ENABLE_PARTITION_ANALYZE, - description = {"临时参数，收否收集分区级别统计信息", "Temp variable, enable to collect partition level statistics."}, + description = "Temp variable, enable to collect partition level statistics.", flag = VarAttrDef.GLOBAL) public boolean enablePartitionAnalyze = false; @VarAttrDef.VarAttr(name = AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD, - description = {"参与自动收集的最大表宽度，列数多于这个参数的表不参与自动收集", - "Maximum table width to enable auto analyze, " - + "table with more columns than this value will not be auto analyzed."}, + description = "Maximum table width to enable auto analyze, " + + "table with more columns than this value will not be auto analyzed.", flag = VarAttrDef.GLOBAL) public int autoAnalyzeTableWidthThreshold = 300; @VarAttrDef.VarAttr(name = AUTO_ANALYZE_START_TIME, needForward = true, checker = "checkAnalyzeTimeFormat", - description = {"该参数定义自动 ANALYZE 例程的开始时间", - "This parameter defines the start time for the automatic ANALYZE routine."}, + description = "This parameter defines the start time for the automatic ANALYZE routine.", flag = VarAttrDef.GLOBAL) public String autoAnalyzeStartTime = "00:00:00"; @VarAttrDef.VarAttr(name = AUTO_ANALYZE_END_TIME, needForward = true, checker = "checkAnalyzeTimeFormat", - description = {"该参数定义自动 ANALYZE 例程的结束时间", - "This parameter defines the end time for the automatic ANALYZE routine."}, + description = "This parameter defines the end time for the automatic ANALYZE routine.", flag = VarAttrDef.GLOBAL) public String autoAnalyzeEndTime = "23:59:59"; @VarAttrDef.VarAttr(name = IGNORE_RUNTIME_FILTER_IDS, - description = {"在 IGNORE_RUNTIME_FILTER_IDS 列表中的 runtime filter 将不会被生成", - "the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list will not be generated"}) + description = "the runtime filter id in IGNORE_RUNTIME_FILTER_IDS list will not be generated") public String ignoreRuntimeFilterIds = ""; - @VarAttrDef.VarAttr(name = STATS_INSERT_MERGE_ITEM_COUNT, flag = VarAttrDef.GLOBAL, description = { - "控制统计信息相关 INSERT 攒批数量", "Controls the batch size for stats INSERT merging." - } + @VarAttrDef.VarAttr(name = STATS_INSERT_MERGE_ITEM_COUNT, flag = VarAttrDef.GLOBAL, description = "Controls the " + + "batch size for stats INSERT merging." ) public int statsInsertMergeItemCount = 200; - @VarAttrDef.VarAttr(name = HUGE_TABLE_DEFAULT_SAMPLE_ROWS, flag = VarAttrDef.GLOBAL, description = { - "定义开启开启大表自动 sample 后，对大表的采样比例", - "This defines the number of sample percent for large tables when automatic sampling for" - + "large tables is enabled" - - }) + @VarAttrDef.VarAttr(name = HUGE_TABLE_DEFAULT_SAMPLE_ROWS, flag = VarAttrDef.GLOBAL, description = "This defines " + + "the number of sample percent for large tables when automatic sampling for" + + "large tables is enabled") public long hugeTableDefaultSampleRows = 4194304; @VarAttrDef.VarAttr(name = HUGE_TABLE_LOWER_BOUND_SIZE_IN_BYTES, flag = VarAttrDef.GLOBAL, - description = { - "大小超过该值的表将会自动通过采样收集统计信息", - "This defines the lower size bound for large tables. " - + "When enable_auto_sample is enabled, tables" - + "larger than this value will automatically collect " - + "statistics through sampling"}) + description = "This defines the lower size bound for large tables. " + + "When enable_auto_sample is enabled, tables" + + "larger than this value will automatically collect " + + "statistics through sampling") public long hugeTableLowerBoundSizeInBytes = 0; @VarAttrDef.VarAttr(name = HUGE_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS, flag = VarAttrDef.GLOBAL, - description = {"控制对大表的自动 ANALYZE 的最小时间间隔，" - + "在该时间间隔内大小超过 huge_table_lower_bound_size_in_bytes 的表仅 ANALYZE 一次", - "This controls the minimum time interval for automatic ANALYZE on large tables." - + "Within this interval," - + "tables larger than huge_table_lower_bound_size_in_bytes are analyzed only once."}) + description = "This controls the minimum time interval for automatic ANALYZE on large tables." + + "Within this interval," + + "tables larger than huge_table_lower_bound_size_in_bytes are analyzed only once.") public long hugeTableAutoAnalyzeIntervalInMillis = TimeUnit.HOURS.toMillis(0); @VarAttrDef.VarAttr(name = EXTERNAL_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS, flag = VarAttrDef.GLOBAL, - description = {"控制对外表的自动 ANALYZE 的最小时间间隔，在该时间间隔内的外表仅 ANALYZE 一次", - "This controls the minimum time interval for automatic ANALYZE on external tables." - + "Within this interval, external tables are analyzed only once."}) + description = "This controls the minimum time interval for automatic ANALYZE on external tables." + + "Within this interval, external tables are analyzed only once.") public long externalTableAutoAnalyzeIntervalInMillis = TimeUnit.HOURS.toMillis(24); @VarAttrDef.VarAttr(name = TABLE_STATS_HEALTH_THRESHOLD, flag = VarAttrDef.GLOBAL, - description = {"取值在 0-100 之间，当自上次统计信息收集操作之后" - + "数据更新量达到 (100 - table_stats_health_threshold)% ，认为该表的统计信息已过时", - "The value should be between 0 and 100. When the data update quantity " - + "exceeds (100 - table_stats_health_threshold)% since the last " - + "statistics collection operation, the statistics for this table are" - + "considered outdated."}) + description = "The value should be between 0 and 100. When the data update quantity " + + "exceeds (100 - table_stats_health_threshold)% since the last " + + "statistics collection operation, the statistics for this table are" + + "considered outdated.") public int tableStatsHealthThreshold = 90; @VarAttrDef.VarAttr(name = PARTITION_SAMPLE_COUNT, flag = VarAttrDef.GLOBAL, - description = { - "大分区表采样的分区数上限", - "The upper limit of the number of partitions for sampling large partitioned tables.\n"}) + description = "The upper limit of the number of partitions for sampling large partitioned tables.\n") public int partitionSampleCount = 30; @VarAttrDef.VarAttr(name = PARTITION_SAMPLE_ROW_COUNT, flag = VarAttrDef.GLOBAL, - description = { - "大分区表采样的行数上限", - "The upper limit of the number of rows for sampling large partitioned tables.\n"}) + description = "The upper limit of the number of rows for sampling large partitioned tables.\n") public long partitionSampleRowCount = 3_000_000_000L; @VarAttrDef.VarAttr(name = FETCH_HIVE_ROW_COUNT_SYNC, fuzzy = true, - description = {"同步获取 Hive 外表行数", "Fetch Hive external table row count synchronously"}) + description = "Fetch Hive external table row count synchronously") public boolean fetchHiveRowCountSync = true; @VarAttrDef.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward = true, - description = {"是否开启基于结构信息的物化视图透明改写", - "Whether to enable materialized view rewriting based on struct info"}) + description = "Whether to enable materialized view rewriting based on struct info") public boolean enableMaterializedViewRewrite = true; @VarAttrDef.VarAttr(name = PRE_MATERIALIZED_VIEW_REWRITE_STRATEGY, needForward = true, fuzzy = true, - description = {"在 RBO 阶段基于结构信息的物化视图透明改写的策略，FORCE_IN_RBO：强制在 RBO 阶段透明改写，" - + "TRY_IN_RBO：如果在 NEED_PRE_REWRITE_RULE_TYPES 中的规则改写成功了，那么就会尝试在 RBO 阶段透明改写" - + "NOT_IN_RBO：不尝试在 RBO 阶段改写，只在 CBO 阶段改写", - "Whether to enable pre materialized view rewriting based on struct info," - + "FORCE_IN_RBO : Force transparent rewriting in the RBO phase," - + "TRY_IN_RBO : Attempt transparent rewriting in the RBO phase " - + "if rules in NEED_PRE_REWRITE_RULE_TYPES, " - + "NOT_IN_RBO : Do not attempt rewriting in the RBO phase; apply only during the CBO phase" - }) + description = "Whether to enable pre materialized view rewriting based on struct info," + + "FORCE_IN_RBO : Force transparent rewriting in the RBO phase," + + "TRY_IN_RBO : Attempt transparent rewriting in the RBO phase " + + "if rules in NEED_PRE_REWRITE_RULE_TYPES, " + + "NOT_IN_RBO : Do not attempt rewriting in the RBO phase; apply only during the CBO phase") public String preMaterializedViewRewriteStrategy = "TRY_IN_RBO"; @VarAttrDef.VarAttr(name = ALLOW_MODIFY_MATERIALIZED_VIEW_DATA, needForward = true, - description = {"是否允许修改物化视图的数据", - "Is it allowed to modify the data of the materialized view"}) + description = "Is it allowed to modify the data of the materialized view") public boolean allowModifyMaterializedViewData = false; @VarAttrDef.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS, needForward = true, - description = {"查询时，当物化视图存在无法实时感知数据的外表时，是否开启基于结构信息的物化视图透明改写", - ""}) + description = "") public boolean enableMaterializedViewRewriteWhenBaseTableUnawareness = false; @VarAttrDef.VarAttr(name = MATERIALIZED_VIEW_REWRITE_SUCCESS_CANDIDATE_NUM, needForward = true, - description = {"异步物化视图透明改写成功的结果集合，允许参与到 CBO 候选的最大数量", - "The max candidate num which participate in CBO when using asynchronous materialized views"}) + description = "The max candidate num which participate in CBO when using asynchronous materialized views") public int materializedViewRewriteSuccessCandidateNum = 3; @VarAttrDef.VarAttr(name = ENABLE_DML_MATERIALIZED_VIEW_REWRITE, needForward = true, - description = {"DML 时，是否开启基于结构信息的物化视图透明改写", - "Whether to enable materialized view rewriting based on struct info"}) + description = "Whether to enable materialized view rewriting based on struct info") public boolean enableDmlMaterializedViewRewrite = true; @VarAttrDef.VarAttr(name = ENABLE_DML_MATERIALIZED_VIEW_REWRITE_WHEN_BASE_TABLE_UNAWARENESS, needForward = true, - description = {"DML 时，当物化视图存在无法实时感知数据的外表时，是否开启基于结构信息的物化视图透明改写", - ""}) + description = "") public boolean enableDmlMaterializedViewRewriteWhenBaseTableUnawareness = false; @VarAttrDef.VarAttr(name = MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT, needForward = true, - description = {"透明改写过程中，relation mapping 最大允许数量，如果超过，进行截取", - "During transparent rewriting, relation mapping specifies the maximum allowed number. " - + "If the number exceeds the allowed number, the number is intercepted"}) + description = "During transparent rewriting, relation mapping specifies the maximum allowed number. " + + "If the number exceeds the allowed number, the number is intercepted") public int materializedViewRelationMappingMaxCount = 8; @VarAttrDef.VarAttr(name = ENABLE_MATERIALIZED_VIEW_UNION_REWRITE, needForward = true, - description = {"当物化视图不足以提供查询的全部数据时，是否允许基表和物化视图 union 来响应查询", - "When the materialized view is not enough to provide all the data for the query, " - + "whether to allow the union of the base table and the materialized view to " - + "respond to the query"}, varType = VariableAnnotation.REMOVED) + description = "When the materialized view is not enough to provide all the data for the query, " + + "whether to allow the union of the base table and the materialized view to " + + "respond to the query", varType = VariableAnnotation.REMOVED) public boolean enableMaterializedViewUnionRewrite = true; @VarAttrDef.VarAttr(name = ENABLE_MATERIALIZED_VIEW_NEST_REWRITE, needForward = true, - description = {"是否允许嵌套物化视图改写", - "Whether enable materialized view nest rewrite"}) + description = "Whether enable materialized view nest rewrite") public boolean enableMaterializedViewNestRewrite = false; @VarAttrDef.VarAttr(name = MATERIALIZED_VIEW_REWRITE_DURATION_THRESHOLD_MS, needForward = true, - description = {"物化视图透明改写允许的最长耗时，超过此时长不再进行透明改写", - "The maximum duration allowed for transparent rewriting of materialized views; " - + "if this duration is exceeded, transparent rewriting will no longer be performed."}) + description = "The maximum duration allowed for transparent rewriting of materialized views; " + + "if this duration is exceeded, transparent rewriting will no longer be performed.") public long materializedViewRewriteDurationThresholdMs = 1000L; @VarAttrDef.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true, - description = {"建表时创建分区的最大数量", - "The maximum number of partitions created during table creation"}) + description = "The maximum number of partitions created during table creation") public int createTablePartitionMaxNum = 10000; @VarAttrDef.VarAttr(name = HIVE_PARQUET_USE_COLUMN_NAMES, affectQueryResultInExecution = true, - description = {"默认情况下按名称访问 Parquet 列。将此属性设置为“false”可按 Hive 表定义中的序号位置访问列。", - "Access Parquet columns by name by default. Set this property to `false` to access columns " - + "by their ordinal position in the Hive table definition."}) + description = "Access Parquet columns by name by default. Set this property to `false` to access columns " + + "by their ordinal position in the Hive table definition.") public boolean hiveParquetUseColumnNames = true; @VarAttrDef.VarAttr(name = HIVE_ORC_USE_COLUMN_NAMES, affectQueryResultInExecution = true, - description = {"默认情况下按照 Hive 表定义中的序号位置访问列。将此属性设置为“true”可按名称访问 Orc 列 。", - "By default, columns are accessed based on their ordinal position in the Hive table definition." - + " Set this property to `true` to access ORC columns by name."}) + description = "By default, columns are accessed based on their ordinal position in the Hive table " + + "definition." + + " Set this property to `true` to access ORC columns by name.") public boolean hiveOrcUseColumnNames = false; @VarAttrDef.VarAttr(name = KEEP_CARRIAGE_RETURN, - description = {"在同时处理\r和\r\n作为 CSV 的行分隔符时，是否保留\r", - "When processing both \\n and \\r\\n as CSV line separators, should \\r be retained?"}) + description = "When processing both \\n and \\r\\n as CSV line separators, should \\r be retained?") public boolean keepCarriageReturn = false; @VarAttrDef.VarAttr(name = EXCHANGE_MULTI_BLOCKS_BYTE_SIZE, - description = {"Enable exchange to send multiple blocks in one RPC. Default is 256KB. A negative" - + " value disables multi-block exchange."}) + description = "Enable exchange to send multiple blocks in one RPC. Default is 256KB. A negative" + + " value disables multi-block exchange.") public int exchangeMultiBlocksByteSize = 256 * 1024; @VarAttrDef.VarAttr(name = FORCE_JNI_SCANNER, fuzzy = true, - description = {"强制使用 jni 方式读取外表", "Force the use of jni mode to read external table"}) + description = "Force the use of jni mode to read external table") private boolean forceJniScanner = false; @VarAttrDef.VarAttr(name = ENABLE_PAIMON_CPP_READER, fuzzy = true, - description = {"Paimon 非原生文件读取使用 paimon-cpp", "Use paimon-cpp for non-native Paimon reads"}) + description = "Use paimon-cpp for non-native Paimon reads") private boolean enablePaimonCppReader = false; @VarAttrDef.VarAttr(name = ENABLE_COUNT_PUSH_DOWN_FOR_EXTERNAL_TABLE, fuzzy = true, - description = {"对外表启用 count(*) 下推优化", "enable count(*) pushdown optimization for external table"}) + description = "enable count(*) pushdown optimization for external table") private boolean enableCountPushDownForExternalTable = true; @VarAttrDef.VarAttr(name = MINIMUM_OPERATOR_MEMORY_REQUIRED_KB, needForward = true, - description = {"一个算子运行需要的最小的内存大小", - "The minimum memory required to be used by an operator, if not meet, the operator will not run"}) + description = "The minimum memory required to be used by an operator, if not meet, the operator will not " + + "run") public int minimumOperatorMemoryRequiredKB = 32000; public static final String IGNORE_RUNTIME_FILTER_IDS = "ignore_runtime_filter_ids"; @@ -3089,44 +2902,37 @@ public Map getForceEagerAggHintMap() { @VarAttrDef.VarAttr( name = ENABLE_EXTERNAL_TABLE_BATCH_MODE, fuzzy = true, - description = {"使能外表的 batch mode 功能", "Enable the batch mode function of the external table."}, + description = "Enable the batch mode function of the external table.", needForward = true) public boolean enableExternalTableBatchMode = true; @VarAttrDef.VarAttr( name = ENABLE_MC_LIMIT_SPLIT_OPTIMIZATION, fuzzy = true, - description = {"开启 MaxCompute 表 LIMIT 查询的 split 优化。当查询仅包含分区等值条件且带有 LIMIT 时，" - + "使用 row_offset 策略减少 split 数量以加速查询。", - "Enable split optimization for LIMIT queries on MaxCompute tables. " + description = "Enable split optimization for LIMIT queries on MaxCompute tables. " + "When the query contains only partition equality predicates with LIMIT, " - + "use row_offset strategy to reduce split count for faster query execution."}, + + "use row_offset strategy to reduce split count for faster query execution.", needForward = true) public boolean enableMcLimitSplitOptimization = false; @VarAttrDef.VarAttr(name = SKEW_REWRITE_AGG_BUCKET_NUM, needForward = true, - description = {"bucketNum 参数控制 count(distinct) 倾斜优化的数据分布。决定不同值在 worker 间的分配方式，" - + "值越大越能处理极端倾斜但增加 shuffle 开销，值越小网络开销越低但可能无法完全解决倾斜。", - "The bucketNum parameter controls data distribution for skew optimization " - + "in count(distinct) queries. Determines how distinct values " - + "are distributed across workers to avoid data skew. " - + "Larger values better handle extreme skew but increase shuffle overhead. " - + "Smaller values reduce network traffic but may not fully resolve skew. " - }, checker = "checkSkewRewriteAggBucketNum") + description = "The bucketNum parameter controls data distribution for skew optimization " + + "in count(distinct) queries. Determines how distinct values " + + "are distributed across workers to avoid data skew. " + + "Larger values better handle extreme skew but increase shuffle overhead. " + + "Smaller values reduce network traffic but may not fully resolve skew. ", + checker = "checkSkewRewriteAggBucketNum") public int skewRewriteAggBucketNum = 1024; - @VarAttrDef.VarAttr(name = AGG_SHUFFLE_USE_PARENT_KEY, description = { - "在聚合算子进行 shuffle 时，是否使用父节点的分组键进行 shuffle", - "Whether to use the parent node's grouping key for shuffling during the aggregation operator" - }, needForward = false) + @VarAttrDef.VarAttr(name = AGG_SHUFFLE_USE_PARENT_KEY, description = "Whether to use the parent node's grouping " + + "key for shuffling during the aggregation operator", needForward = false) public boolean aggShuffleUseParentKey = true; @VarAttrDef.VarAttr(name = ENABLE_SHUFFLE_KEY_PRUNE) public boolean enableShuffleKeyPrune = true; @VarAttrDef.VarAttr(name = ENABLE_PREFER_CACHED_ROWSET, needForward = false, - description = {"是否启用 prefer cached rowset 功能", - "Whether to enable prefer cached rowset feature"}) + description = "Whether to enable prefer cached rowset feature") public boolean enablePreferCachedRowset = false; @VarAttrDef.VarAttr(name = QUERY_FRESHNESS_TOLERANCE_MS, needForward = false) @@ -3137,74 +2943,57 @@ public void setSkewRewriteAggBucketNum(int num) { } @VarAttrDef.VarAttr(name = ENABLE_STRICT_CAST, - description = {"cast 使用严格模式", "Use strict mode for cast"}, affectQueryResultInPlan = true) + description = "Use strict mode for cast", affectQueryResultInPlan = true) public boolean enableStrictCast = false; - @VarAttrDef.VarAttr(name = MULTI_DISTINCT_STRATEGY, description = {"用于控制在包含多个 DISTINCT 函数的 SQL 查询中所采用的" - + "执行策略。默认值为 0，表示由系统自动选择最优策略；设为 1 表示强制使用 MultiDistinct 方式处理；" - + "设为 2 表示强制采用 CTE 拆分方式执行。", - "Used to control the execution strategy used in SQL queries containing multiple DISTINCT " - + "functions. The default value is 0, which means that the system automatically selects " - + "the optimal strategy; setting it to 1 means forcing the use of MultiDistinct processing;" - + " setting it to 2 means forcing the use of CTE splitting execution"}, + @VarAttrDef.VarAttr(name = MULTI_DISTINCT_STRATEGY, description = "Used to control the execution strategy used in " + + "SQL queries containing multiple DISTINCT " + + "functions. The default value is 0, which means that the system automatically selects " + + "the optimal strategy; setting it to 1 means forcing the use of MultiDistinct processing;" + + " setting it to 2 means forcing the use of CTE splitting execution", checker = "checkMultiDistinctStrategy") public int multiDistinctStrategy = 0; - @VarAttrDef.VarAttr(name = AGG_PHASE, description = {"用于控制聚合查询的执行阶段划分策略。默认值为 0，" - + "表示由系统自动选择最优执行阶段；设为 1 至 4 之间的值则表示强制指定使用对应 1 至 4 阶段进行聚合计算。", - "Controls the execution phase strategy for aggregate queries. The default value is 0," - + "which means the system automatically selects the optimal execution phase. Setting this value" - + "between 1 and 4 forces the use of phases 1 to 4 for aggregate calculations."}, + @VarAttrDef.VarAttr(name = AGG_PHASE, description = "Controls the execution phase strategy for aggregate queries. " + + "The default value is 0," + + "which means the system automatically selects the optimal execution phase. Setting this value" + + "between 1 and 4 forces the use of phases 1 to 4 for aggregate calculations.", checker = "checkAggPhase") public int aggPhase = 0; - @VarAttrDef.VarAttr(name = ENABLE_BUCKETED_HASH_AGG, needForward = true, description = { - "是否启用 bucketed hash aggregation 优化。该优化在单 BE 场景下将两阶段聚合融合为单个算子，" - + "消除 Exchange 开销和序列化/反序列化成本。默认开启。", - "Whether to enable bucketed hash aggregation optimization. This optimization fuses two-phase " - + "aggregation into a single operator on single-BE deployments, eliminating exchange overhead " - + "and serialization/deserialization costs. Enabled by default."}) + @VarAttrDef.VarAttr(name = ENABLE_BUCKETED_HASH_AGG, needForward = true, description = "Whether to enable bucketed " + + "hash aggregation optimization. This optimization fuses two-phase " + + "aggregation into a single operator on single-BE deployments, eliminating exchange overhead " + + "and serialization/deserialization costs. Enabled by default.") public boolean enableBucketedHashAgg = true; - @VarAttrDef.VarAttr(name = BUCKETED_AGG_MIN_INPUT_ROWS, fuzzy = true, needForward = true, description = { - "bucketed hash aggregation 要求的最小输入行数。当估算输入行数小于此阈值时，" - + "数据量太小，256-bucket two-level hash table 的初始化和 merge 开销大于收益，" - + "不生成 bucketed agg 候选计划。设为 0 表示不限制。默认 100000。", - "Minimum estimated input rows required for bucketed hash aggregation. When estimated input " - + "rows are below this threshold, the data volume is too small for the 256-bucket two-level " - + "hash table overhead to be worthwhile. Set to 0 to disable this check. Default 100000."}) + @VarAttrDef.VarAttr(name = BUCKETED_AGG_MIN_INPUT_ROWS, fuzzy = true, needForward = true, description = "Minimum " + + "estimated input rows required for bucketed hash aggregation. When estimated input " + + "rows are below this threshold, the data volume is too small for the 256-bucket two-level " + + "hash table overhead to be worthwhile. Set to 0 to disable this check. Default 100000.") public long bucketedAggMinInputRows = 100000; - @VarAttrDef.VarAttr(name = BUCKETED_AGG_MAX_GROUP_KEYS, needForward = true, description = { - "bucketed hash aggregation 允许的最大估算分组数（key 数量）。当估算分组数超过此阈值时，" - + "merge 阶段需要合并大量 key，开销会超过 bucketed agg 带来的收益。" - + "类似于 ClickHouse 的 group_by_two_level_threshold。设为 0 表示不限制。默认 0", - "Maximum estimated number of group keys for bucketed hash aggregation. When the estimated " - + "number of groups exceeds this threshold, the merge phase cost of combining large numbers " - + "of keys outweighs the benefit. Similar to ClickHouse's group_by_two_level_threshold. " - + "Set to 0 to disable this check. Default 0."}) + @VarAttrDef.VarAttr(name = BUCKETED_AGG_MAX_GROUP_KEYS, needForward = true, description = "Maximum estimated " + + "number of group keys for bucketed hash aggregation. When the estimated " + + "number of groups exceeds this threshold, the merge phase cost of combining large numbers " + + "of keys outweighs the benefit. Similar to ClickHouse's group_by_two_level_threshold. " + + "Set to 0 to disable this check. Default 0.") public long bucketedAggMaxGroupKeys = 0; - @VarAttrDef.VarAttr(name = BUCKETED_AGG_HIGH_CARD_THRESHOLD, needForward = true, description = { - "bucketed hash aggregation 的高基数阈值比例。当任意 GROUP BY 列的 NDV 超过" - + "输入行数 * 该阈值，或聚合输出行数超过输入行数 * 该阈值时，跳过 bucketed agg。" - + "取值范围 (0, 1.0]。默认 0.3。", - "High-cardinality ratio threshold for bucketed hash aggregation. When any GROUP BY key's NDV " - + "exceeds input rows * threshold, or aggregation output rows exceed input rows * threshold, " - + "bucketed agg is skipped. Range (0, 1.0]. Default 0.3."}) + @VarAttrDef.VarAttr(name = BUCKETED_AGG_HIGH_CARD_THRESHOLD, needForward = true, description = "High-cardinality " + + "ratio threshold for bucketed hash aggregation. When any GROUP BY key's NDV " + + "exceeds input rows * threshold, or aggregation output rows exceed input rows * threshold, " + + "bucketed agg is skipped. Range (0, 1.0]. Default 0.3.") public double bucketedAggHighCardThreshold = 0.3; - @VarAttrDef.VarAttr(name = MERGE_IO_READ_SLICE_SIZE_BYTES, description = { - "调整 READ_SLICE_SIZE 大小，降低 Merge IO 读放大影响", - "Make the READ_SLICE_SIZE variable configurable to reduce the impact caused by read amplification."}) + @VarAttrDef.VarAttr(name = MERGE_IO_READ_SLICE_SIZE_BYTES, description = "Make the READ_SLICE_SIZE variable " + + "configurable to reduce the impact caused by read amplification.") public int mergeReadSliceSizeBytes = 8388608; @VarAttrDef.VarAttr(name = FILE_CACHE_QUERY_LIMIT_PERCENT, needForward = true, checker = "checkFileCacheQueryLimitPercent", - description = {"限制用户的单个查询能使用的 FILE_CACHE 比例 " - + "（用户设置，取值范围 1 到 Config.file_cache_query_limit_max_percent）。", - "Limit the FILE_CACHE percent that a single query of a user can use " - + "(set by user via session variables, range: 1 to Config.file_cache_query_limit_max_percent)."}) + description = "Limit the FILE_CACHE percent that a single query of a user can use " + + "(set by user via session variables, range: 1 to Config.file_cache_query_limit_max_percent).") public int fileCacheQueryLimitPercent = -1; public void checkFileCacheQueryLimitPercent(String fileCacheQueryLimitPercentStr) { @@ -3217,11 +3006,9 @@ public void checkFileCacheQueryLimitPercent(String fileCacheQueryLimitPercentStr } @VarAttrDef.VarAttr(name = FILE_CACHE_QUERY_LIMIT_BYTES, needForward = true, - description = {"单个查询在每个 BE 上最多允许 read-through 写入 file cache 的远端 scan bytes。" - + "< 0 表示关闭，= 0 表示查询开始即不写 file cache，> 0 表示达到阈值后不写 file cache。", - "Maximum remote scan bytes allowed to write file cache per query on each BE. " - + "< 0 disables it, = 0 disables file cache writes from query start, " - + "> 0 disables file cache writes after the threshold is reached."}) + description = "Maximum remote scan bytes allowed to write file cache per query on each BE. " + + "< 0 disables it, = 0 disables file cache writes from query start, " + + "> 0 disables file cache writes after the threshold is reached.") public long fileCacheQueryLimitBytes = -1; public void setAggPhase(int phase) { @@ -3306,19 +3093,15 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { } @VarAttrDef.VarAttr(name = IGNORE_SHAPE_NODE, - description = {"'explain shape plan' 命令中忽略的 PlanNode 类型", - "the plan node type which is ignored in 'explain shape plan' command"}) + description = "the plan node type which is ignored in 'explain shape plan' command") public String ignoreShapePlanNodes = ""; @VarAttrDef.VarAttr(name = DETAIL_SHAPE_NODES, needForward = true, setter = "setDetailShapePlanNodes", - description = {"'explain shape plan' 命令中显示详细信息的 PlanNode 类型", - "the plan node type show detail in 'explain shape plan' command"}) + description = "the plan node type show detail in 'explain shape plan' command") public String detailShapePlanNodes = ""; - @VarAttrDef.VarAttr(name = ENABLE_EXPLAIN_NONE, needForward = true, description = { - "执行 explain 命令，但不打印 explain 结果", - "execute explain command and return nothing" - }) + @VarAttrDef.VarAttr(name = ENABLE_EXPLAIN_NONE, needForward = true, description = "execute explain command and " + + "return nothing") public boolean enableExplainNone = false; private Set detailShapePlanNodesSet = ImmutableSet.of(); @@ -3333,30 +3116,26 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { this.detailShapePlanNodes = detailShapePlanNodes; } - @VarAttrDef.VarAttr(name = ENABLE_DECIMAL256, needForward = true, description = { "控制是否在计算过程中使用 Decimal256 类型", - "Set to true to enable Decimal256 type" }, affectQueryResultInPlan = true) + @VarAttrDef.VarAttr(name = ENABLE_DECIMAL256, needForward = true, description = "Set to true to enable Decimal256 " + + "type", affectQueryResultInPlan = true) public boolean enableDecimal256 = false; @VarAttrDef.VarAttr(name = FALLBACK_OTHER_REPLICA_WHEN_FIXED_CORRUPT, needForward = true, - description = { "当开启 use_fix_replica 时遇到故障，是否漂移到其他健康的副本", - "use other health replica when the use_fix_replica meet error" }) + description = "use other health replica when the use_fix_replica meet error") public boolean fallbackOtherReplicaWhenFixedCorrupt = false; public static final String FE_DEBUG = "fe_debug"; @VarAttrDef.VarAttr(name = FE_DEBUG, needForward = true, fuzzy = true, - description = {"when set true, FE will throw exceptions instead swallow them. This is used for test", - "when set true, FE will throw exceptions instead swallow them. This is used for test"}) + description = "when set true, FE will throw exceptions instead swallow them. This is used for test") public boolean feDebug = false; @VarAttrDef.VarAttr(name = FETCH_ALL_FE_FOR_SYSTEM_TABLE, - description = {"When the variable is true, some system tables retrieve data from all fe", - "当变量为 true 时，部分系统表从所有 fe 获取数据"}) + description = "When the variable is true, some system tables retrieve data from all fe") public boolean fetchAllFeForSystemTable = true; @VarAttrDef.VarAttr(name = MAX_MSG_SIZE_OF_RESULT_RECEIVER, - description = {"Max message size during result deserialization, change this if you meet error" - + " like \"MaxMessageSize reached\"", - "用于控制结果反序列化时 thrift 字段的最大值，当遇到类似\"MaxMessageSize reached\"这样的错误时可以考虑修改该参数"}) + description = "Max message size during result deserialization, change this if you meet error" + + " like \"MaxMessageSize reached\"") public int maxMsgSizeOfResultReceiver = TConfiguration.DEFAULT_MAX_MESSAGE_SIZE; // CLOUD_VARIABLES_BEGIN @@ -3383,8 +3162,7 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { @VarAttrDef.VarAttr( name = "enable_compress_materialize", - description = {"控制是否启用 compress materialize。", - "enable compress-materialize. "}, + description = "enable compress-materialize. ", needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL ) @@ -3392,34 +3170,29 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { @VarAttrDef.VarAttr( name = DATA_QUEUE_MAX_BLOCKS, - description = {"DataQueue 中每个子队列允许最大的 block 个数", - "Max blocks in DataQueue."}, + description = "Max blocks in DataQueue.", needForward = true, fuzzy = true) public long dataQueueMaxBlocks = 1; // for spill to disk @VarAttrDef.VarAttr( name = ENABLE_SPILL, - description = {"控制是否启用查询算子落盘。默认为 false。", - "Controls whether to enable spill to disk for query. " - + "The default value is false."}, + description = "Controls whether to enable spill to disk for query. " + + "The default value is false.", needForward = true, fuzzy = true) public boolean enableSpill = false; @VarAttrDef.VarAttr( name = ENABLE_FORCE_SPILL, - description = {"控制是否开启强制落盘（即使在内存足够的情况），默认为 false。", - "Controls whether enable force spill." - }, + description = "Controls whether enable force spill.", needForward = true, fuzzy = false ) public boolean enableForceSpill = false; @VarAttrDef.VarAttr( name = ENABLE_RESERVE_MEMORY, - description = {"控制是否启用分配内存前先 reverve memory 的功能。默认为 true。", - "Controls whether to enable reserve memory before allocating memory. " - + "The default value is true."}, + description = "Controls whether to enable reserve memory before allocating memory. " + + "The default value is true.", needForward = true, fuzzy = true) public boolean enableReserveMemory = true; @@ -3427,11 +3200,9 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { public long spillMinRevocableMem = 4 * 1024 * 1024; @VarAttrDef.VarAttr(name = SPILL_BUFFER_SIZE_BYTES, fuzzy = true, needForward = true, - description = {"落盘时写 block 的最大大小（字节）。如果一个 block 超过该阈值，会按此大小拆分后再写入磁盘。" - + "同时也控制 merge sort 阶段每个文件的读 buffer 大小。默认 8MB。", - "Maximum block size for spill writes (in bytes). Blocks larger than this threshold are " + description = "Maximum block size for spill writes (in bytes). Blocks larger than this threshold are " + "split before writing to disk. Also controls per-file read buffer size during merge sort. " - + "Default is 8MB."}) + + "Default is 8MB.") public long spillBufferSizeBytes = 8L * 1024L * 1024L; @VarAttrDef.VarAttr(name = SPILL_AGGREGATION_PARTITION_COUNT, fuzzy = true) @@ -3449,34 +3220,29 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { public int spillHashJoinPartitionCount = 4; @VarAttrDef.VarAttr(name = SPILL_REPARTITION_MAX_DEPTH, fuzzy = true, needForward = true, - description = {"重分区的最大递归深度，超过该深度不再继续重分区，\n默认值为 8", - "Maximum depth for repartition recursion. When exceeded, repartitioning will stop. Default is 8."}) + description = "Maximum depth for repartition recursion. When exceeded, repartitioning will stop. Default " + + "is 8.") public int spillRepartitionMaxDepth = 8; @VarAttrDef.VarAttr(name = SPILL_JOIN_BUILD_SINK_MEM_LIMIT_BYTES, fuzzy = true, needForward = true, - description = {"一旦触发 spill 后，join build sink 的 revocable memory 超过该阈值就主动落盘（字节）。默认 64MB。", - "After spill is triggered, join build sink will proactively spill when revocable memory " - + "exceeds this threshold (in bytes). Default is 64MB."}) + description = "After spill is triggered, join build sink will proactively spill when revocable memory " + + "exceeds this threshold (in bytes). Default is 64MB.") public long spillJoinBuildSinkMemLimitBytes = 64L * 1024L * 1024L; @VarAttrDef.VarAttr(name = SPILL_AGGREGATION_SINK_MEM_LIMIT_BYTES, fuzzy = true, needForward = true, - description = {"一旦触发 spill 后，aggregation sink 的 revocable memory 超过该阈值就主动落盘（字节）。默认 64MB。", - "After spill is triggered, aggregation sink will proactively spill when revocable memory " - + "exceeds this threshold (in bytes). Default is 64GB."}) + description = "After spill is triggered, aggregation sink will proactively spill when revocable memory " + + "exceeds this threshold (in bytes). Default is 64GB.") public long spillAggregationSinkMemLimitBytes = 64L * 1024L * 1024L * 1024L; @VarAttrDef.VarAttr(name = SPILL_SORT_SINK_MEM_LIMIT_BYTES, fuzzy = true, needForward = true, - description = {"一旦触发 spill 后，sort sink 的 revocable memory 超过该阈值就主动落盘（字节）。默认 64MB。", - "After spill is triggered, sort sink will proactively spill when revocable memory " - + "exceeds this threshold (in bytes). Default is 64MB."}) + description = "After spill is triggered, sort sink will proactively spill when revocable memory " + + "exceeds this threshold (in bytes). Default is 64MB.") public long spillSortSinkMemLimitBytes = 64L * 1024L * 1024L; @VarAttrDef.VarAttr(name = SPILL_SORT_MERGE_MEM_LIMIT_BYTES, fuzzy = true, needForward = true, - description = {"一旦触发 spill 后，sort merge 阶段可用的总内存大小（字节）。" - + "该值除以 spill_buffer_size_bytes 即为可并行读取合并的文件数。默认 64MB。", - "After spill is triggered, total memory budget for the sort merge phase (in bytes). " + description = "After spill is triggered, total memory budget for the sort merge phase (in bytes). " + "Divided by spill_buffer_size_bytes gives the number of files that can be merged " - + "in parallel. Default is 64MB."}) + + "in parallel. Default is 64MB.") public long spillSortMergeMemLimitBytes = 64L * 1024L * 1024L; @VarAttrDef.VarAttr(name = SPILL_REVOCABLE_MEMORY_HIGH_WATERMARK_PERCENT, fuzzy = true) @@ -3484,9 +3250,9 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { @VarAttrDef.VarAttr( name = DUMP_HEAP_PROFILE_WHEN_MEM_LIMIT_EXCEEDED, - description = {"查询因为内存不足被 Cancel 时，是否 Dump heap profile 到日志文件。默认为 false。", - "Whether to dump heap profile to log file when query is canceled becuase of memory not enough. " - + "The default value is false."}, + description = "Whether to dump heap profile to log file when query is canceled becuase of memory not " + + "enough. " + + "The default value is false.", needForward = true) public boolean dumpHeapProfileWhenMemLimitExceeded = false; @@ -3496,18 +3262,15 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { @VarAttrDef.VarAttr( name = ENABLE_USE_HYBRID_SORT, - description = {"是否启用混合排序，动态选择 PdqSort 和 TimSort 以适应数据模式。默认为 true。", - "Enable hybrid sorting: dynamically selects between PdqSort and TimSort " - + "based on runtime profiling to choose the most efficient algorithm " - + "for the data pattern. The default value is true."}, + description = "Enable hybrid sorting: dynamically selects between PdqSort and TimSort " + + "based on runtime profiling to choose the most efficient algorithm " + + "for the data pattern. The default value is true.", needForward = true, fuzzy = true) public boolean enableUseHybridSort = true; - @VarAttrDef.VarAttr(name = USE_MAX_LENGTH_OF_VARCHAR_IN_CTAS, needForward = true, description = { - "在 CTAS 中，如果 CHAR / VARCHAR 列不来自于源表，是否是将这一列的长度设置为 MAX，即 65533。默认为 true。", - "In CTAS (Create Table As Select), if CHAR/VARCHAR columns do not originate from the source table," - + " whether to set the length of such a column to MAX, which is 65533. The default is true." - }) + @VarAttrDef.VarAttr(name = USE_MAX_LENGTH_OF_VARCHAR_IN_CTAS, needForward = true, description = "In CTAS (Create " + + "Table As Select), if CHAR/VARCHAR columns do not originate from the source table," + + " whether to set the length of such a column to MAX, which is 65533. The default is true.") public boolean useMaxLengthOfVarcharInCtas = true; // Whether enable segment cache. Segment cache only works when FE's query options sets enableSegmentCache true @@ -3519,184 +3282,127 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { * When enabling shard scroll, FE will plan scan ranges by shards of ES indices. * Otherwise, FE will plan a single query to ES. */ - @VarAttrDef.VarAttr(name = ENABLE_ES_PARALLEL_SCROLL, description = { - "ES catalog 是否开启 shard 级别并发的 scroll 请求，默认开启。", - "Whether to enable shard-level parallel scroll requests for ES catalog, enabled by default." - }) + @VarAttrDef.VarAttr(name = ENABLE_ES_PARALLEL_SCROLL, description = "Whether to enable shard-level parallel scroll " + + "requests for ES catalog, enabled by default.") public boolean enableESParallelScroll = true; - @VarAttrDef.VarAttr(name = ENABLE_MATCH_WITHOUT_INVERTED_INDEX, description = { - "开启无索引 match 查询功能，建议正式环境保持开启", - "Enable no-index match query functionality." - + " it is recommended to keep this enabled in the production environment." - }) + @VarAttrDef.VarAttr(name = ENABLE_MATCH_WITHOUT_INVERTED_INDEX, description = "Enable no-index match query " + + "functionality." + + " it is recommended to keep this enabled in the production environment.") public boolean enableMatchWithoutInvertedIndex = true; - @VarAttrDef.VarAttr(name = ENABLE_FALLBACK_ON_MISSING_INVERTED_INDEX, description = { - "开启后在没有找到索引的情况下直接查询报错，建议正式环境保持开启", - "After enabling, it will directly query and report an error if no index is found." - + " It is recommended to keep this enabled in the production environment." - }) + @VarAttrDef.VarAttr(name = ENABLE_FALLBACK_ON_MISSING_INVERTED_INDEX, description = "After enabling, it will " + + "directly query and report an error if no index is found." + + " It is recommended to keep this enabled in the production environment.") public boolean enableFallbackOnMissingInvertedIndex = true; - @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_SEARCHER_CACHE, description = { - "开启后会缓存倒排索引 searcher", - "Enabling this will cache the inverted index searcher." - }) + @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_SEARCHER_CACHE, description = "Enabling this will cache the " + + "inverted index searcher.") public boolean enableInvertedIndexSearcherCache = true; - @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_QUERY_CACHE, description = { - "开启后会缓存倒排索引查询结果", - "Enabling this will cache the results of inverted index queries." - }) + @VarAttrDef.VarAttr(name = ENABLE_INVERTED_INDEX_QUERY_CACHE, description = "Enabling this will cache the results " + + "of inverted index queries.") public boolean enableInvertedIndexQueryCache = true; - @VarAttrDef.VarAttr(name = ENABLE_ANN_INDEX_RESULT_CACHE, needForward = true, description = { - "开启后会缓存 ANN 索引查询结果", - "Enabling this will cache the results of ANN index queries." - }) + @VarAttrDef.VarAttr(name = ENABLE_ANN_INDEX_RESULT_CACHE, needForward = true, description = "Enabling this will " + + "cache the results of ANN index queries.") public boolean enableAnnIndexResultCache = true; - @VarAttrDef.VarAttr(name = IN_LIST_VALUE_COUNT_THRESHOLD, description = { - "in 条件 value 数量大于这个 threshold 后将不会走 fast_execute", - "When the number of values in the IN condition exceeds this threshold," - + " fast_execute will not be used." - }, affectQueryResultInExecution = true) + @VarAttrDef.VarAttr(name = IN_LIST_VALUE_COUNT_THRESHOLD, description = "When the number of values in the IN " + + "condition exceeds this threshold," + + " fast_execute will not be used.", affectQueryResultInExecution = true) public int inListValueCountThreshold = 10; - @VarAttrDef.VarAttr(name = ENABLE_ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT, needForward = true, description = { - "开启后将会允许自动调整 pipeline task 的并发数。当 scan 节点没有过滤条件，且 limit 参数小于" - + "adaptive_pipeline_task_serial_read_on_limit 中指定的行数时，scanner 的并行度将会被设置为 1", - "When enabled, the pipeline task concurrency will be adjusted automatically. When the scan node has no filter " + @VarAttrDef.VarAttr(name = ENABLE_ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT, needForward = true, description = "W" + + "hen enabled, the pipeline task concurrency will be adjusted automatically. When the scan node has no " + + "filter " + "conditions and the limit parameter is less than the number of rows specified in " - + "adaptive_pipeline_task_serial_read_on_limit, the parallelism of the scan will be set to 1." - }) + + "adaptive_pipeline_task_serial_read_on_limit, the parallelism of the scan will be set to 1.") public boolean enableAdaptivePipelineTaskSerialReadOnLimit = true; - @VarAttrDef.VarAttr(name = ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT, needForward = true, description = { - "当 enable_adaptive_pipeline_task_serial_read_on_limit 开启时，scanner 的并行度将会被设置为 1 的行数阈值", - "When enable_adaptive_pipeline_task_serial_read_on_limit is enabled, " - + "the number of rows at which the parallelism of the scan will be set to 1." - }) + @VarAttrDef.VarAttr(name = ADAPTIVE_PIPELINE_TASK_SERIAL_READ_ON_LIMIT, needForward = true, description = "When " + + "enable_adaptive_pipeline_task_serial_read_on_limit is enabled, " + + "the number of rows at which the parallelism of the scan will be set to 1.") public int adaptivePipelineTaskSerialReadOnLimit = 10000; @VarAttrDef.VarAttr(name = "enable_adjust_conjunct_order_by_cost", needForward = true) public boolean enableAdjustConjunctOrderByCost = true; - @VarAttrDef.VarAttr(name = REQUIRE_SEQUENCE_IN_INSERT, needForward = true, description = { - "该变量用于控制，使用了 sequence 列的 unique key 表，insert into 操作是否要求必须提供每一行的 sequence 列的值", - "This variable controls whether the INSERT INTO operation on unique key tables with a sequence" - + " column requires a sequence column to be provided for each row" - }) + @VarAttrDef.VarAttr(name = REQUIRE_SEQUENCE_IN_INSERT, needForward = true, description = "This variable controls " + + "whether the INSERT INTO operation on unique key tables with a sequence" + + " column requires a sequence column to be provided for each row") public boolean requireSequenceInInsert = true; @VarAttrDef.VarAttr(name = ENABLE_COOLDOWN_REPLICA_AFFINITY, needForward = true) public boolean enableCooldownReplicaAffinity = true; - @VarAttrDef.VarAttr(name = ENABLE_AUTO_CREATE_WHEN_OVERWRITE, needForward = true, description = { - "开启后对自动分区表的 insert overwrite 操作会对没有找到分区的插入数据按自动分区规则创建分区，默认关闭", - "The insert overwrite operation on an auto-partitioned table will create partitions for inserted data" - + " for which no partition is found according to the auto-partitioning rules, which is turned off" - + " by default." - }) + @VarAttrDef.VarAttr(name = ENABLE_AUTO_CREATE_WHEN_OVERWRITE, needForward = true, description = "The insert " + + "overwrite operation on an auto-partitioned table will create partitions for inserted data" + + " for which no partition is found according to the auto-partitioning rules, which is turned off" + + " by default.") public boolean enableAutoCreateWhenOverwrite = false; - @VarAttrDef.VarAttr(name = ENABLE_TEXT_VALIDATE_UTF8, needForward = true, description = { - "对于 text 类型的文件读取，是否开启 utf8 编码检查。非 utf8 字符会显示成乱码。", - "For text type file reading, whether to enable utf8 encoding check." - + "non-utf8 characters will be displayed as garbled characters." - }) + @VarAttrDef.VarAttr(name = ENABLE_TEXT_VALIDATE_UTF8, needForward = true, description = "For text type file " + + "reading, whether to enable utf8 encoding check." + + "non-utf8 characters will be displayed as garbled characters.") public boolean enableTextValidateUtf8 = true; @VarAttrDef.VarAttr(name = SKIP_CHECKING_ACID_VERSION_FILE, needForward = true, affectQueryResultInPlan = true, - description = { - "跳过检查 transactional hive 版本文件 '_orc_acid_version.'", - "Skip checking transactional hive version file '_orc_acid_version.'" - } + description = "Skip checking transactional hive version file '_orc_acid_version.'" ) public boolean skipCheckingAcidVersionFile = false; @VarAttrDef.VarAttr(name = ENABLE_SQL_CONVERTOR_FEATURES, needForward = true, checker = "checkSqlConvertorFeatures", - description = { - "开启 SQL 转换器的指定功能。多个功能使用逗号分隔", - "enable SQL convertor features. Multiple features are separated by commas" - }) + description = "enable SQL convertor features. Multiple features are separated by commas") public String enableSqlConvertorFeatures = ""; // The default value is true, // which throughs reducing rpc call from follower node to meta service to improve query performance // for getting version is memory operation in master node, // but it will slightly increase the pressure on the FE master. - @VarAttrDef.VarAttr(name = ENABLE_SCHEMA_SCAN_FROM_MASTER_FE, description = { - "在 follower 节点查询时，是否允许从 master 节点扫描 information_schema.tables 的结果", - "Whether to allow scanning information_schema.tables from the master node" - }) + @VarAttrDef.VarAttr(name = ENABLE_SCHEMA_SCAN_FROM_MASTER_FE, description = "Whether to allow scanning " + + "information_schema.tables from the master node") public boolean enableSchemaScanFromMasterFe = true; @VarAttrDef.VarAttr(name = SHOW_COLUMN_COMMENT_IN_DESCRIBE, needForward = true, - description = { - "是否在 DESCRIBE TABLE 语句中显示列注释", - "whether to show column comments in DESCRIBE TABLE statement" - }) + description = "whether to show column comments in DESCRIBE TABLE statement") public boolean showColumnCommentInDescribe = false; @VarAttrDef.VarAttr(name = SQL_CONVERTOR_CONFIG, needForward = true, - description = { - "SQL 转换器的相关配置，使用 Json 格式。以 {} 为根元素。", - "SQL convertor config, use Json format. The root element is {}" - }) + description = "SQL convertor config, use Json format. The root element is {}") public String sqlConvertorConfig = "{}"; @VarAttrDef.VarAttr(name = PREFER_UDF_OVER_BUILTIN, needForward = true, - description = { - "是否优先查找 UDF 而不是内置函数", - "Whether to prefer UDF over builtin functions" - }) + description = "Whether to prefer UDF over builtin functions") public boolean preferUdfOverBuiltin = false; - @VarAttrDef.VarAttr(name = SKEW_REWRITE_JOIN_SALT_EXPLODE_FACTOR, description = { - "join 加盐优化的扩展因子，对指定的倾斜值，join 倾斜侧生成 0 到 ExplodeFactor - 1 的随机值，" - + "join 扩展侧复制为 ExplodeFactor 个副本，使 hash shuffle 之后计算负载均匀分布。" - + "可以配置为 0-65535 中的数字：0 代表根据集群中 be 的数量和 cpu 核数自适应，1-65535 中的数量代表扩展倍数", - "ExplodeFactor: The expansion factor for join skew optimization. " - + "For specified skewed values, it generates random values between 0 and ExplodeFactor-1 " - + "on the skewed side, while replicating the expanded side into ExplodeFactor copies," - + "ensuring even load distribution after hash shuffling. " - + "Configurable range: 0-65535 (0=auto-adapt based on BEs and CPU cores;" - + "1-65535=manual expansion multiplier)" - }, checker = "checkSkewRewriteJoinSaltExplodeFactor") + @VarAttrDef.VarAttr(name = SKEW_REWRITE_JOIN_SALT_EXPLODE_FACTOR, description = "ExplodeFactor: The expansion " + + "factor for join skew optimization. " + + "For specified skewed values, it generates random values between 0 and ExplodeFactor-1 " + + "on the skewed side, while replicating the expanded side into ExplodeFactor copies," + + "ensuring even load distribution after hash shuffling. " + + "Configurable range: 0-65535 (0=auto-adapt based on BEs and CPU cores;" + + "1-65535=manual expansion multiplier)", checker = "checkSkewRewriteJoinSaltExplodeFactor") public int skewRewriteJoinSaltExplodeFactor = 0; @VarAttrDef.VarAttr(name = DEFAULT_AI_RESOURCE, needForward = true, - description = { - "当函数参数未指定 AI Resource 时，系统将默认使用此变量定义的 Resource。", - "Defines the default AI resource to be used when no specific AI resource is specified " - + "in the function arguments." - }) + description = "Defines the default AI resource to be used when no specific AI resource is specified " + + "in the function arguments.") public String defaultAIResource = ""; @VarAttrDef.VarAttr(name = FILE_PRESIGNED_URL_TTL_SECONDS, needForward = true, - description = { - "EMBED 多模态场景中，S3 预签名 URL 的有效期（秒）。", - "Expiration time in seconds for S3 presigned URL used by multimodal EMBED." - }) + description = "Expiration time in seconds for S3 presigned URL used by multimodal EMBED.") public long filePresignedUrlTtlSeconds = 3600; @VarAttrDef.VarAttr(name = EMBED_MAX_BATCH_SIZE, needForward = true, checker = "checkEmbedMaxBatchSize", - description = { - "EMBED 场景中，单次批量请求允许携带的最大输入数量，文本与多模态共用。", - "Maximum number of inputs allowed in one EMBED batch request for both text and multimodal." - }) + description = "Maximum number of inputs allowed in one EMBED batch request for both text and multimodal.") public int embedMaxBatchSize = 5; @VarAttrDef.VarAttr(name = AI_CONTEXT_WINDOW_SIZE, needForward = true, checker = "checkAiContextWindowSize", - description = { - "AI 函数批量请求时使用的上下文窗口字节上限。", - "Context window size in bytes for AI function batching." - }) + description = "Context window size in bytes for AI function batching.") public long aiContextWindowSize = 128 * 1024; public void setEnableEsParallelScroll(boolean enableESParallelScroll) { @@ -3707,50 +3413,40 @@ public boolean isEnableESParallelScroll() { return enableESParallelScroll; } - @VarAttrDef.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, needForward = true, description = { - "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", - "Whether to enable add index mode that only affects new data, " - + "when enabled new indexes only affect subsequently written data, " - + "when disabled rebuild indexes for all data" - }) + @VarAttrDef.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, needForward = true, description = "Whether to enable add " + + "index mode that only affects new data, " + + "when enabled new indexes only affect subsequently written data, " + + "when disabled rebuild indexes for all data") public boolean enableAddIndexForNewData = false; @VarAttrDef.VarAttr(name = HNSW_EF_SEARCH, needForward = true, checker = "checkHnswEfSearch", - description = {"HNSW 索引的 EF 搜索参数，控制搜索的精度和速度", - "HNSW index EF search parameter, controls the precision and speed of the search"}) + description = "HNSW index EF search parameter, controls the precision and speed of the search") public int hnswEFSearch = 32; @VarAttrDef.VarAttr(name = HNSW_CHECK_RELATIVE_DISTANCE, needForward = true, - description = {"是否启用相对距离检查机制，以提升 HNSW 搜索的准确性", - "Enable relative distance checking to improve HNSW search accuracy"}) + description = "Enable relative distance checking to improve HNSW search accuracy") public boolean hnswCheckRelativeDistance = true; @VarAttrDef.VarAttr(name = HNSW_BOUNDED_QUEUE, needForward = true, - description = {"是否使用有界优先队列来优化 HNSW 的搜索性能", - "Whether to use a bounded priority queue to optimize HNSW search performance"}) + description = "Whether to use a bounded priority queue to optimize HNSW search performance") public boolean hnswBoundedQueue = true; @VarAttrDef.VarAttr(name = IVF_NPROBE, needForward = true, checker = "checkIvfNprobe", - description = {"IVF 索引的 nprobe 参数，控制搜索时访问的聚类数量", - "IVF index nprobe parameter, controls the number of clusters to search"}) + description = "IVF index nprobe parameter, controls the number of clusters to search") public int ivfNprobe = 32; @VarAttrDef.VarAttr(name = ANN_INDEX_CANDIDATE_ROWS_THRESHOLD, needForward = true, checker = "checkAnnIndexCandidateRowsThreshold", - description = {"Skip ANN index when candidate rows before ANN search are less " - + "than this threshold. 0 disables the absolute row threshold", - "Skip ANN index when candidate rows before ANN search are less " - + "than this threshold. 0 disables the absolute row threshold"}) + description = "Skip ANN index when candidate rows before ANN search are less " + + "than this threshold. 0 disables the absolute row threshold") public long annIndexCandidateRowsThreshold = 0; @VarAttrDef.VarAttr(name = ANN_INDEX_CANDIDATE_ROWS_PERCENT_THRESHOLD, needForward = true, checker = "checkAnnIndexCandidateRowsPercentThreshold", - description = {"Skip ANN index when candidate row ratio before ANN search is less " - + "than this threshold", - "Skip ANN index when candidate row ratio before ANN search is less " - + "than this threshold"}) + description = "Skip ANN index when candidate row ratio before ANN search is less " + + "than this threshold") public double annIndexCandidateRowsPercentThreshold = 0.3; public void checkAnnIndexCandidateRowsThreshold(String value) { @@ -3781,11 +3477,8 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { name = ENABLE_VARIANT_SCHEMA_AUTO_CAST, needForward = true, affectQueryResultInExecution = true, - description = { - "是否启用基于 schema template 的 variant 自动 cast，默认开启。", - "Whether to enable schema-template-based auto cast for variant expressions. " - + "The default is true." - } + description = "Whether to enable schema-template-based auto cast for variant expressions. " + + "The default is true." ) public boolean enableVariantSchemaAutoCast = true; @@ -3794,10 +3487,7 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { needForward = true, affectQueryResultInPlan = true, varType = VariableAnnotation.EXPERIMENTAL, - description = { - "是否对纯计算表达式启用 ColumnVariantV2，默认关闭。", - "Whether to enable ColumnVariantV2 for compute expressions. The default is false." - } + description = "Whether to enable ColumnVariantV2 for compute expressions. The default is false." ) public boolean enableVariantV2 = false; @@ -3817,8 +3507,7 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { public int defaultVariantMaxSparseColumnStatisticsSize = 10000; @VarAttrDef.VarAttr(name = ENABLE_EXTENDED_REGEX, needForward = true, affectQueryResultInExecution = true, - description = {"是否启用扩展的正则表达式，支持如 look-around 类的零宽断言", - "Enable extended regular expressions, support look-around zero-width assertions"}) + description = "Enable extended regular expressions, support look-around zero-width assertions") public boolean enableExtendedRegex = false; @VarAttrDef.VarAttr( @@ -3829,9 +3518,8 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { public int defaultVariantSparseHashShardCount = 0; @VarAttrDef.VarAttr(name = CLOUD_PARTITIONS_TABLE_USE_CACHED_VISIBLE_VERSION, needForward = false, - description = {"partitions系统表的visible_version列在cloud模式是否使用cached", - "Whether cache is used for the visible_version column" - + "in the partitions system table on cloud mode"}) + description = "Whether cache is used for the visible_version column" + + "in the partitions system table on cloud mode") public boolean cloudPartitionsTableUseCachedVisibleVersion = true; @VarAttrDef.VarAttr( @@ -3865,9 +3553,7 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { @VarAttrDef.VarAttr( name = "use_v3_storage_format", fuzzy = true, - description = { - "In fuzzy tests, randomly use V3 storage_format (ext_meta) for some tables.", - "Only takes effect when user does not explicitly specify storage_format."} + description = "In fuzzy tests, randomly use V3 storage_format (ext_meta) for some tables." ) public boolean useV3StorageFormat = false; @@ -3876,21 +3562,18 @@ public void checkAnnIndexCandidateRowsPercentThreshold(String value) { public static final String IGNORE_ICEBERG_DANGLING_DELETE = "ignore_iceberg_dangling_delete"; @VarAttrDef.VarAttr(name = IGNORE_ICEBERG_DANGLING_DELETE, - description = {"是否忽略 Iceberg 表中 dangling delete 文件对 COUNT(*) 统计信息的影响。" - + "默认为 true，COUNT(*) 会直接从元信息中获取行数，性能更好，但是如果有 dangling delete，结果可能是不准确的。" - + "设置为 false 时，COUNT(*) 会扫描数据文件以排除 dangling delete 文件的影响。", - " Whether to ignore the impact of dangling delete files in Iceberg tables on COUNT(*) statistics. " - + "The default is true, COUNT(*) will directly obtain the number of rows from metadata, " - + "which has better performance, but if there are dangling deletes, " - + "the result may be inaccurate. " - + "When set to false, COUNT(*) will scan data files " - + "to exclude the impact of dangling delete files."}) + description = " Whether to ignore the impact of dangling delete files in Iceberg tables on COUNT(*) " + + "statistics. " + + "The default is true, COUNT(*) will directly obtain the number of rows from metadata, " + + "which has better performance, but if there are dangling deletes, " + + "the result may be inaccurate. " + + "When set to false, COUNT(*) will scan data files " + + "to exclude the impact of dangling delete files.") public boolean ignoreIcebergDanglingDelete = false; @VarAttrDef.VarAttr(name = ENABLE_ICEBERG_MERGE_PARTITIONING, - description = {"是否启用 Iceberg UPDATE/DELETE 合并写入的双分支分发（INSERT 按分区列，DELETE 按 row_id）。", - "Enable merge partitioning for Iceberg UPDATE/DELETE (INSERT by partition columns, " - + "DELETE by row_id)."}) + description = "Enable merge partitioning for Iceberg UPDATE/DELETE (INSERT by partition columns, " + + "DELETE by row_id).") public boolean enableIcebergMergePartitioning = true; // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index 75bc69ba7c5531..f1f0bd14033c47 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -152,11 +152,10 @@ public void testInsertVisibleTimeoutReturnMode() throws Exception { Field field = SessionVariable.class.getDeclaredField("insertVisibleTimeoutReturnMode"); VarAttrDef.VarAttr varAttr = field.getAnnotation(VarAttrDef.VarAttr.class); - Assertions.assertArrayEquals(new String[] { - "控制普通内表 INSERT 在 publish timeout 时返回给客户端的状态。", + Assertions.assertEquals( "Controls the status returned to the client when a normal internal-table INSERT times out " - + "while waiting for publish visibility." - }, varAttr.description()); + + "while waiting for publish visibility.", + varAttr.description()); Assertions.assertArrayEquals(new String[] { SessionVariable.INSERT_VISIBLE_TIMEOUT_RETURN_MODE_COMMITTED, SessionVariable.INSERT_VISIBLE_TIMEOUT_RETURN_MODE_ERROR @@ -196,13 +195,11 @@ public void testRuntimeFilterBroadcastJoinProducerNumDescription() throws Except Field field = SessionVariable.class.getDeclaredField("runtimeFilterBroadcastJoinProducerNum"); VarAttrDef.VarAttr varAttr = field.getAnnotation(VarAttrDef.VarAttr.class); - Assertions.assertArrayEquals(new String[] { - "控制 Nereids 分布式规划中每个 broadcast join runtime filter 的生产 BE 数量。" - + "设置为小于等于 0 时不限制。Legacy Coordinator 路径保持原行为。", + Assertions.assertEquals( "Controls the number of producer BEs for each broadcast join runtime filter in " + "the Nereids distributed planner. Values less than or equal to 0 disable the limit. " - + "The legacy Coordinator path keeps the existing behavior." - }, varAttr.description()); + + "The legacy Coordinator path keeps the existing behavior.", + varAttr.description()); } @Test From cd3c078042267998a9d12b8b263df0736d447c7d Mon Sep 17 00:00:00 2001 From: wudi Date: Thu, 6 Aug 2026 15:35:54 +0800 Subject: [PATCH 02/12] [fix](streaming-job) Optimize snapshot offset persistence (#66238) ### What problem does this PR solve? Problem Summary: Snapshot tasks persisted the complete offset provider after every split commit, producing frequent edit logs while the snapshot state kept growing. The previous default snapshot split size also produced too many small snapshot tasks. Natural terminal transitions needed explicit persistence so finished jobs replay with their terminal metadata. This change: - Limits snapshot offset edit logs with a mutable 300-second interval while keeping binlog offset commits immediate. - Increases the default snapshot split size from 8,096 to 40,960 rows, while preserving explicitly configured values. - Removes obsolete snapshot recovery state after the first committed binlog offset for FROM-TO and CDC stream TVF jobs. - Persists natural FINISHED transitions and restores terminal replay fields and callbacks consistently. - Adds focused unit coverage and a MySQL snapshot-only FE restart regression case. --- .../java/org/apache/doris/common/Config.java | 4 + .../doris/job/cdc/DataSourceConfigKeys.java | 1 + .../insert/streaming/StreamingInsertJob.java | 30 +- .../streaming/StreamingJobSchedulerTask.java | 1 + .../offset/jdbc/JdbcSourceOffsetProvider.java | 30 ++ .../jdbc/JdbcTvfSourceOffsetProvider.java | 9 +- ...reamingInsertJobOffsetPersistenceTest.java | 263 ++++++++++++++++++ .../JdbcSourceOffsetProviderOffsetTest.java | 211 ++++++++++++++ .../reader/mysql/MySqlSourceReader.java | 9 +- .../reader/postgres/PostgresSourceReader.java | 10 +- ...ql_job_snapshot_finished_restart_fe.groovy | 160 +++++++++++ 11 files changed, 708 insertions(+), 20 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java create mode 100644 regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_finished_restart_fe.groovy diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index b626cba8cf15a3..114112d9e58523 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1174,6 +1174,10 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int streaming_task_min_timeout_sec = 300; + @ConfField(mutable = true, masterOnly = true, description = { + "Minimum interval in seconds between snapshot offset persistence operations"}) + public static int streaming_job_snapshot_offset_persist_interval_sec = 300; + @ConfField(mutable = true, masterOnly = true) public static int streaming_cdc_light_rpc_timeout_sec = 90; diff --git a/fe/fe-common/src/main/java/org/apache/doris/job/cdc/DataSourceConfigKeys.java b/fe/fe-common/src/main/java/org/apache/doris/job/cdc/DataSourceConfigKeys.java index fb0f1825324674..95956cbeb4919c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/job/cdc/DataSourceConfigKeys.java +++ b/fe/fe-common/src/main/java/org/apache/doris/job/cdc/DataSourceConfigKeys.java @@ -35,6 +35,7 @@ public class DataSourceConfigKeys { public static final String OFFSET_LATEST = "latest"; public static final String OFFSET_SNAPSHOT = "snapshot"; public static final String SNAPSHOT_SPLIT_SIZE = "snapshot_split_size"; + public static final String SNAPSHOT_SPLIT_SIZE_DEFAULT = "40960"; public static final String SNAPSHOT_SPLIT_KEY = "snapshot_split_key"; public static final String SNAPSHOT_PARALLELISM = "snapshot_parallelism"; public static final String SNAPSHOT_PARALLELISM_DEFAULT = "1"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index cc7e7e34a18ef3..affb6d6c5992fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -152,6 +152,7 @@ public class StreamingInsertJob extends AbstractJob(binlogSplit.getStartingOffset()); binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + clearSnapshotState(); currentOffset = newOffset; hasMoreData = true; } @@ -266,6 +271,31 @@ public void updateOffset(Offset offset) { this.currentOffset = newOffset; } + protected void clearSnapshotState() { + if (MapUtils.isNotEmpty(chunkHighWatermarkMap)) { + chunkHighWatermarkMap = new HashMap<>(); + } + remainingSplits.clear(); + finishedSplits.clear(); + if (committedSplitProgress != null) { + clearProgress(committedSplitProgress); + } + if (cdcSplitProgress != null) { + clearProgress(cdcSplitProgress); + } + } + + public boolean shouldPersistOffset(long lastPersistTimeMs, long currentTimeMs) { + synchronized (splitsLock) { + if (currentOffset == null || !currentOffset.snapshotSplit()) { + return true; + } + } + long intervalMs = Math.max(1L, + (long) Config.streaming_job_snapshot_offset_persist_interval_sec) * 1000L; + return lastPersistTimeMs == 0L || currentTimeMs - lastPersistTimeMs >= intervalMs; + } + @Override public void setBoundBackendId(long boundBackendId) { this.boundBackendId = boundBackendId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java index 0e5bb8fb75319a..e7324015d937d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java @@ -294,10 +294,13 @@ public void updateOffset(Offset offset) { synchronized (splitsLock) { // Mirror binlog offset into bop so it survives FE checkpoint BinlogSplit bs = (BinlogSplit) newOffset.getSplits().get(0); - if (MapUtils.isNotEmpty(bs.getStartingOffset())) { - binlogOffsetPersist = new HashMap<>(bs.getStartingOffset()); - binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + if (MapUtils.isEmpty(bs.getStartingOffset())) { + log.warn("Skip empty committed binlog offset for job {}", getJobId()); + return; } + binlogOffsetPersist = new HashMap<>(bs.getStartingOffset()); + binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + clearSnapshotState(); currentOffset = newOffset; hasMoreData = true; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java new file mode 100644 index 00000000000000..0b13d0ec74586e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJobOffsetPersistenceTest.java @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.extensions.insert.streaming; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.job.cdc.request.CommitOffsetRequest; +import org.apache.doris.job.cdc.split.SnapshotSplit; +import org.apache.doris.job.common.JobStatus; +import org.apache.doris.job.common.TaskStatus; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.manager.JobManager; +import org.apache.doris.job.manager.StreamingTaskManager; +import org.apache.doris.job.offset.jdbc.JdbcSourceOffsetProvider; +import org.apache.doris.transaction.GlobalTransactionMgrIface; +import org.apache.doris.transaction.TxnStateCallbackFactory; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class StreamingInsertJobOffsetPersistenceTest { + + @Test + public void testFirstSnapshotCommitPersistsImmediately() throws Exception { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.getRemainingSplits().add(snapshotSplit("source_table:0")); + TestStreamingInsertJob job = newJob(provider, 1001L); + + job.commitOffset(snapshotRequest(1001L, "source_table:0", null)); + + Assert.assertEquals(1, job.journalCount); + Assert.assertNotNull(job.getOffsetProviderPersist()); + } + + @Test + public void testSnapshotCommitWithinIntervalDoesNotPersistAgain() throws Exception { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.getRemainingSplits().add(snapshotSplit("source_table:0")); + TestStreamingInsertJob job = newJob(provider, 1003L); + job.commitOffset(snapshotRequest(1003L, "source_table:0", null)); + + provider.getRemainingSplits().add(snapshotSplit("source_table:1")); + job.commitOffset(snapshotRequest(1003L, "source_table:1", null)); + + Assert.assertEquals(1, job.journalCount); + Assert.assertNotNull(job.getOffsetProviderPersist()); + } + + @Test + public void testBinlogCommitPersistsImmediately() throws Exception { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + TestStreamingInsertJob job = newJob(provider, 1002L); + + job.commitOffset(binlogRequest(1002L, "100")); + job.commitOffset(binlogRequest(1002L, "200")); + + Assert.assertEquals(2, job.journalCount); + Assert.assertNotNull(job.getOffsetProviderPersist()); + } + + @Test + public void testSnapshotToBinlogTransitionPersistsCompactedState() throws Exception { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.getRemainingSplits().add(snapshotSplit("source_table:0")); + TestStreamingInsertJob job = newJob(provider, 1008L); + job.commitOffset(snapshotRequest(1008L, "source_table:0", null)); + Assert.assertEquals(1, job.journalCount); + + job.commitOffset(binlogRequest(1008L, "200")); + + Assert.assertEquals(2, job.journalCount); + Assert.assertFalse(job.getOffsetProviderPersist().contains("source_table:0")); + Assert.assertTrue(provider.getFinishedSplits().isEmpty()); + Assert.assertTrue(provider.getChunkHighWatermarkMap().isEmpty()); + } + + @Test + public void testSnapshotOffsetPersistsOnNextCommitAfterInterval() throws Exception { + int oldInterval = Config.streaming_job_snapshot_offset_persist_interval_sec; + Config.streaming_job_snapshot_offset_persist_interval_sec = 300; + try { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.getRemainingSplits().add(snapshotSplit("source_table:0")); + TestStreamingInsertJob job = newJob(provider, 1011L); + + job.commitOffset(snapshotRequest(1011L, "source_table:0", null)); + Assert.assertEquals(1, job.journalCount); + Deencapsulation.setField(job, "lastOffsetPersistTimeMs", + System.currentTimeMillis() - 300_000L); + provider.getRemainingSplits().add(snapshotSplit("source_table:1")); + job.commitOffset(snapshotRequest(1011L, "source_table:1", null)); + + Assert.assertEquals(2, job.journalCount); + Assert.assertTrue((long) Deencapsulation.getField(job, "lastOffsetPersistTimeMs") > 0L); + } finally { + Config.streaming_job_snapshot_offset_persist_interval_sec = oldInterval; + } + } + + @Test + public void testAlterOffsetReplacesSnapshotState() throws Exception { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.getRemainingSplits().add(snapshotSplit("source_table:0")); + TestStreamingInsertJob job = newJob(provider, 1009L); + job.commitOffset(snapshotRequest(1009L, "source_table:0", null)); + + HashMap properties = new HashMap<>(); + properties.put(StreamingJobProperties.OFFSET_PROPERTY, "{\"lsn\":\"300\"}"); + Deencapsulation.invoke(job, "modifyPropertiesInternal", properties); + + Assert.assertTrue(job.getOffsetProviderPersist().contains("300")); + Assert.assertTrue(provider.getFinishedSplits().isEmpty()); + Assert.assertTrue(provider.getChunkHighWatermarkMap().isEmpty()); + } + + @Test + public void testNaturalFinishPersistsFinalState() throws Exception { + TestStreamingInsertJob job = newJob(new EndJdbcSourceOffsetProvider(), 1012L); + NoopStreamingMultiTblTask task = + (NoopStreamingMultiTblTask) Deencapsulation.getField(job, "runningStreamTask"); + + try (MockedStatic envMockedStatic = Mockito.mockStatic(Env.class)) { + Env env = Mockito.mock(Env.class); + JobManager jobManager = Mockito.mock(JobManager.class); + StreamingTaskManager streamingTaskManager = Mockito.mock(StreamingTaskManager.class); + GlobalTransactionMgrIface transactionMgr = Mockito.mock(GlobalTransactionMgrIface.class); + TxnStateCallbackFactory callbackFactory = Mockito.mock(TxnStateCallbackFactory.class); + envMockedStatic.when(Env::getCurrentEnv).thenReturn(env); + envMockedStatic.when(Env::getCurrentGlobalTransactionMgr).thenReturn(transactionMgr); + Mockito.when(env.getJobManager()).thenReturn(jobManager); + Mockito.when(jobManager.getStreamingTaskManager()).thenReturn(streamingTaskManager); + Mockito.when(transactionMgr.getCallbackFactory()).thenReturn(callbackFactory); + + long beforeFinish = System.currentTimeMillis(); + job.onStreamTaskSuccess(task); + + Assert.assertEquals(JobStatus.FINISHED, job.getJobStatus()); + Assert.assertTrue(job.getFinishTimeMs() >= beforeFinish); + Assert.assertEquals(1, job.journalCount); + Mockito.verify(callbackFactory).removeCallback(9001L); + } + } + + @Test + public void testReplayUpdatedRestoresFinalStateAndRemovesCallback() { + TestStreamingInsertJob job = newJob(new JdbcSourceOffsetProvider(), 1013L); + TestStreamingInsertJob replayJob = newJob(new JdbcSourceOffsetProvider(), 1014L); + replayJob.setJobStatus(JobStatus.FINISHED); + replayJob.setFinishTimeMs(1234L); + + try (MockedStatic envMockedStatic = Mockito.mockStatic(Env.class)) { + GlobalTransactionMgrIface transactionMgr = Mockito.mock(GlobalTransactionMgrIface.class); + TxnStateCallbackFactory callbackFactory = Mockito.mock(TxnStateCallbackFactory.class); + envMockedStatic.when(Env::getCurrentGlobalTransactionMgr).thenReturn(transactionMgr); + Mockito.when(transactionMgr.getCallbackFactory()).thenReturn(callbackFactory); + + job.replayOnUpdated(replayJob); + + Assert.assertEquals(JobStatus.FINISHED, job.getJobStatus()); + Assert.assertEquals(1234L, job.getFinishTimeMs()); + Mockito.verify(callbackFactory).removeCallback(9001L); + } + } + + @Test + public void testReplayUpdatedRestoresStartTime() { + TestStreamingInsertJob job = newJob(new JdbcSourceOffsetProvider(), 1015L); + TestStreamingInsertJob replayJob = newJob(new JdbcSourceOffsetProvider(), 1016L); + replayJob.setStartTimeMs(1234L); + + job.replayOnUpdated(replayJob); + + Assert.assertEquals(1234L, job.getStartTimeMs()); + } + + private static TestStreamingInsertJob newJob(JdbcSourceOffsetProvider provider, long taskId) { + TestStreamingInsertJob job = new TestStreamingInsertJob(); + Deencapsulation.setField(job, "lock", new ReentrantReadWriteLock(true)); + Deencapsulation.setField(job, "jobId", 9001L); + Deencapsulation.setField(job, "jobName", "test_job"); + Deencapsulation.setField(job, "jobStatus", JobStatus.RUNNING); + Deencapsulation.setField(job, "offsetProvider", provider); + Deencapsulation.setField(job, "properties", new HashMap()); + Deencapsulation.setField(job, "targetProperties", new HashMap()); + Deencapsulation.setField(job, "runningStreamTask", new NoopStreamingMultiTblTask(taskId)); + return job; + } + + private static SnapshotSplit snapshotSplit(String splitId) { + return new SnapshotSplit( + splitId, + "source_db.source_table", + Collections.singletonList("id"), + new Object[]{1L}, + new Object[]{2L}, + null); + } + + private static CommitOffsetRequest snapshotRequest(long taskId, String splitId, String tableSchemas) { + CommitOffsetRequest request = new CommitOffsetRequest(); + request.setTaskId(taskId); + request.setOffset("[{\"splitId\":\"" + splitId + "\",\"lsn\":\"100\"}]"); + request.setTableSchemas(tableSchemas); + return request; + } + + private static CommitOffsetRequest binlogRequest(long taskId, String lsn) { + CommitOffsetRequest request = new CommitOffsetRequest(); + request.setTaskId(taskId); + request.setOffset("[{\"splitId\":\"binlog-split\",\"lsn\":\"" + lsn + "\"}]"); + return request; + } + + private static class TestStreamingInsertJob extends StreamingInsertJob { + private int journalCount; + + @Override + public void logUpdateOperation() { + journalCount++; + } + } + + private static class EndJdbcSourceOffsetProvider extends JdbcSourceOffsetProvider { + @Override + public boolean hasReachedEnd() { + return true; + } + } + + private static class NoopStreamingMultiTblTask extends StreamingMultiTblTask { + NoopStreamingMultiTblTask(long taskId) { + super(9001L, taskId, null, null, null, null, null, + new StreamingJobProperties(new HashMap<>()), null, null); + Deencapsulation.setField(this, "status", TaskStatus.RUNNING); + } + + @Override + public void successCallback(CommitOffsetRequest offsetRequest) throws JobException { + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java index 6efb9959748bcd..d71416fd3c5faf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java @@ -17,16 +17,47 @@ package org.apache.doris.job.offset.jdbc; +import org.apache.doris.common.Config; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.job.cdc.split.BinlogSplit; +import org.apache.doris.job.cdc.split.SnapshotSplit; +import org.apache.doris.job.extensions.insert.streaming.StreamingInsertJob; import org.junit.Assert; import org.junit.Test; import java.util.Collections; +import java.util.HashMap; import java.util.Map; public class JdbcSourceOffsetProviderOffsetTest { + @Test + public void testSnapshotOffsetUsesConfiguredPersistInterval() { + int oldInterval = Config.streaming_job_snapshot_offset_persist_interval_sec; + try { + Config.streaming_job_snapshot_offset_persist_interval_sec = 123; + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.currentOffset = new JdbcOffset( + Collections.singletonList(snapshotSplit("source_table:0"))); + + Assert.assertTrue(provider.shouldPersistOffset(0L, 1_000L)); + Assert.assertFalse(provider.shouldPersistOffset(1_000L, 123_999L)); + Assert.assertTrue(provider.shouldPersistOffset(1_000L, 124_000L)); + } finally { + Config.streaming_job_snapshot_offset_persist_interval_sec = oldInterval; + } + } + + @Test + public void testBinlogOffsetPersistsImmediately() { + JdbcSourceOffsetProvider provider = new JdbcSourceOffsetProvider(); + provider.currentOffset = new JdbcOffset(Collections.singletonList( + new BinlogSplit(Collections.singletonMap("lsn", "100")))); + + Assert.assertTrue(provider.shouldPersistOffset(1_000L, 1_001L)); + } + @Test public void testEndOffsetAdvancesWhenCurrentOffsetIsAhead() { assertEndOffsetAdvancesWhenCurrentOffsetIsAhead(new TestJdbcSourceOffsetProvider(-1)); @@ -76,6 +107,71 @@ public void testStaleCompareDoesNotOverwriteAlteredCurrentOffsetState() { ((BinlogSplit) provider.currentOffset.getSplits().get(0)).getStartingOffset()); } + @Test + public void testValidBinlogOffsetClearsSnapshotState() { + assertValidBinlogOffsetClearsSnapshotState(new TestJdbcSourceOffsetProvider(-1)); + } + + @Test + public void testTvfValidBinlogOffsetClearsSnapshotState() { + assertValidBinlogOffsetClearsSnapshotState(new TestJdbcTvfSourceOffsetProvider(-1)); + } + + @Test + public void testEmptyBinlogOffsetKeepsPreviousState() { + assertEmptyBinlogOffsetKeepsPreviousState(new TestJdbcSourceOffsetProvider(-1)); + } + + @Test + public void testTvfEmptyBinlogOffsetKeepsPreviousState() { + assertEmptyBinlogOffsetKeepsPreviousState(new TestJdbcTvfSourceOffsetProvider(-1)); + } + + @Test + public void testRepeatedValidBinlogOffsetCleanupIsIdempotent() { + assertRepeatedValidBinlogOffsetCleanupIsIdempotent(new TestJdbcSourceOffsetProvider(-1)); + } + + @Test + public void testTvfRepeatedValidBinlogOffsetCleanupIsIdempotent() { + assertRepeatedValidBinlogOffsetCleanupIsIdempotent(new TestJdbcTvfSourceOffsetProvider(-1)); + } + + @Test + public void testBinlogOffsetRestoredFromPersistInfo() throws Exception { + JdbcSourceOffsetProvider source = new TestJdbcSourceOffsetProvider(-1); + source.updateOffset(new JdbcOffset(Collections.singletonList( + new BinlogSplit(Collections.singletonMap("lsn", "200"))))); + StreamingInsertJob job = mockJobWithPersistInfo(source.getPersistInfo()); + JdbcSourceOffsetProvider restored = new JdbcSourceOffsetProvider(); + + restored.replayIfNeed(job); + + Assert.assertNotNull(restored.currentOffset); + Assert.assertFalse(restored.currentOffset.snapshotSplit()); + Assert.assertEquals("200", ((BinlogSplit) restored.currentOffset.getSplits().get(0)) + .getStartingOffset().get("lsn")); + Assert.assertTrue(restored.chunkHighWatermarkMap.isEmpty()); + } + + @Test + public void testTvfBinlogOffsetRestoredFromPersistInfo() throws Exception { + JdbcSourceOffsetProvider source = new TestJdbcTvfSourceOffsetProvider(-1); + source.updateOffset(new JdbcOffset(Collections.singletonList( + new BinlogSplit(Collections.singletonMap("lsn", "200"))))); + StreamingInsertJob job = mockJobWithPersistInfo(source.getPersistInfo()); + JdbcTvfSourceOffsetProvider restored = new JdbcTvfSourceOffsetProvider(); + + restored.restoreFromPersistInfo(source.getPersistInfo()); + restored.replayIfNeed(job); + + Assert.assertNotNull(restored.currentOffset); + Assert.assertFalse(restored.currentOffset.snapshotSplit()); + Assert.assertEquals("200", ((BinlogSplit) restored.currentOffset.getSplits().get(0)) + .getStartingOffset().get("lsn")); + Assert.assertTrue(restored.chunkHighWatermarkMap.isEmpty()); + } + private static void assertEndOffsetAdvancesWhenCurrentOffsetIsAhead(JdbcSourceOffsetProvider provider) { Map staleEndOffset = Collections.singletonMap("lsn", "100"); Map committedOffset = Collections.singletonMap("lsn", "200"); @@ -91,6 +187,115 @@ private static void assertEndOffsetAdvancesWhenCurrentOffsetIsAhead(JdbcSourceOf Assert.assertEquals("{\"lsn\":\"200\"}", provider.getShowMaxOffset()); } + private static void assertValidBinlogOffsetClearsSnapshotState(JdbcSourceOffsetProvider provider) { + seedSnapshotState(provider); + Map binlogOffset = Collections.singletonMap("lsn", "200"); + + provider.updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(binlogOffset)))); + + Assert.assertTrue(provider.chunkHighWatermarkMap.isEmpty()); + Assert.assertTrue(provider.remainingSplits.isEmpty()); + Assert.assertTrue(provider.finishedSplits.isEmpty()); + assertProgressCleared(provider.committedSplitProgress); + assertProgressCleared(provider.cdcSplitProgress); + Assert.assertEquals("table-schemas", provider.tableSchemas); + Map expectedPersist = new HashMap<>(binlogOffset); + expectedPersist.put(JdbcSourceOffsetProvider.SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + Assert.assertEquals(expectedPersist, provider.binlogOffsetPersist); + String persistInfo = provider.getPersistInfo(); + Assert.assertFalse(persistInfo.contains("source_table:0")); + Assert.assertFalse(persistInfo.contains("source_table:1")); + Assert.assertTrue(persistInfo.contains("table-schemas")); + } + + private static void assertEmptyBinlogOffsetKeepsPreviousState(JdbcSourceOffsetProvider provider) { + seedSnapshotState(provider); + JdbcOffset previousOffset = new JdbcOffset( + Collections.singletonList(snapshotSplit("source_table:0"))); + provider.currentOffset = previousOffset; + provider.hasMoreData = false; + + provider.updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(Collections.emptyMap())))); + + Assert.assertSame(previousOffset, provider.currentOffset); + Assert.assertFalse(provider.hasMoreData); + Assert.assertFalse(provider.chunkHighWatermarkMap.isEmpty()); + Assert.assertFalse(provider.remainingSplits.isEmpty()); + Assert.assertFalse(provider.finishedSplits.isEmpty()); + Assert.assertEquals("source_table", provider.committedSplitProgress.getCurrentSplittingTable()); + Assert.assertEquals("source_table", provider.cdcSplitProgress.getCurrentSplittingTable()); + Assert.assertNull(provider.binlogOffsetPersist); + } + + private static void assertRepeatedValidBinlogOffsetCleanupIsIdempotent( + JdbcSourceOffsetProvider provider) { + seedSnapshotState(provider); + JdbcOffset binlogOffset = new JdbcOffset(Collections.singletonList( + new BinlogSplit(Collections.singletonMap("lsn", "200")))); + + provider.updateOffset(binlogOffset); + String firstPersistInfo = provider.getPersistInfo(); + Map>> clearedHighWatermarkMap = + provider.chunkHighWatermarkMap; + provider.updateOffset(binlogOffset); + + Assert.assertEquals(firstPersistInfo, provider.getPersistInfo()); + Assert.assertSame(clearedHighWatermarkMap, provider.chunkHighWatermarkMap); + Assert.assertTrue(provider.chunkHighWatermarkMap.isEmpty()); + Assert.assertTrue(provider.remainingSplits.isEmpty()); + Assert.assertTrue(provider.finishedSplits.isEmpty()); + assertProgressCleared(provider.committedSplitProgress); + assertProgressCleared(provider.cdcSplitProgress); + } + + private static StreamingInsertJob mockJobWithPersistInfo(String persistInfo) { + StreamingInsertJob job = new ReplayStreamingInsertJob(); + Deencapsulation.setField(job, "jobId", 9001L); + Deencapsulation.setField(job, "syncTables", Collections.emptyList()); + job.setOffsetProviderPersist(persistInfo); + return job; + } + + private static void seedSnapshotState(JdbcSourceOffsetProvider provider) { + SnapshotSplit remaining = snapshotSplit("source_table:1"); + SnapshotSplit finished = snapshotSplit("source_table:0"); + provider.remainingSplits.add(remaining); + provider.finishedSplits.add(finished); + provider.chunkHighWatermarkMap + .computeIfAbsent("source_db.source_table", key -> new HashMap<>()) + .put(finished.getSplitId(), finished.getHighWatermark()); + provider.committedSplitProgress = splitProgress(); + provider.cdcSplitProgress = splitProgress(); + provider.tableSchemas = "table-schemas"; + } + + private static SnapshotSplit snapshotSplit(String splitId) { + return new SnapshotSplit( + splitId, + "source_db.source_table", + Collections.singletonList("id"), + new Object[]{1L}, + new Object[]{2L}, + Collections.singletonMap("lsn", "100")); + } + + private static JdbcSourceOffsetProvider.SplitProgress splitProgress() { + JdbcSourceOffsetProvider.SplitProgress progress = new JdbcSourceOffsetProvider.SplitProgress(); + progress.setCurrentSplittingTable("source_table"); + progress.setNextSplitStart(new Object[]{2L}); + progress.setNextSplitId(2); + return progress; + } + + private static void assertProgressCleared(JdbcSourceOffsetProvider.SplitProgress progress) { + Assert.assertNotNull(progress); + Assert.assertNull(progress.getCurrentSplittingTable()); + Assert.assertNull(progress.getNextSplitStart()); + Assert.assertNull(progress.getNextSplitId()); + } + private static class TestJdbcSourceOffsetProvider extends JdbcSourceOffsetProvider { private final int compareResult; @@ -104,6 +309,12 @@ protected int compareOffset(Map offsetFirst, Map } } + private static class ReplayStreamingInsertJob extends StreamingInsertJob { + ReplayStreamingInsertJob() { + super(); + } + } + private static class TestJdbcTvfSourceOffsetProvider extends JdbcTvfSourceOffsetProvider { private final int compareResult; diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java index 11075ea2d81964..0ad2629ce94db6 100644 --- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java @@ -1039,10 +1039,11 @@ private MySqlSourceConfig generateMySqlConfig( configFactory.debeziumProperties(dbzProps); configFactory.heartbeatInterval(Duration.ofMillis(DEBEZIUM_HEARTBEAT_INTERVAL_MS)); - if (cdcConfig.containsKey(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE)) { - configFactory.splitSize( - Integer.parseInt(cdcConfig.get(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE))); - } + configFactory.splitSize( + Integer.parseInt( + cdcConfig.getOrDefault( + DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE, + DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE_DEFAULT))); // todo: Currently, only one split key is supported; future will require multiple split // keys. diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReader.java index e9cc9e8d9859dc..330f461510bf99 100644 --- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReader.java +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReader.java @@ -294,11 +294,11 @@ private PostgresSourceConfig generatePostgresConfig( throw new RuntimeException("Unknown offset " + startupMode); } - // Set split size if provided - if (cdcConfig.containsKey(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE)) { - configFactory.splitSize( - Integer.parseInt(cdcConfig.get(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE))); - } + configFactory.splitSize( + Integer.parseInt( + cdcConfig.getOrDefault( + DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE, + DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE_DEFAULT))); if (cdcConfig.containsKey(DataSourceConfigKeys.SNAPSHOT_SPLIT_KEY)) { configFactory.chunkKeyColumn(cdcConfig.get(DataSourceConfigKeys.SNAPSHOT_SPLIT_KEY)); diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_finished_restart_fe.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_finished_restart_fe.groovy new file mode 100644 index 00000000000000..6bf0a57eb8269c --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_finished_restart_fe.groovy @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_mysql_job_snapshot_finished_restart_fe", + "docker,mysql,external_docker,external_docker_mysql,nondatalake") { + def jobName = "test_streaming_mysql_job_snapshot_finished_restart_fe" + def tableName = "snapshot_finished_restart_fe" + def mysqlDb = "test_cdc_db" + def totalRows = 5 + def options = new ClusterOptions() + options.setFeNum(1) + options.cloudMode = null + + docker(options) { + def currentDb = (sql "select database()")[0][0] + + sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.${tableName} FORCE""" + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String mysqlPort = context.config.otherConfigs.get("mysql_57_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysqlPort}") { + sql """CREATE DATABASE IF NOT EXISTS ${mysqlDb}""" + sql """DROP TABLE IF EXISTS ${mysqlDb}.${tableName}""" + sql """CREATE TABLE ${mysqlDb}.${tableName} ( + `id` int NOT NULL, + `name` varchar(200), + PRIMARY KEY (`id`) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${mysqlDb}.${tableName} (id, name) VALUES + (1, 'name_1'), + (2, 'name_2'), + (3, 'name_3'), + (4, 'name_4'), + (5, 'name_5')""" + } + + sql """CREATE JOB ${jobName} + ON STREAMING + FROM MYSQL ( + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysqlPort}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root", + "password" = "123456", + "database" = "${mysqlDb}", + "include_tables" = "${tableName}", + "offset" = "snapshot", + "snapshot_split_size" = "1", + "snapshot_parallelism" = "1" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + ) + """ + + try { + Awaitility.await().atMost(300, SECONDS) + .pollInterval(2, SECONDS).until( + { + def jobStatus = sql """ + SELECT Status + FROM jobs("type"="insert") + WHERE Name='${jobName}' AND ExecuteType='STREAMING' + """ + log.info("jobStatus before FE restart: " + jobStatus) + jobStatus.size() == 1 && jobStatus.get(0).get(0) == "FINISHED" + } + ) + + def jobIdRows = sql """ + SELECT Id + FROM jobs("type"="insert") + WHERE Name='${jobName}' AND ExecuteType='STREAMING' + """ + assert jobIdRows.size() == 1 + def jobId = jobIdRows.get(0).get(0).toString() + + def rowsBeforeRestart = sql """ + SELECT COUNT(*), COUNT(DISTINCT id) + FROM ${currentDb}.${tableName} + """ + assert rowsBeforeRestart.size() == 1 + assert rowsBeforeRestart.get(0).get(0) == totalRows + assert rowsBeforeRestart.get(0).get(1) == totalRows + + cluster.restartFrontends() + sleep(60000) + context.reconnectFe() + + // A terminal job must survive replay with its finish time. Otherwise + // the scheduler may treat it as an expired job and remove it. + Awaitility.await().atMost(120, SECONDS) + .pollInterval(2, SECONDS).until( + { + def jobAfterRestart = sql """ + SELECT Id, Status + FROM jobs("type"="insert") + WHERE Name='${jobName}' AND ExecuteType='STREAMING' + """ + log.info("job after FE restart: " + jobAfterRestart) + jobAfterRestart.size() == 1 + && jobAfterRestart.get(0).get(0).toString() == jobId + && jobAfterRestart.get(0).get(1) == "FINISHED" + } + ) + + def rowsAfterRestart = sql """ + SELECT COUNT(*), COUNT(DISTINCT id) + FROM ${currentDb}.${tableName} + """ + assert rowsAfterRestart.size() == 1 + assert rowsAfterRestart.get(0).get(0) == totalRows + assert rowsAfterRestart.get(0).get(1) == totalRows + } catch (Exception ex) { + def showJob = sql """ + SELECT * + FROM jobs("type"="insert") + WHERE Name='${jobName}' + """ + def showTask = sql """ + SELECT * + FROM tasks("type"="insert") + WHERE JobName='${jobName}' + """ + log.info("show job: " + showJob) + log.info("show task: " + showTask) + throw ex + } finally { + sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" + } + } + } +} From 4e3c1b84dd5936b8d751e0a95719350e8cd8e980 Mon Sep 17 00:00:00 2001 From: wudi Date: Thu, 6 Aug 2026 15:37:06 +0800 Subject: [PATCH 03/12] [fix](regression) Fetch streaming job metrics from master FE (#66460) --- .../streaming_job/cdc/test_streaming_mysql_job_metrics.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_metrics.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_metrics.groovy index fc317a3963ae9b..a780a16aaaf314 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_metrics.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_metrics.groovy @@ -92,10 +92,11 @@ suite("test_streaming_mysql_job_metrics", int count = 0 int metricCount = 0 + String masterHttpAddress = getMasterIp() + ":" + getMasterPort("http") while (true) { metricCount = 0 httpTest { - endpoint context.config.feHttpAddress + endpoint masterHttpAddress uri "/metrics?type=json" op "get" check { code, body -> @@ -233,4 +234,3 @@ suite("test_streaming_mysql_job_metrics", assert jobCountRsp.get(0).get(0) == 0 } } - From 30e91ecf53e191f7fdb2bdb0a14d12d8b8a5f974 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Thu, 6 Aug 2026 16:51:49 +0800 Subject: [PATCH 04/12] [fix](fe) give ConfField.description a single string in the streaming-job config (#66541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? Issue Number: close #xxx Related PR: #66342, #66238 Problem Summary: master does not compile. `fe-common` fails at ``` Config.java:[1177,65] annotation value not of an allowable type ``` #66342 retyped `ConfigBase.ConfField.description()` from `String[]` to `String` and rewrote all 419 call sites accordingly. #66238 landed shortly after with ```java @ConfField(mutable = true, masterOnly = true, description = { "Minimum interval in seconds between snapshot offset persistence operations"}) public static int streaming_job_snapshot_offset_persist_interval_sec = 300; ``` written against the older `String[]` signature. The two changes are textually disjoint, so git merges them without a conflict and neither pull request could see the other — each was green on its own base. The seven further errors reported in the same module are secondary. The bad annotation value aborts the annotation-processing round, so lombok never contributes its generated members, and `@Slf4j`'s `log` plus `@AllArgsConstructor`/`@Data`'s constructors go missing: ``` DiskUtils.java:[70,13] cannot find symbol JobBaseConfig ... constructor cannot be applied to given types AbstractSourceSplit ... constructor cannot be applied to given types ``` All seven disappear once the annotation value is fixed; nothing else in the tree needed a change. This is the only remaining array-form description under `fe/` (`grep -rnE 'description\s*=\s*\{'`), and the wrapped-argument layout matches the neighbouring long descriptions such as `max_create_table_timeout_second`. `ConfigTest.testConfFieldDescriptionsAreEnglishStrings`, the guard #66342 added, reflects over the annotation at runtime, so it cannot catch a compile-time signature mismatch; it passes here because the text is already English. Verified on a clean checkout of master `4e3c1b84dd5`: - reproduced the failure before the change, and confirmed all eight errors are gone after it - full FE reactor `mvn test-compile` (checkstyle included): **74/74 modules SUCCESS** - `fe-common` module tests: **157 tests, 0 failures, 0 skipped**, including `ConfigTest.testConfFieldDescriptionsAreEnglishStrings` - the tests both colliding PRs added — `StreamingInsertJobOffsetPersistenceTest`, `JdbcSourceOffsetProviderOffsetTest`, `SessionVariablesTest`: **39 tests, 0 failures, 0 skipped** --- .../src/main/java/org/apache/doris/common/Config.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 114112d9e58523..af7d3a434082e1 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1174,8 +1174,8 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int streaming_task_min_timeout_sec = 300; - @ConfField(mutable = true, masterOnly = true, description = { - "Minimum interval in seconds between snapshot offset persistence operations"}) + @ConfField(mutable = true, masterOnly = true, + description = "Minimum interval in seconds between snapshot offset persistence operations") public static int streaming_job_snapshot_offset_persist_interval_sec = 300; @ConfField(mutable = true, masterOnly = true) From e307ec6b94d1d4d6c95af8f681bd015fb7600d76 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 6 Aug 2026 16:56:10 +0800 Subject: [PATCH 05/12] [fix](regression) Stabilize BE compaction tasks test (#65637) Problem Summary: The test inserts five batches, exactly matching the default cumulative compaction threshold. The partition can become visible in FE before its latest visible version reaches BE, so manual cumulative compaction may return `E-2000: _input_rowsets is empty`. The generic helper ignored E-2000, leaving no tracker record for the test to verify. The fix targets the BE hosting the tablet, retries only E-2000 while requesting tablet reports, and waits for that MANUAL cumulative tracker record to reach FINISHED or FAILED. The case still uses the original five batches, and other trigger errors fail immediately. --- .../test_be_compaction_tasks.groovy | 93 ++++++++++++------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/regression-test/suites/compaction/test_be_compaction_tasks.groovy b/regression-test/suites/compaction/test_be_compaction_tasks.groovy index 69ec6a5cc6ae22..04334b31640c11 100644 --- a/regression-test/suites/compaction/test_be_compaction_tasks.groovy +++ b/regression-test/suites/compaction/test_be_compaction_tasks.groovy @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -import org.codehaus.groovy.runtime.IOGroovyMethods +import org.awaitility.Awaitility + +import java.util.concurrent.TimeUnit suite("test_be_compaction_tasks", "p0") { def tableName = "test_be_compaction_tasks_tbl" @@ -25,8 +27,6 @@ suite("test_be_compaction_tasks", "p0") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) - String backend_id = backendId_to_backendIP.keySet()[0] - try { // Step 2: Create table with disable_auto_compaction sql """ DROP TABLE IF EXISTS ${tableName} """ @@ -72,35 +72,64 @@ suite("test_be_compaction_tasks", "p0") { assertTrue(tablets.size() > 0) def tablet = tablets[0] String tablet_id = tablet.TabletId + String backend_id = tablet.BackendId + String backend_ip = backendId_to_backendIP[backend_id] + String backend_http_port = backendId_to_backendHttpPort[backend_id] + assertTrue(backend_ip != null && backend_http_port != null, + "Backend address should exist for backend " + backend_id) + + // Step 6: A new partition's visible version reaches BE asynchronously. Trigger tablet reports and + // retry only E-2000 until cumulative compaction has visible input rowsets. + long lastReportTime = 0 + Awaitility.await().atMost(120, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until { + def (code, out, err) = be_run_cumulative_compaction(backend_ip, backend_http_port, tablet_id) + assertEquals(0, code, "Failed to trigger compaction: stdout=${out}, stderr=${err}") + def triggerResult = parseJson(out.trim()) + String triggerStatus = triggerResult.status.toString() + if (triggerStatus.equalsIgnoreCase("success")) { + return true + } + assertEquals("E-2000", triggerStatus, + "Unexpected compaction response: stdout=${out}, stderr=${err}") + + long now = System.currentTimeMillis() + if (now - lastReportTime >= 5000) { + be_report_tablet(backend_ip, backend_http_port.toInteger()) + lastReportTime = now + } + return false + } - // Step 6: Trigger cumulative compaction via HTTP API and wait for completion - trigger_and_wait_compaction(tableName, "cumulative") - - // Step 7: After compaction - query system table with WHERE STATUS = 'FINISHED' - def finishedResult = sql """ SELECT * FROM information_schema.be_compaction_tasks WHERE STATUS = 'FINISHED' """ - logger.info("Finished compaction tasks: " + finishedResult.size()) - assertTrue(finishedResult.size() > 0, "Expected at least one FINISHED compaction task") - - // Step 8: Field validation - verify key fields are non-null and reasonable - // Query specific fields for the completed compaction on our tablet - def fieldResult = sql_return_maparray """ - SELECT BACKEND_ID, COMPACTION_ID, TABLE_ID, PARTITION_ID, TABLET_ID, - COMPACTION_TYPE, STATUS, TRIGGER_METHOD, COMPACTION_SCORE, - SCHEDULED_TIME, START_TIME, END_TIME, ELAPSED_TIME_MS, - INPUT_ROWSETS_COUNT, INPUT_ROW_NUM, INPUT_DATA_SIZE, - INPUT_INDEX_SIZE, INPUT_TOTAL_SIZE, INPUT_SEGMENTS_NUM, - INPUT_VERSION_RANGE, - OUTPUT_ROW_NUM, OUTPUT_DATA_SIZE, OUTPUT_SEGMENTS_NUM, - IS_VERTICAL - FROM information_schema.be_compaction_tasks - WHERE TABLET_ID = ${tablet_id} AND STATUS = 'FINISHED' - ORDER BY COMPACTION_ID DESC - LIMIT 1 - """ + // Step 7: Wait for the tracker record itself to reach a terminal state. The HTTP endpoint can return + // success after two seconds while compaction is still running. + def fieldResult = [] + Awaitility.await().atMost(60, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until { + fieldResult = sql_return_maparray """ + SELECT BACKEND_ID, COMPACTION_ID, TABLE_ID, PARTITION_ID, TABLET_ID, + COMPACTION_TYPE, STATUS, TRIGGER_METHOD, COMPACTION_SCORE, + SCHEDULED_TIME, START_TIME, END_TIME, ELAPSED_TIME_MS, + INPUT_ROWSETS_COUNT, INPUT_ROW_NUM, INPUT_DATA_SIZE, + INPUT_INDEX_SIZE, INPUT_TOTAL_SIZE, INPUT_SEGMENTS_NUM, + INPUT_VERSION_RANGE, + OUTPUT_ROW_NUM, OUTPUT_DATA_SIZE, OUTPUT_SEGMENTS_NUM, + IS_VERTICAL, STATUS_MSG + FROM information_schema.be_compaction_tasks + WHERE TABLET_ID = ${tablet_id} + AND COMPACTION_TYPE = 'cumulative' + AND TRIGGER_METHOD = 'MANUAL' + ORDER BY COMPACTION_ID DESC + LIMIT 1 + """ + return fieldResult.size() == 1 && + (fieldResult[0].STATUS.toString() in ["FINISHED", "FAILED"]) + } logger.info("Field validation result: " + fieldResult) - assertTrue(fieldResult.size() > 0, "Expected FINISHED record for tablet " + tablet_id) def record = fieldResult[0] + assertEquals("FINISHED", record.STATUS.toString(), + "Compaction failed for tablet ${tablet_id}: ${record.STATUS_MSG}") + + // Verify key fields are non-null and reasonable // Verify key fields are non-null assertTrue(record.BACKEND_ID != null && Long.parseLong(record.BACKEND_ID.toString()) > 0, "BACKEND_ID should be positive") @@ -140,11 +169,11 @@ suite("test_be_compaction_tasks", "p0") { assertTrue(record.COMPACTION_SCORE != null && Long.parseLong(record.COMPACTION_SCORE.toString()) >= 0, "COMPACTION_SCORE should be non-negative") - // Step 9: TRIGGER_METHOD check - manual triggered should show 'MANUAL' + // Step 8: TRIGGER_METHOD check - manual triggered should show 'MANUAL' assertTrue(record.TRIGGER_METHOD != null && record.TRIGGER_METHOD.toString() == "MANUAL", "TRIGGER_METHOD should be MANUAL for manually triggered compaction, got: " + record.TRIGGER_METHOD) - // Step 10: Filter test - WHERE tablet_id = X AND status = 'FINISHED' + // Step 9: Filter test - WHERE tablet_id = X AND status = 'FINISHED' def filterResult = sql """ SELECT COUNT(*) FROM information_schema.be_compaction_tasks WHERE TABLET_ID = ${tablet_id} AND STATUS = 'FINISHED' @@ -152,7 +181,7 @@ suite("test_be_compaction_tasks", "p0") { logger.info("Filter result (tablet_id + status): " + filterResult) assertTrue(filterResult[0][0] > 0, "Expected at least one record matching filter") - // Step 11: Non-existent tablet - WHERE tablet_id = 999999999 returns empty + // Step 10: Non-existent tablet - WHERE tablet_id = 999999999 returns empty def emptyResult = sql """ SELECT * FROM information_schema.be_compaction_tasks WHERE TABLET_ID = 999999999 @@ -161,7 +190,7 @@ suite("test_be_compaction_tasks", "p0") { assertEquals(0, emptyResult.size()) } finally { - // Step 12: Cleanup + // Step 11: Cleanup try_sql("DROP TABLE IF EXISTS ${tableName} FORCE") } } From e1a198b6d657885c5f77e9efa99e31fc02f2a37b Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 6 Aug 2026 16:59:32 +0800 Subject: [PATCH 06/12] [fix](compaction) Keep latest 10 versions unmerged during schema change (#66337) Related PR: #63112 Problem Summary: This PR takes over #63112 and adds explicit local and Cloud unit-test coverage. While a schema change target tablet is `TABLET_NOTREADY`, cumulative compaction should merge only older rowsets and leave the newest 10 versions unmerged. The filter used the inverse comparison, skipping older rowsets and selecting the newest versions. A compaction output could then cross the base tablet's maximum version and prevent incremental schema-change conversion with `VERSION_ALREADY_MERGED`. This change reverses the comparison in both local and Cloud size-based cumulative compaction policies. The new tests verify that versions 2 through 10 are selected and versions 11 through 20 remain unmerged. ### Release note Fix schema changes that could fail when cumulative compaction merged the latest versions on the new tablet. --- .../cloud_cumulative_compaction_policy.cpp | 2 +- .../cumulative_compaction_policy.cpp | 2 +- ...loud_cumulative_compaction_policy_test.cpp | 34 ++++++++++++++ .../cumulative_compaction_policy_test.cpp | 47 +++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 465f2c2c31e040..890527b7e173c5 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -169,7 +169,7 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets( if (tablet->tablet_state() == TABLET_NOTREADY) { // If tablet under alter, keep latest 10 version so that base tablet max version // not merged in new tablet, and then we can copy data from base tablet - if (rowset->version().second < max_version - 10) { + if (rowset->version().second > max_version - 10) { continue; } } diff --git a/be/src/storage/compaction/cumulative_compaction_policy.cpp b/be/src/storage/compaction/cumulative_compaction_policy.cpp index 7ab7a8e34ef875..503fca42066e02 100644 --- a/be/src/storage/compaction/cumulative_compaction_policy.cpp +++ b/be/src/storage/compaction/cumulative_compaction_policy.cpp @@ -313,7 +313,7 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets( if (tablet->tablet_state() == TABLET_NOTREADY) { // If tablet under alter, keep latest 10 version so that base tablet max version // not merged in new tablet, and then we can copy data from base tablet - if (rowset->version().second < max_version - 10) { + if (rowset->version().second > max_version - 10) { continue; } } diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp index 2a20b5b3cbafd6..743391b3af7ce0 100644 --- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp +++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp @@ -178,6 +178,40 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, new_cumulative_point) { EXPECT_EQ(policy.new_cumulative_point(&_tablet, output_rowset, version, 2), 6); } +TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, + pick_input_rowsets_notready_keeps_latest_versions) { + auto base_rowset = create_rowset(Version(0, 1), 1, false, kGiB); + ASSERT_NE(nullptr, base_rowset); + ASSERT_TRUE(_tablet_meta->add_rs_meta(base_rowset->rowset_meta()).ok()); + + std::vector candidate_rowsets; + for (int64_t version = 2; version <= 20; ++version) { + auto rowset = create_rowset(Version(version, version), 1, true, kMiB); + ASSERT_NE(nullptr, rowset); + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset->rowset_meta()).ok()); + candidate_rowsets.push_back(rowset); + } + + CloudTablet tablet(_engine, _tablet_meta); + ASSERT_TRUE(tablet.set_tablet_state(TABLET_NOTREADY).ok()); + tablet._base_size = kGiB; + + std::vector input_rowsets; + Version last_delete_version {-1, -1}; + size_t compaction_score = 0; + + CloudSizeBasedCumulativeCompactionPolicy policy; + auto picked_size = policy.pick_input_rowsets(&tablet, candidate_rowsets, 100, 5, &input_rowsets, + &last_delete_version, &compaction_score, true); + + EXPECT_EQ(9, picked_size); + ASSERT_EQ(9, input_rowsets.size()); + EXPECT_EQ(9, compaction_score); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(10, input_rowsets.back()->end_version()); + EXPECT_EQ(Version(-1, -1), last_delete_version); +} + TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_large_head_not_repeated_when_output_below_promotion) { CloudTablet _tablet(_engine, _tablet_meta); diff --git a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp index ce722df19b6ff9..ad1c536451da7f 100644 --- a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp +++ b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp @@ -506,6 +506,53 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_normal) { EXPECT_EQ(-1, last_delete_version.second); } +TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_notready_keeps_latest_versions) { + std::vector rs_metas; + + RowsetMetaSharedPtr base_rowset(new RowsetMeta()); + init_rs_meta(base_rowset, 0, 1); + base_rowset->set_total_disk_size(kGiB); + base_rowset->set_segments_overlap(NONOVERLAPPING); + rs_metas.push_back(base_rowset); + + for (int64_t version = 2; version <= 20; ++version) { + RowsetMetaSharedPtr rowset(new RowsetMeta()); + init_rs_meta(rowset, version, version); + rowset->set_total_disk_size(kMiB); + rowset->set_num_segments(1); + rowset->set_segments_overlap(OVERLAPPING); + rs_metas.push_back(rowset); + } + + for (const auto& rowset : rs_metas) { + ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok()); + } + + TabletSharedPtr tablet( + new Tablet(_engine, _tablet_meta, nullptr, CUMULATIVE_SIZE_BASED_POLICY)); + ASSERT_TRUE(tablet->init().ok()); + ASSERT_TRUE(tablet->set_tablet_state(TABLET_NOTREADY).ok()); + tablet->calculate_cumulative_point(); + + EXPECT_EQ(2, tablet->cumulative_layer_point()); + auto candidate_rowsets = tablet->pick_candidate_rowsets_to_cumulative_compaction(); + ASSERT_EQ(19, candidate_rowsets.size()); + + std::vector input_rowsets; + Version last_delete_version {-1, -1}; + size_t compaction_score = 0; + auto picked_size = tablet->_cumulative_compaction_policy->pick_input_rowsets( + tablet.get(), candidate_rowsets, 100, 5, &input_rowsets, &last_delete_version, + &compaction_score, config::enable_delete_when_cumu_compaction); + + EXPECT_EQ(9, picked_size); + ASSERT_EQ(9, input_rowsets.size()); + EXPECT_EQ(9, compaction_score); + EXPECT_EQ(2, input_rowsets.front()->start_version()); + EXPECT_EQ(10, input_rowsets.back()->end_version()); + EXPECT_EQ(Version(-1, -1), last_delete_version); +} + TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_big_base) { std::vector rs_metas; init_rs_meta_big_base(&rs_metas); From 8e4bd4d5f11d68ddd5fd4ae2ee153ad8d6779323 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Thu, 6 Aug 2026 17:14:46 +0800 Subject: [PATCH 07/12] [fix](be) Stop spill GC threads in workload group test teardown (#66534) ### What problem does this PR solve? Issue Number: None Related PR: #66328 Problem Summary: `WorkloadGroupManagerTest` creates and initializes a `SpillFileManager` for every test case. Initialization starts a background spill GC thread, but the fixture did not stop or delete the manager during teardown. Each subsequent test overwrote the global manager pointer, leaving the previous manager and its GC thread alive. The leaked threads could continue accessing process-wide test utilities during static destruction and trigger an ASAN heap-use-after-free when the test binary exited. They could also interfere with tests that use global spill GC debug points. This change releases workload-group-owned query resources first, then stops and joins the spill GC thread, deletes the manager, and finally removes the temporary spill directory. This matches the lifecycle used by other spill-related test fixtures. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - `GTEST_REPEAT=10 ./run-be-ut.sh --run --filter='WorkloadGroupManagerTest.*:DebugPointsTest.AddTest:SpillFileTest.RetryPreservesDirectoryQueuedAfterPendingDrain' -j 8` - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. This only corrects test fixture resource teardown. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/test/runtime/workload_group/workload_group_manager_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/be/test/runtime/workload_group/workload_group_manager_test.cpp b/be/test/runtime/workload_group/workload_group_manager_test.cpp index 55213d9922fcbc..bf7bc9e57d10ad 100644 --- a/be/test/runtime/workload_group/workload_group_manager_test.cpp +++ b/be/test/runtime/workload_group/workload_group_manager_test.cpp @@ -91,6 +91,8 @@ class WorkloadGroupManagerTest : public testing::Test { } void TearDown() override { _wg_manager.reset(); + ExecEnv::GetInstance()->spill_file_mgr()->stop(); + SAFE_DELETE(ExecEnv::GetInstance()->_spill_file_mgr); ExecEnv::GetInstance()->_runtime_query_statistics_mgr->stop_report_thread(); SAFE_DELETE(ExecEnv::GetInstance()->_runtime_query_statistics_mgr); From 38fb05eaaab72739b34c942503bb0000f0967dd5 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Thu, 6 Aug 2026 21:35:07 +0800 Subject: [PATCH 08/12] [fix](be) Normalize constant nullable TopN predicate results (#66442) Problem Summary: Nullable TopN comparisons can return a `ColumnConst` that wraps a `ColumnNullable` result, especially when an input batch is entirely NULL. The previous normalization only handled a top-level `ColumnNullable`, so the constant nullable result could reach filter execution without converting SQL NULL into the NULLS FIRST or NULLS LAST filter decision. This change recursively normalizes constant wrappers, collapses nullable Boolean results according to the TopN null ordering, preserves the constant shape, and aligns the unit-test expression metadata with production nullability. The new unit test covers both NULLS FIRST and NULLS LAST through `execute_column` and `execute_filter`. --- be/src/exprs/vtopn_pred.h | 37 ++++++++++------ .../format_v2/parquet/parquet_scan_test.cpp | 2 +- be/test/runtime/runtime_predicate_test.cpp | 44 ++++++++++++++++++- 3 files changed, 68 insertions(+), 15 deletions(-) diff --git a/be/src/exprs/vtopn_pred.h b/be/src/exprs/vtopn_pred.h index c2fe7c2fd0d5aa..af680955d05c3b 100644 --- a/be/src/exprs/vtopn_pred.h +++ b/be/src/exprs/vtopn_pred.h @@ -121,19 +121,7 @@ class VTopNPred : public VExpr { RETURN_IF_ERROR(_function->execute(nullptr, temp_block, arguments, num_columns_without_result, temp_block.rows())); result_column = std::move(temp_block.get_by_position(num_columns_without_result).column); - if (auto mutable_result = IColumn::mutate(std::move(result_column)); - auto* nullable = check_and_get_column(*mutable_result)) { - auto& values = assert_cast(*nullable->get_nested_column_ptr()).get_data(); - const auto& null_map = nullable->get_null_map_data(); - // Master validates execute_type() against the physical result column. Collapse SQL - // NULL to the filter decision here: NULLS FIRST keeps it, while NULLS LAST rejects it. - for (size_t row = 0; row < values.size(); ++row) { - values[row] = null_map[row] ? _predicate->nulls_first() : values[row]; - } - result_column = nullable->get_nested_column_ptr(); - } else { - result_column = std::move(mutable_result); - } + result_column = _normalize_filter_result(std::move(result_column)); DCHECK_EQ(result_column->size(), count); return Status::OK(); } @@ -284,6 +272,29 @@ class VTopNPred : public VExpr { } private: + ColumnPtr _normalize_filter_result(ColumnPtr column) const { + const size_t rows = column->size(); + if (const auto* constant = check_and_get_column(*column)) { + auto nested = _normalize_filter_result(constant->get_data_column_ptr()); + return ColumnConst::create(std::move(nested), rows); + } + + auto mutable_column = IColumn::mutate(std::move(column)); + if (auto* nullable = check_and_get_column(*mutable_column)) { + auto& values = assert_cast(*nullable->get_nested_column_ptr()).get_data(); + auto& null_map = nullable->get_null_map_data(); + // Collapse SQL NULL to the filter decision: NULLS FIRST keeps it, while NULLS LAST + // rejects it. Preserve the nullable wrapper so strict block execution still matches + // the expression's declared Nullable(Boolean) type. + for (size_t row = 0; row < values.size(); ++row) { + values[row] = null_map[row] ? _predicate->nulls_first() : values[row]; + } + nullable->fill_false_to_nullmap(rows); + return mutable_column; + } + return mutable_column; + } + int _source_node_id; std::string _expr_name; RuntimePredicate* _predicate = nullptr; diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index cc95b51cebf751..eaca8f772c8c93 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -931,7 +931,7 @@ PreparedTopNConjunct create_topn_conjunct(RuntimeState* state, int column_id, TExprNode topn_node; topn_node.__set_type(std::make_shared()->to_thrift()); - topn_node.__set_is_nullable(false); + topn_node.__set_is_nullable(data_type->is_nullable()); auto topn = VTopNPred::create_shared(topn_node, 10, nullptr); topn->add_child(VSlotRef::create_shared(column_id, column_id, -1, data_type, "topn_column")); auto conjunct = VExprContext::create_shared(std::move(topn)); diff --git a/be/test/runtime/runtime_predicate_test.cpp b/be/test/runtime/runtime_predicate_test.cpp index c6298b91a5b480..baf27e72ddc859 100644 --- a/be/test/runtime/runtime_predicate_test.cpp +++ b/be/test/runtime/runtime_predicate_test.cpp @@ -76,7 +76,7 @@ VExprContextSPtr create_prepared_topn_expr(MockRuntimeState* state, const DataTy TExprNode node; node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN)); - node.__set_is_nullable(false); + node.__set_is_nullable(data_type->is_nullable()); auto expr = VTopNPred::create_shared(node, SOURCE_NODE_ID, nullptr); expr->add_child(VSlotRef::create_shared(SLOT_ID, 0, -1, data_type, "topn_column")); auto context = VExprContext::create_shared(std::move(expr)); @@ -306,4 +306,46 @@ TEST(RuntimePredicateTest, NullableNullsFirstTopNSupportsRawNullSemantics) { context->close(); } +TEST(RuntimePredicateTest, TopNPredicateNormalizesConstNullableComparisonResult) { + constexpr size_t rows = 3; + const auto type = make_nullable(std::make_shared()); + + for (const bool nulls_first : {false, true}) { + SCOPED_TRACE(nulls_first ? "NULLS FIRST" : "NULLS LAST"); + MockRuntimeState state; + auto context = create_prepared_topn_expr(&state, type, Field::create_field(3), + true, nulls_first); + + Block block; + block.insert( + {ColumnNullable::create(ColumnInt32::create(rows, 0), ColumnUInt8::create(rows, 1)), + type, "topn_column"}); + + ColumnPtr result_column; + auto status = context->root()->execute_column(context.get(), &block, nullptr, block.rows(), + result_column); + ASSERT_TRUE(status.ok()) << status; + ASSERT_TRUE(is_column_const(*result_column)); + EXPECT_FALSE(result_column->is_null_at(0)); + EXPECT_EQ(result_column->get_bool(0), nulls_first); + + int result_column_id = -1; + status = context->execute(&block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + const auto& strict_result = block.get_by_position(result_column_id); + ASSERT_TRUE(is_column_const(*strict_result.column)); + EXPECT_NE(nullptr, check_and_get_column_with_const(*strict_result.column)); + EXPECT_FALSE(strict_result.column->is_null_at(0)); + EXPECT_EQ(strict_result.column->get_bool(0), nulls_first); + + IColumn::Filter filter(rows, 1); + bool can_filter_all = false; + status = context->execute_filter(&block, filter.data(), rows, false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(filter, IColumn::Filter(rows, nulls_first)); + EXPECT_EQ(can_filter_all, !nulls_first); + context->close(); + } +} + } // namespace doris From a82564ced5dba0c00317a892f30d1336a9e22016 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 6 Aug 2026 22:19:26 +0800 Subject: [PATCH 09/12] [fix](iceberg) Fix MVCC and nested schema evolution edge cases (#66345) ### What problem does this PR solve? This PR fixes correctness and memory-usage edge cases in Iceberg MVCC writes and nested schema evolution: - An explicitly empty read could be reinterpreted as the latest snapshot after a concurrent first append. A MERGE could then miss the concurrent row and silently insert a duplicate. - Write-schema validation used the INSERT column subset instead of the complete bind-time target schema, rejecting valid partial and static-partition INSERTs while weakening ordinal drift checks. - Iceberg v3 row-lineage metadata columns could be mistaken for user-schema drift. - Parent null-mask handling for nested ARRAY/MAP values could scan descendant entry maps and allocate entry-sized scratch even when the inherited mask was absent or all-clear. - Strict cast failure semantics were incorrectly used to mark a struct field physically non-nullable even though BE narrowing casts still return `ColumnNullable`. - Nested DESCRIBE comments were not emitted as round-trippable SQL literals. The fix preserves the explicit empty-snapshot marker through scan planning and keeps RowDelta conflict validation anchored at table creation. In the concurrent empty-table MERGE/INSERT window, MERGE keeps reading the original empty view and its commit conflicts with the concurrent first append instead of producing duplicate data. Write planning now carries an immutable complete target-schema snapshot separately from the write subset, validates column order, type, and Iceberg field id, and excludes engine-generated row-lineage metadata from the user-schema comparison. Nested collection alignment gates descendant scans on an inherited mask that can actually hide a row, and struct constructors preserve the physical nullability of cast results. ### Release note Fix Iceberg empty-snapshot concurrency, partial-write schema validation, nested collection memory usage, cast-result nullability, and nested DESCRIBE comment rendering. ### Check List (For Author) - Test - [x] Regression tests - [x] Unit tests - Behavior changed: - [x] Yes. Correctness and memory-usage fixes for the affected edge cases. - Does this need documentation? - [x] No. ### Tests - Iceberg write-plan tests: 50 passed - Iceberg concurrent empty-snapshot barrier test: passed - BE `TableReaderTest`: 95 passed - FE connector sink, binding, executor, and struct tests: 38 passed - FE Checkstyle: 0 violations - Clang Formatter 16: passed - `git diff --check`: passed --- be/src/format_v2/table_reader.h | 241 +++++++- be/test/format_v2/table_reader_test.cpp | 229 +++++++- .../java/org/apache/doris/catalog/Column.java | 4 + .../apache/doris/common/util/SqlUtils.java | 6 + .../connector/hive/HiveConnectorMetadata.java | 112 +--- .../hive/HiveConnectorTransaction.java | 170 +++++- .../hive/HiveWriteMetadataSnapshot.java | 258 +++++++++ .../connector/hive/HiveWritePlanProvider.java | 87 ++- .../hive/HiveConnectorMetadataSchemaTest.java | 18 + ...onnectorMetadataSiblingDelegationTest.java | 8 +- .../hive/HiveConnectorTransactionTest.java | 104 +++- .../hive/HiveWritePlanProviderTest.java | 214 +++++++- .../doris/connector/hms/CachingHmsClient.java | 16 + .../apache/doris/connector/hms/HmsClient.java | 21 + .../doris/connector/hms/HmsTableInfo.java | 26 + .../doris/connector/hms/ThriftHmsClient.java | 70 ++- .../hms/ThriftHmsClientWriteAcidTest.java | 48 ++ .../iceberg/IcebergConnectorMetadata.java | 53 +- .../iceberg/IcebergConnectorTransaction.java | 42 +- .../iceberg/IcebergLatestSnapshotCache.java | 22 +- .../iceberg/IcebergPartitionUtils.java | 10 + .../iceberg/IcebergScanPlanProvider.java | 49 +- .../connector/iceberg/IcebergTableHandle.java | 67 ++- .../connector/iceberg/IcebergTypeMapping.java | 4 +- .../iceberg/IcebergWriteContext.java | 57 +- .../iceberg/IcebergWritePlanProvider.java | 245 ++++++++- .../IcebergConnectorMetadataMvccTest.java | 7 +- ...nnectorMetadataPartitionViewCacheTest.java | 84 +++ .../iceberg/IcebergConnectorMetadataTest.java | 3 + .../IcebergConnectorTransactionTest.java | 118 ++++ .../iceberg/IcebergScanPlanProviderTest.java | 91 +++ .../iceberg/IcebergWritePlanProviderTest.java | 519 +++++++++++++++++- .../connector/spi/ConnectorTableSchema.java | 29 +- .../spi/handle/ConnectorWriteHandle.java | 17 + .../spi/write/ConnectorWritePlanProvider.java | 19 + .../spi/ConnectorPluginSurfaceTest.java | 10 +- .../resources/connector-plugin-surface.txt | 21 + fe/fe-connector/pom.xml | 2 +- .../common/proc/IndexSchemaProcNode.java | 4 +- .../converter/ConnectorColumnConverter.java | 87 ++- .../mvcc/PluginDrivenMvccExternalTable.java | 5 + .../plugin/PluginDrivenExternalTable.java | 71 ++- .../plugin/PluginDrivenSchemaCacheValue.java | 17 +- .../doris/nereids/StatementContext.java | 12 + .../translator/PhysicalPlanTranslator.java | 49 +- .../nereids/rules/analysis/BindSink.java | 83 ++- ...TableSinkToPhysicalConnectorTableSink.java | 3 + ...kToPhysicalExternalRowLevelDeleteSink.java | 1 + ...nkToPhysicalExternalRowLevelMergeSink.java | 1 + .../functions/scalar/CreateNamedStruct.java | 4 +- .../expressions/literal/StructLiteral.java | 11 +- .../ExternalRowLevelDeletePlanBuilder.java | 6 + .../ExternalRowLevelMergePlanBuilder.java | 6 + .../ExternalRowLevelUpdatePlanBuilder.java | 6 + .../logical/LogicalConnectorTableSink.java | 82 ++- .../LogicalExternalRowLevelDeleteSink.java | 37 +- .../LogicalExternalRowLevelMergeSink.java | 38 +- .../physical/PhysicalConnectorTableSink.java | 147 ++++- .../PhysicalExternalRowLevelDeleteSink.java | 53 +- .../PhysicalExternalRowLevelMergeSink.java | 122 ++-- .../doris/nereids/util/SqlLiteralUtils.java | 5 +- .../doris/planner/PluginDrivenTableSink.java | 59 +- .../common/proc/IndexSchemaProcNodeTest.java | 24 +- .../ConnectorColumnConverterTest.java | 42 +- .../PluginDrivenMvccExternalTableTest.java | 12 + ...luginDrivenExternalTablePartitionTest.java | 5 +- .../plugin/PluginDrivenExternalTableTest.java | 36 ++ ...ysicalPlanTranslatorAdmissionGateTest.java | 136 ++++- ...lPlanTranslatorIcebergRowLevelDmlTest.java | 4 + .../BindConnectorSinkStaticPartitionTest.java | 107 +++- .../literal/StructLiteralTest.java | 41 ++ .../ExternalRowLevelMergePlanBuilderTest.java | 23 +- ...ExternalRowLevelUpdatePlanBuilderTest.java | 28 +- .../IcebergRowLevelDmlTransformTest.java | 11 +- .../PhysicalConnectorTableSinkTest.java | 30 +- ...PhysicalExternalRowLevelMergeSinkTest.java | 14 + .../planner/PluginDrivenTableSinkTest.java | 33 ++ .../java/org/apache/doris/catalog/Type.java | 19 +- 78 files changed, 4149 insertions(+), 426 deletions(-) create mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteMetadataSnapshot.java diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 07d2ed652a4306..e95d8219576bce 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -1232,8 +1232,22 @@ class TableReader { if (const auto* array_type = typeid_cast(table_type.get())) { const auto& array_column = assert_cast(**column); ColumnPtr nested_column = array_column.get_data_ptr(); - RETURN_IF_ERROR( - _align_column_nullability(&nested_column, array_type->get_nested_type())); + NullMap descendant_parent_null_map; + // Collection entries use offset coordinates, so inherited row masks must be projected + // only when a required descendant can consume them. This avoids scratch proportional + // to all array entries for the common all-required schema. + const NullMap* descendant_parent_null_map_ptr = nullptr; + if (_requires_collection_parent_null_map( + nullable_parent_null_map, nested_column, array_type->get_nested_type(), + array_column.size(), array_column.get_offsets())) { + descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + nullptr, nullable_parent_null_map, array_column.size(), + array_column.get_offsets(), nested_column->size(), + &descendant_parent_null_map); + } + RETURN_IF_ERROR(_align_column_nullability(&nested_column, array_type->get_nested_type(), + descendant_parent_null_map_ptr)); *column = ColumnArray::create(nested_column, array_column.get_offsets_ptr()); return Status::OK(); } @@ -1241,8 +1255,25 @@ class TableReader { const auto& map_column = assert_cast(**column); ColumnPtr key_column = map_column.get_keys_ptr(); ColumnPtr value_column = map_column.get_values_ptr(); - RETURN_IF_ERROR(_align_column_nullability(&key_column, map_type->get_key_type())); - RETURN_IF_ERROR(_align_column_nullability(&value_column, map_type->get_value_type())); + NullMap descendant_parent_null_map; + const NullMap* descendant_parent_null_map_ptr = nullptr; + if (_requires_collection_parent_null_map(nullable_parent_null_map, key_column, + map_type->get_key_type(), map_column.size(), + map_column.get_offsets()) || + _requires_collection_parent_null_map(nullable_parent_null_map, value_column, + map_type->get_value_type(), map_column.size(), + map_column.get_offsets())) { + // Keys and values share offsets, so one projected mask safely covers both streams. + descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + nullptr, nullable_parent_null_map, map_column.size(), + map_column.get_offsets(), key_column->size(), + &descendant_parent_null_map); + } + RETURN_IF_ERROR(_align_column_nullability(&key_column, map_type->get_key_type(), + descendant_parent_null_map_ptr)); + RETURN_IF_ERROR(_align_column_nullability(&value_column, map_type->get_value_type(), + descendant_parent_null_map_ptr)); *column = ColumnMap::create(key_column, value_column, map_column.get_offsets_ptr()); return Status::OK(); } @@ -1575,27 +1606,193 @@ class TableReader { return column.get(); } + static bool _requires_parent_null_map_for_alignment(const ColumnPtr& column, + const DataTypePtr& table_type) { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(table_type != nullptr); + if (table_type->is_nullable()) { + const auto& nested_type = + assert_cast(*table_type).get_nested_type(); + if (const auto* nullable_column = check_and_get_column(*column)) { + return _requires_parent_null_map_for_alignment( + nullable_column->get_nested_column_ptr(), nested_type); + } + return _requires_parent_null_map_for_alignment(column, nested_type); + } + if (const auto* nullable_column = check_and_get_column(*column)) { + if (nullable_column->has_null()) { + return true; + } + return _requires_parent_null_map_for_alignment(nullable_column->get_nested_column_ptr(), + table_type); + } + if (const auto* array_type = typeid_cast(table_type.get())) { + const auto& array_column = assert_cast(*column); + return _requires_parent_null_map_for_alignment(array_column.get_data_ptr(), + array_type->get_nested_type()); + } + if (const auto* map_type = typeid_cast(table_type.get())) { + const auto& map_column = assert_cast(*column); + return _requires_parent_null_map_for_alignment(map_column.get_keys_ptr(), + map_type->get_key_type()) || + _requires_parent_null_map_for_alignment(map_column.get_values_ptr(), + map_type->get_value_type()); + } + if (const auto* struct_type = typeid_cast(table_type.get())) { + const auto& struct_column = assert_cast(*column); + DORIS_CHECK(struct_column.tuple_size() == struct_type->get_elements().size()); + for (size_t i = 0; i < struct_column.tuple_size(); ++i) { + if (_requires_parent_null_map_for_alignment(struct_column.get_column_ptr(i), + struct_type->get_element(i))) { + return true; + } + } + } + return false; + } + + static bool _requires_parent_null_map_for_alignment_at(const ColumnPtr& column, + const DataTypePtr& table_type, + const size_t row) { + DORIS_CHECK(column.get() != nullptr); + DORIS_CHECK(table_type != nullptr); + DORIS_CHECK(row < column->size()); + if (table_type->is_nullable()) { + const auto& nested_type = + assert_cast(*table_type).get_nested_type(); + if (const auto* nullable_column = check_and_get_column(*column)) { + // A nearer nullable wrapper already protects its descendants at this entry, so an + // inherited collection mask cannot be needed there. + if (nullable_column->is_null_at(row)) { + return false; + } + return _requires_parent_null_map_for_alignment_at( + nullable_column->get_nested_column_ptr(), nested_type, row); + } + return _requires_parent_null_map_for_alignment_at(column, nested_type, row); + } + if (const auto* nullable_column = check_and_get_column(*column)) { + if (nullable_column->is_null_at(row)) { + return true; + } + return _requires_parent_null_map_for_alignment_at( + nullable_column->get_nested_column_ptr(), table_type, row); + } + if (const auto* array_type = typeid_cast(table_type.get())) { + const auto& array_column = assert_cast(*column); + const auto& offsets = array_column.get_offsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_requires_parent_null_map_for_alignment_at(array_column.get_data_ptr(), + array_type->get_nested_type(), + child_row)) { + return true; + } + } + return false; + } + if (const auto* map_type = typeid_cast(table_type.get())) { + const auto& map_column = assert_cast(*column); + const auto& offsets = map_column.get_offsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_requires_parent_null_map_for_alignment_at( + map_column.get_keys_ptr(), map_type->get_key_type(), child_row) || + _requires_parent_null_map_for_alignment_at( + map_column.get_values_ptr(), map_type->get_value_type(), child_row)) { + return true; + } + } + return false; + } + if (const auto* struct_type = typeid_cast(table_type.get())) { + const auto& struct_column = assert_cast(*column); + DORIS_CHECK(struct_column.tuple_size() == struct_type->get_elements().size()); + for (size_t i = 0; i < struct_column.tuple_size(); ++i) { + if (_requires_parent_null_map_for_alignment_at(struct_column.get_column_ptr(i), + struct_type->get_element(i), row)) { + return true; + } + } + } + return false; + } + + static bool _requires_collection_parent_null_map(const NullMap* parent_null_map, + const ColumnPtr& column, + const DataTypePtr& table_type) { + // Descendant null maps can be entry-sized. Scan them only when an inherited mask can + // actually hide a row; absent/all-clear masks cannot authorize any physical child NULL. + if (parent_null_map == nullptr || + std::ranges::none_of(*parent_null_map, [](const auto value) { return value != 0; })) { + return false; + } + return _requires_parent_null_map_for_alignment(column, table_type); + } + template - static const NullMap* _project_collection_parent_null_map( - const NullMap* container_null_map, const NullMap* ancestor_null_map, const size_t rows, - const Offsets& offsets, const size_t child_rows, NullMap* const projected_null_map) { + static bool _parent_null_map_hides_collection_entries(const NullMap* container_null_map, + const NullMap* ancestor_null_map, + const size_t rows, + const Offsets& offsets) { if (container_null_map == nullptr && ancestor_null_map == nullptr) { - return nullptr; + return false; } DORIS_CHECK(container_null_map == nullptr || container_null_map->size() == rows); DORIS_CHECK(ancestor_null_map == nullptr || ancestor_null_map->size() == rows); DORIS_CHECK(offsets.size() == rows); - bool has_hidden_row = false; + size_t begin = 0; for (size_t row = 0; row < rows; ++row) { - if ((container_null_map != nullptr && (*container_null_map)[row]) || - (ancestor_null_map != nullptr && (*ancestor_null_map)[row])) { - has_hidden_row = true; - break; + const size_t end = offsets[row]; + const bool hidden = (container_null_map != nullptr && (*container_null_map)[row]) || + (ancestor_null_map != nullptr && (*ancestor_null_map)[row]); + // A hidden collection row protects descendants only when its offset span is nonempty. + if (hidden && end > begin) { + return true; + } + begin = end; + } + return false; + } + + template + static bool _requires_collection_parent_null_map(const NullMap* parent_null_map, + const ColumnPtr& column, + const DataTypePtr& table_type, + const size_t rows, const Offsets& offsets) { + if (parent_null_map == nullptr) { + return false; + } + DORIS_CHECK(parent_null_map->size() == rows); + DORIS_CHECK(offsets.size() == rows); + DORIS_CHECK(offsets.empty() || offsets.back() == column->size()); + size_t begin = 0; + for (size_t row = 0; row < rows; ++row) { + const size_t end = offsets[row]; + if ((*parent_null_map)[row]) { + // Only ancestor-hidden entries can consume this projection. Restricting the probe + // to their spans avoids scanning visible payload covered by nearer nullable masks. + for (size_t child_row = begin; child_row < end; ++child_row) { + if (_requires_parent_null_map_for_alignment_at(column, table_type, child_row)) { + return true; + } + } } + begin = end; } - if (!has_hidden_row) { + return false; + } + + template + static const NullMap* _project_collection_parent_null_map_for_hidden_entries( + const NullMap* container_null_map, const NullMap* ancestor_null_map, const size_t rows, + const Offsets& offsets, const size_t child_rows, NullMap* const projected_null_map) { + if (!_parent_null_map_hides_collection_entries(container_null_map, ancestor_null_map, rows, + offsets)) { // Nullable collection wrappers expose a null-map even when every row is present; avoid - // allocating entry-coordinate scratch proportional to a potentially huge collection. + // allocating entry-coordinate scratch unless a hidden row owns physical entries. return nullptr; } projected_null_map->resize(child_rows); @@ -1736,9 +1933,10 @@ class TableReader { // storage invariant, so add it only at the materialization boundary. element_mapping.table_type = make_nullable(element_mapping.table_type); NullMap descendant_parent_null_map; - const NullMap* descendant_parent_null_map_ptr = _project_collection_parent_null_map( - parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), - nested_column->size(), &descendant_parent_null_map); + const NullMap* descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + parent_null_map, nullable_parent_null_map, rows, file_array->get_offsets(), + nested_column->size(), &descendant_parent_null_map); RETURN_IF_ERROR(_materialize_present_child_mapping_column( element_mapping, nested_column, nested_column->size(), &nested_column, descendant_parent_null_map_ptr)); @@ -1790,9 +1988,10 @@ class TableReader { ColumnPtr value_column = file_map->get_values_ptr(); DORIS_CHECK(key_column->size() == value_column->size()); NullMap descendant_parent_null_map; - const NullMap* descendant_parent_null_map_ptr = _project_collection_parent_null_map( - parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), - key_column->size(), &descendant_parent_null_map); + const NullMap* descendant_parent_null_map_ptr = + _project_collection_parent_null_map_for_hidden_entries( + parent_null_map, nullable_parent_null_map, rows, file_map->get_offsets(), + key_column->size(), &descendant_parent_null_map); const ColumnMapping* key_mapping = nullptr; const ColumnMapping* value_mapping = nullptr; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 02a0744426bbe8..d959e27b15e907 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1064,7 +1064,9 @@ class TableReaderCastTestHelper final : public TableReader { using TableReader::_materialize_mapping_column; using TableReader::_materialize_present_child_mapping_column; using TableReader::_materialize_struct_mapping_column; - using TableReader::_project_collection_parent_null_map; + using TableReader::_project_collection_parent_null_map_for_hidden_entries; + using TableReader::_requires_collection_parent_null_map; + using TableReader::_requires_parent_null_map_for_alignment; }; TEST(TableReaderTest, TruncateCharOrVarcharPredicateOnlyAppliesToParquetStringWidthMismatch) { @@ -3149,6 +3151,76 @@ TEST(TableReaderTest, ScalarProjectionRejectsNullInRequiredTableColumn) { std::string::npos); } +TEST(TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenNearerMaskCoversVisibleNull) { + constexpr size_t visible_entries = 500000; + const size_t entries = visible_entries + 1; + const auto int_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_struct_type = make_nullable(struct_type); + + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto element_null_map = ColumnUInt8::create(entries, 0); + element_null_map->get_data().back() = 1; + ColumnPtr nullable_elements = ColumnNullable::create( + ColumnStruct::create(std::move(struct_children)), std::move(element_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, entries}; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_elements, nullable_struct_type, 2, offsets)); +} + +TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenNearerMaskCoversVisibleNull) { + constexpr size_t visible_entries = 500000; + const size_t entries = visible_entries + 1; + const auto int_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_struct_type = make_nullable(struct_type); + + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto value_struct_null_map = ColumnUInt8::create(entries, 0); + value_struct_null_map->get_data().back() = 1; + ColumnPtr nullable_value_structs = ColumnNullable::create( + ColumnStruct::create(std::move(struct_children)), std::move(value_struct_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, entries}; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_value_structs, nullable_struct_type, 2, offsets)); +} + +TEST(TableReaderTest, CollectionParentMaskKeepsRequiredNullInHiddenEntry) { + const auto int_type = std::make_shared(); + const auto struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_struct_type = make_nullable(struct_type); + + auto values = ColumnInt32::create(2, 0); + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 1}); + MutableColumns struct_children; + struct_children.push_back(ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto element_null_map = ColumnUInt8::create(); + element_null_map->get_data().assign({0, 1}); + ColumnPtr nullable_elements = ColumnNullable::create( + ColumnStruct::create(std::move(struct_children)), std::move(element_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {1, 2}; + + EXPECT_TRUE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_elements, nullable_struct_type, 2, offsets)); +} + TEST(TableReaderTest, ReopenSplitAfterClose) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_test"; std::filesystem::remove_all(test_dir); @@ -4924,6 +4996,112 @@ TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { } } +TEST(TableReaderTest, TrivialArrayChildProjectsNullableStructParentMask) { + const auto int_type = std::make_shared(); + const auto element_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"value"}); + const auto nullable_element_struct_type = make_nullable(element_struct_type); + const auto array_type = std::make_shared(nullable_element_struct_type); + const auto table_struct_type = make_nullable(std::make_shared( + DataTypes {array_type, int_type}, Strings {"items", "added"})); + const auto file_struct_type = make_nullable( + std::make_shared(DataTypes {array_type}, Strings {"items"})); + + auto table_items = make_table_column(0, "items", array_type); + table_items.type = array_type; + auto table_element = make_table_column(0, "element", element_struct_type); + table_element.type = element_struct_type; + auto table_value = make_table_column(0, "value", int_type); + table_value.type = int_type; + table_element.children = {table_value}; + table_items.children = {table_element}; + auto table_added = make_table_column(1, "added", int_type); + table_added.type = int_type; + auto table_struct = make_table_column(0, "payload", table_struct_type); + table_struct.type = table_struct_type; + table_struct.children = {table_items, table_added}; + + auto file_items = make_file_column(0, "items", array_type); + file_items.type = array_type; + auto file_element = make_file_column(0, "element", element_struct_type); + file_element.type = element_struct_type; + auto file_value = make_file_column(0, "value", int_type); + file_value.type = int_type; + file_element.children = {file_value}; + file_items.children = {file_element}; + auto file_struct = make_file_column(0, "payload", file_struct_type); + file_struct.type = file_struct_type; + file_struct.children = {file_items}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_FALSE(mapper.mappings()[0].is_trivial); + ASSERT_TRUE(mapper.mappings()[0].child_mappings[0].is_trivial); + + auto values = ColumnInt32::create(); + values->get_data().assign({0, 7}); + auto value_null_map = ColumnUInt8::create(); + value_null_map->get_data().assign({1, 0}); + MutableColumns element_children; + element_children.push_back( + ColumnNullable::create(std::move(values), std::move(value_null_map))); + auto element_null_map = ColumnUInt8::create(2, 0); + auto array_values = ColumnNullable::create(ColumnStruct::create(std::move(element_children)), + std::move(element_null_map)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().assign({1, 2}); + MutableColumns struct_children; + struct_children.push_back(ColumnArray::create(std::move(array_values), std::move(offsets))); + auto parent_null_map = ColumnUInt8::create(); + parent_null_map->get_data().assign({1, 0}); + ColumnPtr file_data = ColumnNullable::create(ColumnStruct::create(std::move(struct_children)), + std::move(parent_null_map)); + + TableReaderCastTestHelper reader; + ColumnPtr result; + const auto status = + reader._materialize_struct_mapping_column(mapper.mappings()[0], file_data, 2, &result); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& result_struct = assert_cast( + assert_cast(*result).get_nested_column()); + const auto& result_array = assert_cast(result_struct.get_column(0)); + const auto& result_elements = assert_cast(result_array.get_data()); + const auto& result_element_struct = + assert_cast(result_elements.get_nested_column()); + ASSERT_FALSE(result_element_struct.get_column(0).is_nullable()); + EXPECT_EQ(assert_cast(result_element_struct.get_column(0)).get_element(1), + 7); +} + +TEST(TableReaderTest, ParentMaskProjectionOnlyWhenRequiredDescendantCanConsumeIt) { + const auto int_type = std::make_shared(); + auto mutable_values = ColumnInt32::create(); + mutable_values->get_data().assign({1, 2}); + ColumnPtr values = std::move(mutable_values); + EXPECT_FALSE( + TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(values, int_type)); + + ColumnPtr nullable_values = ColumnNullable::create(values->clone(), ColumnUInt8::create(2, 0)); + EXPECT_FALSE(TableReaderCastTestHelper::_requires_parent_null_map_for_alignment(nullable_values, + int_type)); + auto null_map = ColumnUInt8::create(2, 0); + null_map->get_data()[0] = 1; + ColumnPtr nullable_values_with_null = + ColumnNullable::create(values->clone(), std::move(null_map)); + EXPECT_TRUE(TableReaderCastTestHelper::_requires_parent_null_map_for_alignment( + nullable_values_with_null, int_type)); + + NullMap all_clear_parent_mask(2, 0); + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &all_clear_parent_mask, nullable_values_with_null, int_type)); + NullMap hidden_parent_mask {1, 0}; + EXPECT_TRUE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &hidden_parent_mask, nullable_values_with_null, int_type)); + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + nullptr, nullable_values_with_null, int_type)); +} + TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) { const auto int_type = std::make_shared(); const std::vector projected_columns = { @@ -6452,9 +6630,52 @@ TEST(TableReaderTest, CollectionParentMaskFastPathSkipsEntryScratchForClearMasks ColumnArray::Offsets64 offsets {500000, 1000000}; NullMap projected_null_map; - const auto* result = TableReaderCastTestHelper::_project_collection_parent_null_map( - &container_null_map, &ancestor_null_map, 2, offsets, offsets.back(), - &projected_null_map); + const auto* result = + TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries( + &container_null_map, &ancestor_null_map, 2, offsets, offsets.back(), + &projected_null_map); + + EXPECT_EQ(nullptr, result); + EXPECT_TRUE(projected_null_map.empty()); +} + +TEST(TableReaderTest, CollectionParentMaskSkipsLargeArrayWhenOnlyEmptyRowIsHidden) { + constexpr size_t entries = 500000; + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + ColumnPtr nullable_values = + ColumnNullable::create(std::move(values), std::move(value_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {0, entries}; + NullMap projected_null_map; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_values, std::make_shared(), 2, offsets)); + const auto* result = + TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries( + nullptr, &parent_null_map, 2, offsets, entries, &projected_null_map); + + EXPECT_EQ(nullptr, result); + EXPECT_TRUE(projected_null_map.empty()); +} + +TEST(TableReaderTest, CollectionParentMaskSkipsLargeMapWhenOnlyEmptyRowIsHidden) { + constexpr size_t entries = 500000; + auto values = ColumnInt32::create(entries, 0); + auto value_null_map = ColumnUInt8::create(entries, 0); + value_null_map->get_data().back() = 1; + ColumnPtr nullable_values = + ColumnNullable::create(std::move(values), std::move(value_null_map)); + NullMap parent_null_map {1, 0}; + ColumnArray::Offsets64 offsets {0, entries}; + NullMap projected_null_map; + + EXPECT_FALSE(TableReaderCastTestHelper::_requires_collection_parent_null_map( + &parent_null_map, nullable_values, std::make_shared(), 2, offsets)); + const auto* result = + TableReaderCastTestHelper::_project_collection_parent_null_map_for_hidden_entries( + nullptr, &parent_null_map, 2, offsets, entries, &projected_null_map); EXPECT_EQ(nullptr, result); EXPECT_TRUE(projected_null_map.empty()); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index f3c16f10226f53..805bc7db19b78e 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -656,6 +656,10 @@ public String getDefaultValueSql() { } } + public String getConnectorDefaultValueSql() { + return connectorDefaultValueSql; + } + public void setConnectorDefaultValueSql(String connectorDefaultValueSql) { this.connectorDefaultValueSql = connectorDefaultValueSql; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/util/SqlUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/util/SqlUtils.java index ff867e529b784d..ffe92ffe830fc6 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/util/SqlUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/util/SqlUtils.java @@ -54,6 +54,12 @@ public static String escapeQuota(String str) { return str.replaceAll("\"", "\\\\\""); } + /** Quote a value as a SQL string literal under the requested backslash-escape mode. */ + public static String quoteStringLiteral(String value, boolean noBackslashEscapes) { + String escaped = noBackslashEscapes ? value : value.replace("\\", "\\\\"); + return "\"" + escaped.replace("\"", "\"\"") + "\""; + } + public static List splitMultiStmts(String sql) { DorisSqlSeparatorLexer lexer = new DorisSqlSeparatorLexer(CharStreams.fromString(sql)); CommonTokenStream tokenStream = new CommonTokenStream(lexer); diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java index 66742e5aced792..d648880e437d92 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java @@ -41,7 +41,6 @@ import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.connector.spi.ConnectorTableSchema; import org.apache.doris.connector.spi.ConnectorTableStatistics; -import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.ConnectorViewDefinition; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.ddl.BranchChange; @@ -133,11 +132,6 @@ public class HiveConnectorMetadata implements ConnectorMetadata { // (namespaced under __internal.) lives in ConnectorTableSchema. private static final String PARTITION_COLUMNS_PROPERTY = ConnectorTableSchema.PARTITION_COLUMNS_KEY; - // Connector-side spelling of fe-type ScalarType.MAX_VARCHAR_LENGTH (the connector must not import fe-type); - // a hive `string` partition column is widened to varchar(65533) for legacy parity. Paimon hardcodes the - // identical 65533. - private static final int MAX_VARCHAR_LENGTH = 65533; - // Hive-canonical partition text for a DATETIME/TIMESTAMP literal: space separator, full seconds. See // hiveDateTimeString / extractLiteralValue (H2: String.valueOf(LocalDateTime) would yield ISO "…T…" and drop // zero seconds, never matching the stored Hive partition value). @@ -499,8 +493,10 @@ public ConnectorTableSchema getTableSchema( String tableName = hiveHandle.getTableName(); HmsTableInfo tableInfo = hmsClient.getTable(dbName, tableName); - List columns = buildColumns(tableInfo); - List partitionKeys = coercePartitionKeyStringToVarchar(buildPartitionKeys(tableInfo)); + HiveWriteMetadataSnapshot writeMetadata = HiveWriteMetadataSnapshot.of( + tableInfo, getDefaultValues(tableInfo)); + List columns = writeMetadata.getDataColumns(); + List partitionKeys = writeMetadata.getPartitionColumns(); // Merge: regular columns + partition columns (partition columns last, mirroring legacy // HMSExternalTable full-schema order: data columns then partition keys). @@ -554,7 +550,7 @@ public ConnectorTableSchema getTableSchema( } return new ConnectorTableSchema(tableName, allColumns, formatType, tableProperties, - perTableCapabilities); + perTableCapabilities, writeMetadata.getIdentity()); } /** @@ -596,8 +592,10 @@ private ConnectorTableSchema reflectSiblingCapabilities(Connector owner, Connect return siblingSchema; } inherited.addAll(siblingSchema.getTableCapabilities()); + // Delegation may enrich capabilities, but it must preserve the sibling generation bound with the schema. return new ConnectorTableSchema(siblingSchema.getTableName(), siblingSchema.getColumns(), - siblingSchema.getTableFormatType(), siblingSchema.getProperties(), inherited); + siblingSchema.getTableFormatType(), siblingSchema.getProperties(), inherited, + siblingSchema.getWriteMetadataIdentity()); } // ========== ConnectorTableOps: Column Handles ========== @@ -2186,99 +2184,7 @@ private static String resolveTextCompressionDefault(ConnectorSession session) { // ========== Internal helpers ========== private List buildColumns(HmsTableInfo tableInfo) { - List spiColumns = tableInfo.getColumns(); - if (spiColumns == null || spiColumns.isEmpty()) { - return Collections.emptyList(); - } - // HmsTableInfo already returns ConnectorColumn with types mapped by HmsTypeMapping - // during ThriftHmsClient.getTable(). Enrich with default values if available. - List columns = spiColumns; - Map defaults = getDefaultValues(tableInfo); - if (!defaults.isEmpty()) { - List enriched = new ArrayList<>(spiColumns.size()); - for (ConnectorColumn col : spiColumns) { - String defaultVal = defaults.get(col.getName()); - if (defaultVal != null && col.getDefaultValue() == null) { - enriched.add(new ConnectorColumn( - col.getName(), col.getType(), col.getComment(), - col.isNullable(), defaultVal)); - } else { - enriched.add(col); - } - } - columns = enriched; - } - return coerceOpenCsvColumnsToString(tableInfo, columns); - } - - private List buildPartitionKeys(HmsTableInfo tableInfo) { - List partKeys = tableInfo.getPartitionKeys(); - if (partKeys == null) { - return Collections.emptyList(); - } - return partKeys; - } - - /** - * Widens a hive {@code string} partition column to {@code varchar(65533)}, replicating legacy - * {@code HMSExternalTable.initPartitionColumns}: a bare-string partition column is coerced to - * {@code varchar(ScalarType.MAX_VARCHAR_LENGTH)} "to be same as doris managed table", while every other - * declared type (int/date/timestamp/decimal/varchar(n)/char(n)/...) is kept exactly as - * {@code HmsTypeMapping} produced it. The gate is the mapped connector type name {@code STRING} (hive - * {@code string}, and {@code binary} when not mapped to varbinary, both land on it), matching legacy's - * {@code PrimitiveType.STRING} check. The widened column keeps the same name/comment/nullability/flags, so - * the full-schema entry and the partition-column view carry the identical type (legacy mutated one shared - * {@code Column} in place). - */ - private static List coercePartitionKeyStringToVarchar(List partitionKeys) { - if (partitionKeys.isEmpty()) { - return partitionKeys; - } - List coerced = new ArrayList<>(partitionKeys.size()); - for (ConnectorColumn col : partitionKeys) { - if ("STRING".equals(col.getType().getTypeName())) { - coerced.add(new ConnectorColumn(col.getName(), - ConnectorType.of("VARCHAR", MAX_VARCHAR_LENGTH, -1), - col.getComment(), col.isNullable(), col.getDefaultValue(), - col.isKey(), col.isAutoInc(), col.isAggregated())); - } else { - coerced.add(col); - } - } - return coerced; - } - - /** - * Flattens every DATA column of a hive {@code OpenCSVSerde} table to {@code STRING}, reproducing the - * schema legacy obtained through the metastore {@code get_schema} RPC. {@code OpenCSVSerde} - * ({@code org.apache.hadoop.hive.serde2.OpenCSVSerde}) reads a delimited file as PLAIN text: its - * deserializer's ObjectInspector reports every top-level column as {@code string}, so a declared - * {@code int}/{@code date}/{@code boolean} — and even an {@code array}/{@code map}/{@code struct} — is - * served verbatim as a string and never parsed. The SPI reads the RAW stored column types - * ({@code sd.getCols()}), which for OpenCSV disagree with what the reader actually returns; legacy's - * default {@code get_schema} path (server-side deserializer) collapsed them to all-string. We reproduce - * that RESULT connector-side, WITHOUT the extra per-table RPC, by forcing the whole column type to a flat - * {@code STRING} here. Partition keys are left untouched (hive appends them after the deserializer, so - * they keep their declared types — see {@link #coercePartitionKeyStringToVarchar}); a view is never an - * OpenCSV data table (guarded). Placing the rule in this hive metadata layer (not the shared hms - * {@code ThriftHmsClient}, which also feeds the hudi connector) keeps the serde-specific typing off the - * shared path and mirrors where Trino applies CSV=all-string. Non-OpenCSV tables return unchanged, so - * every other serde stays byte-identical to the raw {@code sd.getCols()} path. - */ - private static List coerceOpenCsvColumnsToString( - HmsTableInfo tableInfo, List columns) { - if (isView(tableInfo) - || !HiveTextProperties.HIVE_OPEN_CSV_SERDE.equals(tableInfo.getSerializationLib())) { - return columns; - } - ConnectorType stringType = ConnectorType.of("STRING"); - List forced = new ArrayList<>(columns.size()); - for (ConnectorColumn col : columns) { - forced.add(new ConnectorColumn(col.getName(), stringType, col.getComment(), - col.isNullable(), col.getDefaultValue(), - col.isKey(), col.isAutoInc(), col.isAggregated())); - } - return forced; + return HiveWriteMetadataSnapshot.of(tableInfo, getDefaultValues(tableInfo)).getDataColumns(); } private Map getDefaultValues(HmsTableInfo tableInfo) { diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java index 978144329f8bde..375604ecd4a9e1 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorTransaction.java @@ -23,16 +23,19 @@ import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsCommonStatistics; +import org.apache.doris.connector.hms.HmsNotificationEvent; import org.apache.doris.connector.hms.HmsPartitionInfo; import org.apache.doris.connector.hms.HmsPartitionStatistics; import org.apache.doris.connector.hms.HmsPartitionWithStatistics; import org.apache.doris.connector.hms.HmsTableInfo; import org.apache.doris.connector.hms.HmsTypeMapping; +import org.apache.doris.connector.hms.event.HmsEventParser; import org.apache.doris.connector.spi.ConnectorColumn; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.connector.spi.handle.ConnectorTransaction; import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileSystem; @@ -75,6 +78,8 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -105,6 +110,9 @@ public class HiveConnectorTransaction implements ConnectorTransaction { private static final Logger LOG = LogManager.getLogger(HiveConnectorTransaction.class); + private static final long COMMIT_LOCK_TIMEOUT_MS = 5000; + private static final long COMMIT_LOCK_HEARTBEAT_SECONDS = 10; + private static final int NOTIFICATION_BATCH_SIZE = 1000; private final long transactionId; private final HmsClient hmsClient; @@ -113,7 +121,7 @@ public class HiveConnectorTransaction implements ConnectorTransaction { // Plugin-owned async pool for staging renames + MPU complete/abort (the legacy class had this injected // by fe-core). Shut down in close(). authWrappingExecutor wraps each submitted task in the catalog auth // context (D5: plugin-owned pool threads do not inherit the caller pin). - private final ExecutorService fileSystemExecutor; + private final ScheduledExecutorService fileSystemExecutor; private final Executor authWrappingExecutor; // Captured at beginWrite (D6). Threaded to storage().getFileSystem(session) so the engine hands back the @@ -123,6 +131,8 @@ public class HiveConnectorTransaction implements ConnectorTransaction { private NameMapping nameMapping; private volatile HmsTableInfo hmsTableInfo; + private String writeMetadataIdentity; + private long writeMetadataEventId = -1; private String queryId; private boolean isOverwrite; private TFileType fileType; @@ -140,7 +150,8 @@ public HiveConnectorTransaction(long transactionId, HmsClient hmsClient, Connect this.transactionId = transactionId; this.hmsClient = hmsClient; this.context = context; - this.fileSystemExecutor = Executors.newFixedThreadPool(16, namedDaemonThreadFactory("hive-write-fs-%d")); + this.fileSystemExecutor = Executors.newScheduledThreadPool( + 16, namedDaemonThreadFactory("hive-write-fs-%d")); this.authWrappingExecutor = command -> fileSystemExecutor.execute(() -> { try { context.executeAuthenticated(() -> { @@ -199,6 +210,20 @@ public void close() { * only pre-commit point that has the table — so the full-ACID reject (D7) can run here. */ public void beginWrite(ConnectorSession session, String db, String tableName, HiveWriteContext ctx) { + HiveWriteMetadataSnapshot writeMetadata; + try { + writeMetadata = context.executeAuthenticated( + () -> HiveWriteMetadataSnapshot.loadFresh(hmsClient, db, tableName)); + } catch (Exception e) { + throw new DorisConnectorException( + "Failed to begin write for hive table " + tableName + ": " + e.getMessage(), e); + } + beginWrite(session, db, tableName, ctx, writeMetadata.getTable(), writeMetadata.getIdentity(), + writeMetadata.getNotificationEventId()); + } + + void beginWrite(ConnectorSession session, String db, String tableName, HiveWriteContext ctx, + HmsTableInfo table, String metadataIdentity, long metadataEventId) { this.session = session; this.queryId = ctx.getQueryId(); this.isOverwrite = ctx.isOverwrite(); @@ -207,12 +232,10 @@ public void beginWrite(ConnectorSession session, String db, String tableName, Hi ? Optional.empty() : Optional.of(ctx.getWritePath()); this.nameMapping = new NameMapping(context.getCatalogId(), db, tableName, db, tableName); try { - context.executeAuthenticated(() -> { - HmsTableInfo table = hmsClient.getTable(db, tableName); - rejectTransactionalWrite(table.getParameters()); - this.hmsTableInfo = table; - return null; - }); + rejectTransactionalWrite(table.getParameters()); + this.hmsTableInfo = table; + this.writeMetadataIdentity = metadataIdentity; + this.writeMetadataEventId = metadataEventId; } catch (Exception e) { throw new DorisConnectorException( "Failed to begin write for hive table " + tableName + ": " + e.getMessage(), e); @@ -221,10 +244,57 @@ public void beginWrite(ConnectorSession session, String db, String tableName, Hi @Override public void commit() { + long lockId; + try { + String user = session == null ? "doris" : session.getUser(); + lockId = context.executeAuthenticated(() -> hmsClient.acquireExclusiveTableLock( + queryId, user, nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName(), + COMMIT_LOCK_TIMEOUT_MS)); + } catch (Exception e) { + throw new DorisConnectorException("Failed to lock hive table generation before commit: " + + e.getMessage(), e); + } + ScheduledFuture heartbeat = startCommitLockHeartbeat(lockId); + try { + commitWhileTableLocked(); + } finally { + heartbeat.cancel(false); + try { + context.executeAuthenticated(() -> { + hmsClient.releaseLock(lockId); + return null; + }); + } catch (Exception e) { + // Publication may already have succeeded, so a lock-cleanup failure must not turn success into + // a retry that could duplicate data. HMS expires abandoned standalone locks as a fallback. + LOG.warn("Failed to release hive commit lock {} for {}", lockId, queryId, e); + } + } + } + + private ScheduledFuture startCommitLockHeartbeat(long lockId) { + // File moves may outlive the HMS lock timeout. Heartbeating on the transaction-owned daemon pool keeps + // the table-generation exclusion valid until every publication action has completed. + return fileSystemExecutor.scheduleAtFixedRate(() -> { + try { + context.executeAuthenticated(() -> { + hmsClient.heartbeatLock(lockId); + return null; + }); + } catch (Exception e) { + LOG.warn("Failed to heartbeat hive commit lock {} for {}; will retry", lockId, queryId, e); + } + }, COMMIT_LOCK_HEARTBEAT_SECONDS, COMMIT_LOCK_HEARTBEAT_SECONDS, TimeUnit.SECONDS); + } + + private void commitWhileTableLocked() { + validateWriteMetadataBeforePublication(); // The classification (finishInsertTable) ran from the executor in the legacy class; the unified SPI // exposes only commit(), so it runs here (before the committer) to populate the action maps. If it // throws, the committer was never created and the engine's subsequent rollback() cleans up. finishInsertTable(nameMapping); + // Classification can perform metastore reads, so close that interval before any file or HMS mutation. + validateWriteMetadataBeforePublication(); hmsCommitter = new HmsCommitter(); try { for (Map.Entry> entry : tableActions.entrySet()) { @@ -282,6 +352,90 @@ public void commit() { } } + private void validateWriteMetadataBeforePublication() { + HiveWriteMetadataSnapshot current; + try { + current = context.executeAuthenticated(() -> { + validateTableIncarnationEvents(); + return HiveWriteMetadataSnapshot.loadFresh( + hmsClient, nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + }); + } catch (Exception e) { + throw new DorisConnectorException("Failed to validate hive table generation before commit: " + + e.getMessage(), e); + } + // Files and HMS mutations must target the same incarnation and effective schema used by the BE writer. + if (!Objects.equals(writeMetadataIdentity, current.getIdentity())) { + throw new DorisConnectorException( + "Hive table metadata changed while the write was running; retry the statement"); + } + } + + private void validateTableIncarnationEvents() { + if (writeMetadataEventId < 0) { + return; + } + long endEventId = hmsClient.getCurrentNotificationEventId(); + long cursor = writeMetadataEventId; + while (cursor < endEventId) { + List events = hmsClient.getNextNotification(cursor, NOTIFICATION_BATCH_SIZE); + if (events.isEmpty()) { + throw new DorisConnectorException( + "Hive table metadata event history is incomplete; retry the statement"); + } + long nextCursor = cursor; + for (HmsNotificationEvent event : events) { + if (event.getEventId() <= cursor) { + continue; + } + if (event.getEventId() > endEventId) { + break; + } + nextCursor = Math.max(nextCursor, event.getEventId()); + if (isTargetTableGenerationEvent(event)) { + // HMS has no portable table UUID. The notification lineage is therefore the incarnation + // token that distinguishes an identical drop/recreate from the object bound by the writer. + throw new DorisConnectorException( + "Hive table metadata changed while the write was running; retry the statement"); + } + } + if (nextCursor <= cursor) { + throw new DorisConnectorException( + "Hive table metadata event history did not advance; retry the statement"); + } + cursor = nextCursor; + } + } + + private boolean isTargetTableGenerationEvent(HmsNotificationEvent event) { + String type = event.getEventType(); + if (!("CREATE_TABLE".equalsIgnoreCase(type) + || "DROP_TABLE".equalsIgnoreCase(type) + || "ALTER_TABLE".equalsIgnoreCase(type))) { + return false; + } + if (isTargetTable(event.getDbName(), event.getTableName())) { + return true; + } + if (!"ALTER_TABLE".equalsIgnoreCase(type)) { + return false; + } + // ALTER_TABLE identifies a rename by its old name in the event envelope. Inspect the parsed after-name + // too, or a same-shaped table renamed onto this target could evade the incarnation fence. + for (MetastoreChangeDescriptor change : HmsEventParser.parse(event)) { + if (isTargetTable(change.getDbName(), change.getTableName()) + || isTargetTable(change.getDbNameAfter(), change.getTableNameAfter())) { + return true; + } + } + return false; + } + + private boolean isTargetTable(String dbName, String tableName) { + return nameMapping.getRemoteDbName().equalsIgnoreCase(dbName) + && nameMapping.getRemoteTblName().equalsIgnoreCase(tableName); + } + @Override public void rollback() { if (hmsCommitter == null) { diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteMetadataSnapshot.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteMetadataSnapshot.java new file mode 100644 index 00000000000000..ef1eac66bb6867 --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWriteMetadataSnapshot.java @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.hms.HmsClient; +import org.apache.doris.connector.hms.HmsClientException; +import org.apache.doris.connector.hms.HmsTableInfo; +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorType; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Formatter; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +/** + * One effective Hive write generation: the raw HMS object plus the exact columns exposed to binding. + */ +final class HiveWriteMetadataSnapshot { + + private static final int MAX_VARCHAR_LENGTH = 65533; + + private final HmsTableInfo table; + private final List dataColumns; + private final List partitionColumns; + private final String identity; + private final long notificationEventId; + + private HiveWriteMetadataSnapshot(HmsTableInfo table, Map defaults, + long notificationEventId) { + this.table = table; + this.dataColumns = Collections.unmodifiableList(buildDataColumns(table, defaults)); + this.partitionColumns = Collections.unmodifiableList(buildPartitionColumns(table)); + this.identity = buildIdentity(); + this.notificationEventId = notificationEventId; + } + + static HiveWriteMetadataSnapshot of(HmsTableInfo table, Map defaults) { + return new HiveWriteMetadataSnapshot(table, + defaults == null ? Collections.emptyMap() : defaults, -1); + } + + static HiveWriteMetadataSnapshot loadFresh(HmsClient client, String dbName, String tableName) { + long notificationEventId; + try { + // Capture the watermark before loading the table: a concurrent DDL after this point is then visible + // to commit-time event validation, including a same-shaped drop/recreate that has no HMS UUID. + notificationEventId = client.getCurrentNotificationEventId(); + } catch (UnsupportedOperationException e) { + notificationEventId = -1; + } + HmsTableInfo table = client.getTableFresh(dbName, tableName); + Map defaults; + try { + defaults = client.getDefaultColumnValues(dbName, tableName); + } catch (HmsClientException e) { + // Some Hive versions cannot expose default constraints; match binding's empty-default fallback. + defaults = Collections.emptyMap(); + } + return new HiveWriteMetadataSnapshot(table, defaults, notificationEventId); + } + + HmsTableInfo getTable() { + return table; + } + + List getDataColumns() { + return dataColumns; + } + + List getPartitionColumns() { + return partitionColumns; + } + + String getIdentity() { + return identity; + } + + long getNotificationEventId() { + return notificationEventId; + } + + private static List buildDataColumns( + HmsTableInfo table, Map defaults) { + List raw = table.getColumns(); + if (raw == null || raw.isEmpty()) { + return Collections.emptyList(); + } + boolean openCsv = !isView(table) + && HiveTextProperties.HIVE_OPEN_CSV_SERDE.equals(table.getSerializationLib()); + ConnectorType stringType = ConnectorType.of("STRING"); + List effective = new ArrayList<>(raw.size()); + for (ConnectorColumn column : raw) { + String defaultValue = column.getDefaultValue(); + if (defaultValue == null) { + defaultValue = defaults.get(column.getName()); + } + ConnectorType type = openCsv ? stringType : column.getType(); + if (type == column.getType() && Objects.equals(defaultValue, column.getDefaultValue())) { + effective.add(column); + } else { + effective.add(copyColumn(column, type, defaultValue)); + } + } + return effective; + } + + private static List buildPartitionColumns(HmsTableInfo table) { + List raw = table.getPartitionKeys(); + if (raw == null || raw.isEmpty()) { + return Collections.emptyList(); + } + List effective = new ArrayList<>(raw.size()); + for (ConnectorColumn column : raw) { + if ("STRING".equals(column.getType().getTypeName())) { + effective.add(copyColumn(column, + ConnectorType.of("VARCHAR", MAX_VARCHAR_LENGTH, -1), column.getDefaultValue())); + } else { + effective.add(column); + } + } + return effective; + } + + private static ConnectorColumn copyColumn( + ConnectorColumn column, ConnectorType type, String defaultValue) { + return new ConnectorColumn(column.getName(), type, column.getComment(), column.isNullable(), + defaultValue, column.isKey(), column.isAutoInc(), column.isAggregated()); + } + + private static boolean isView(HmsTableInfo table) { + return table.getViewOriginalText() != null || table.getViewExpandedText() != null + || "VIRTUAL_VIEW".equalsIgnoreCase(table.getTableType()); + } + + private String buildIdentity() { + MetadataDigest digest = new MetadataDigest(); + digest.add("hive-write-metadata-v2"); + digest.add(table.getDbName()); + digest.add(table.getTableName()); + // HMS has no portable UUID; createTime plus owner is its stable incarnation signal across alterations. + digest.add(table.getCreateTime()); + digest.add(table.getOwner()); + digest.add(table.getTableType()); + digest.add(table.getLocation()); + digest.add(table.getInputFormat()); + digest.add(table.getOutputFormat()); + digest.add(table.getSerializationLib()); + addMap(digest, table.getParameters()); + addMap(digest, table.getSdParameters()); + addStrings(digest, table.getBucketCols()); + digest.add(table.getNumBuckets()); + addColumns(digest, "data", dataColumns); + addColumns(digest, "partition", partitionColumns); + return digest.finish(); + } + + private static void addColumns( + MetadataDigest digest, String role, List columns) { + digest.add(role); + digest.add(columns.size()); + for (ConnectorColumn column : columns) { + digest.add(column.getName().toLowerCase(Locale.ROOT)); + digest.add(column.isNullable()); + digest.add(column.getDefaultValue()); + addType(digest, column.getType()); + } + } + + private static void addType(MetadataDigest digest, ConnectorType type) { + digest.add(type.getTypeName().toUpperCase(Locale.ROOT)); + digest.add(type.getPrecision()); + digest.add(type.getScale()); + List children = type.getChildren(); + digest.add(children.size()); + for (int i = 0; i < children.size(); i++) { + digest.add(i < type.getFieldNames().size() + ? type.getFieldNames().get(i).toLowerCase(Locale.ROOT) : null); + digest.add(type.isChildNullable(i)); + addType(digest, children.get(i)); + } + } + + private static void addMap(MetadataDigest digest, Map values) { + Map ordered = values == null + ? Collections.emptyMap() : new TreeMap<>(values); + digest.add(ordered.size()); + for (Map.Entry entry : ordered.entrySet()) { + digest.add(entry.getKey()); + digest.add(entry.getValue()); + } + } + + private static void addStrings(MetadataDigest digest, List values) { + List safe = values == null ? Collections.emptyList() : values; + digest.add(safe.size()); + for (String value : safe) { + digest.add(value); + } + } + + private static final class MetadataDigest { + private final MessageDigest digest; + + private MetadataDigest() { + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private void add(Object value) { + if (value == null) { + digest.update((byte) 0); + return; + } + byte[] bytes = String.valueOf(value).getBytes(StandardCharsets.UTF_8); + digest.update((byte) 1); + digest.update((byte) (bytes.length >>> 24)); + digest.update((byte) (bytes.length >>> 16)); + digest.update((byte) (bytes.length >>> 8)); + digest.update((byte) bytes.length); + digest.update(bytes); + } + + private String finish() { + try (Formatter formatter = new Formatter(Locale.ROOT)) { + for (byte value : digest.digest()) { + formatter.format("%02x", value); + } + return formatter.toString(); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java index 703d3e47479b50..862a7ad049062a 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveWritePlanProvider.java @@ -26,6 +26,7 @@ import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorStorageContext; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.handle.ConnectorTransaction; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.handle.WriteOperation; @@ -62,8 +63,8 @@ * *

Builds the opaque {@link THiveTableSink} for a bound write and binds the write to the current * {@link HiveConnectorTransaction}: it loads the table under the catalog auth context, opens the write via - * {@link HiveConnectorTransaction#beginWrite} (which re-loads the table and applies the begin-guards, incl. - * the transactional-table reject), then assembles the sink Thrift from the loaded table. The Thrift is + * {@link HiveConnectorTransaction#beginWrite} (which applies the begin-guards, incl. the transactional-table + * reject), then assembles the sink Thrift from the same loaded table. The Thrift is * byte-identical to the legacy fe-core {@code planner.HiveTableSink.bindDataSink} (zero BE change), so the * BE writer is unaffected by the migration.

* @@ -83,6 +84,7 @@ public class HiveWritePlanProvider implements ConnectorWritePlanProvider { // DEFAULT_STAGING_BASE_DIR — connectors must not import fe-core). private static final String HIVE_STAGING_DIR = "hive.staging_dir"; private static final String DEFAULT_STAGING_BASE_DIR = "/tmp/.doris_staging"; + private static final String WRITE_METADATA_SCOPE_PREFIX = "hive-write-metadata:"; private final HmsClient hmsClient; private final Map properties; @@ -99,12 +101,14 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl HiveTableHandle tableHandle = (HiveTableHandle) handle.getTableHandle(); HiveConnectorTransaction transaction = currentTransaction(session); - // Load the table under the catalog auth context; it drives both the location resolution - // (buildWriteContext) and the sink assembly (buildSink). beginWrite re-loads it for its own - // begin-guard — the double-load is accepted (mirrors iceberg), keeping the flow simple. - HmsTableInfo table = loadTable(tableHandle); + // One fresh table generation drives validation, transaction state, location resolution, and sink + // assembly. Reloading in beginWrite would reopen a TOCTOU window after the schema fence. + HiveWriteMetadataSnapshot writeMetadata = loadWriteMetadata(session, tableHandle); + HmsTableInfo table = writeMetadata.getTable(); + validateBoundWriteMetadata(writeMetadata, handle); HiveWriteContext writeContext = buildWriteContext(session, tableHandle, table, handle); - transaction.beginWrite(session, tableHandle.getDbName(), tableHandle.getTableName(), writeContext); + transaction.beginWrite(session, tableHandle.getDbName(), tableHandle.getTableName(), writeContext, + table, writeMetadata.getIdentity(), writeMetadata.getNotificationEventId()); THiveTableSink sink = buildSink(session, tableHandle, table, handle, writeContext); TDataSink dataSink = new TDataSink(TDataSinkType.HIVE_TABLE_SINK); @@ -112,6 +116,61 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl return new ConnectorSinkPlan(dataSink); } + @Override + public Optional> getWriteColumns(ConnectorSession session, + ConnectorTableHandle handle, Optional branchName) { + HiveWriteMetadataSnapshot snapshot = loadWriteMetadata(session, (HiveTableHandle) handle); + List columns = new ArrayList<>( + snapshot.getDataColumns().size() + snapshot.getPartitionColumns().size()); + columns.addAll(snapshot.getDataColumns()); + columns.addAll(snapshot.getPartitionColumns()); + return Optional.of(Collections.unmodifiableList(columns)); + } + + @Override + public String getWriteMetadataIdentity(ConnectorSession session, ConnectorTableHandle handle) { + return loadWriteMetadata(session, (HiveTableHandle) handle).getIdentity(); + } + + private static void validateBoundWriteMetadata( + HiveWriteMetadataSnapshot writeMetadata, ConnectorWriteHandle handle) { + String boundIdentity = handle.getBoundWriteMetadataIdentity(); + if (boundIdentity != null && !boundIdentity.equals(writeMetadata.getIdentity())) { + // Bound expressions and live THiveTableSink ordinals must describe one HMS schema generation; + // accepting a reorder here silently writes each value under another column name. + throw new DorisConnectorException( + "Hive write metadata changed after the write was bound; retry the statement"); + } + + List boundColumns = handle.getBoundTargetColumns(); + List dataColumns = writeMetadata.getDataColumns(); + List partitionColumns = writeMetadata.getPartitionColumns(); + int liveColumnCount = dataColumns.size() + partitionColumns.size(); + if (boundColumns.isEmpty()) { + return; + } + if (boundColumns.size() != liveColumnCount) { + throw schemaChangedException(); + } + for (int i = 0; i < boundColumns.size(); i++) { + ConnectorColumn live = i < dataColumns.size() + ? dataColumns.get(i) + : partitionColumns.get(i - dataColumns.size()); + if (!boundColumns.get(i).getName().equalsIgnoreCase(live.getName())) { + throw schemaChangedException(); + } + } + } + + private static DorisConnectorException schemaChangedException() { + return new DorisConnectorException( + "Hive table schema changed after the write was bound; retry the statement"); + } + + static String writeMetadataIdentity(HmsTableInfo table) { + return HiveWriteMetadataSnapshot.of(table, Collections.emptyMap()).getIdentity(); + } + @Override public void appendExplainInfo(StringBuilder output, String prefix, ConnectorSession session, ConnectorWriteHandle handle) { @@ -363,10 +422,18 @@ private List resolveBrokerAddresses() { return result; } - private HmsTableInfo loadTable(HiveTableHandle tableHandle) { + private HiveWriteMetadataSnapshot loadWriteMetadata( + ConnectorSession session, HiveTableHandle tableHandle) { + String scopeKey = WRITE_METADATA_SCOPE_PREFIX + session.getCatalogId() + ":" + + tableHandle.getDbName() + "." + tableHandle.getTableName(); + return session.getStatementScope().computeIfAbsent(scopeKey, + () -> loadWriteMetadataFresh(tableHandle)); + } + + private HiveWriteMetadataSnapshot loadWriteMetadataFresh(HiveTableHandle tableHandle) { try { - return context.executeAuthenticated( - () -> hmsClient.getTable(tableHandle.getDbName(), tableHandle.getTableName())); + return context.executeAuthenticated(() -> HiveWriteMetadataSnapshot.loadFresh( + hmsClient, tableHandle.getDbName(), tableHandle.getTableName())); } catch (Exception e) { throw new DorisConnectorException("Failed to load hive table " + tableHandle.getDbName() + "." + tableHandle.getTableName() + ": " + e.getMessage(), e); diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSchemaTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSchemaTest.java index ffcab1ff89cae1..a02c74da3fd4ed 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSchemaTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSchemaTest.java @@ -195,6 +195,24 @@ public void testPartitionColumnsComeAfterDataColumns() { Assertions.assertEquals(Arrays.asList("id", "name", "year", "region"), names); } + @Test + public void testWriteMetadataIdentityPreservesDataAndPartitionRoles() { + ConnectorTableSchema partitioned = schemaOf(HmsTableInfo.builder() + .dbName("db").tableName("t").inputFormat(PARQUET_INPUT_FORMAT) + .columns(Collections.singletonList(col("a", "INT"))) + .partitionKeys(Collections.singletonList(col("b", "INT"))) + .parameters(Collections.emptyMap()).build()); + ConnectorTableSchema unpartitioned = schemaOf(HmsTableInfo.builder() + .dbName("db").tableName("t").inputFormat(PARQUET_INPUT_FORMAT) + .columns(Arrays.asList(col("a", "INT"), col("b", "INT"))) + .partitionKeys(Collections.emptyList()) + .parameters(Collections.emptyMap()).build()); + + Assertions.assertNotNull(partitioned.getWriteMetadataIdentity()); + Assertions.assertNotEquals(partitioned.getWriteMetadataIdentity(), unpartitioned.getWriteMetadataIdentity(), + "the write fence must distinguish equal flattened columns with different partition roles"); + } + @Test public void testUnpartitionedTableEmitsNoPartitionColumnsProperty() { HmsTableInfo tableInfo = HmsTableInfo.builder() diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java index 7b0ff59901c52d..774ff3df0f25f4 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java @@ -392,6 +392,8 @@ public void foreignHandleSchemaReflectsSiblingScanCapabilitiesAsPerTableMarker() "Top-N lazy must survive the delegation as a per-table capability"); Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE), "nested-column prune must survive the delegation as a per-table capability"); + Assertions.assertEquals("sibling-generation", schema.getWriteMetadataIdentity(), + "capability reflection must not discard the sibling's write-generation fence"); } @Test @@ -615,14 +617,16 @@ private static final class RecordingSiblingMetadata implements ConnectorMetadata @Override public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle) { calls.add("getTableSchema"); - return new ConnectorTableSchema("sibling", Collections.emptyList(), "iceberg", Collections.emptyMap()); + return new ConnectorTableSchema("sibling", Collections.emptyList(), "iceberg", Collections.emptyMap(), + Collections.emptySet(), "sibling-generation"); } @Override public ConnectorTableSchema getTableSchema(ConnectorSession session, ConnectorTableHandle handle, ConnectorMvccSnapshot snapshot) { calls.add("getTableSchemaAtSnapshot"); - return new ConnectorTableSchema("sibling", Collections.emptyList(), "iceberg", Collections.emptyMap()); + return new ConnectorTableSchema("sibling", Collections.emptyList(), "iceberg", Collections.emptyMap(), + Collections.emptySet(), "sibling-generation"); } @Override diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java index 8e8436e2ec4ca8..5b5ded04976ddb 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorTransactionTest.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; +import org.apache.doris.connector.hms.HmsNotificationEvent; import org.apache.doris.connector.hms.HmsPartitionInfo; import org.apache.doris.connector.hms.HmsPartitionStatistics; import org.apache.doris.connector.hms.HmsPartitionWithStatistics; @@ -117,13 +118,19 @@ private static ConnectorColumn col(String name, String type) { } private static HmsTableInfo table(boolean partitioned, Map params) { + return table(partitioned, params, 10, Collections.singletonList(col("c1", "int"))); + } + + private static HmsTableInfo table(boolean partitioned, Map params, + int createTime, List columns) { return HmsTableInfo.builder() .dbName(DB).tableName(TBL).tableType("MANAGED_TABLE") + .owner("owner").createTime(createTime) .location("s3://bucket/db/t") .inputFormat("org.apache.hadoop.mapred.TextInputFormat") .outputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat") .serializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") - .columns(Collections.singletonList(col("c1", "int"))) + .columns(columns) .partitionKeys(partitioned ? Collections.singletonList(col("dt", "string")) : Collections.emptyList()) @@ -255,6 +262,72 @@ public void testBeginWriteAllowsPlainTable() { txn.beginWrite(null, DB, TBL, ctx(false)); // must not throw } + @Test + public void testCommitRejectsSameShapeTableReplacementBeforePublishing() throws TException { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table(false, Collections.emptyMap(), 10, + Collections.singletonList(col("c1", "int"))); + client.currentNotificationEventId = 10; + HiveConnectorTransaction txn = newTxn(client); + txn.beginWrite(null, DB, TBL, ctx(false)); + txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", + Collections.singletonList("f1"), 100, 4))); + + // Recreate the table with every digest-visible field unchanged. Only the HMS event stream can + // distinguish this new object from the table whose schema and location were bound above. + client.table = table(false, Collections.emptyMap(), 10, + Collections.singletonList(col("c1", "int"))); + client.notificationEvents = Arrays.asList( + event(11, "DROP_TABLE"), event(12, "CREATE_TABLE")); + client.currentNotificationEventId = 12; + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, txn::commit); + Assertions.assertTrue(ex.getMessage().contains("metadata changed")); + Assertions.assertFalse(client.calls.contains("updateTableStatistics:" + DB + "." + TBL), + "a replacement must be detected before files or HMS statistics are published"); + } + + @Test + public void testCommitHoldsExclusiveTableLockThroughPublication() throws TException { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table(false, Collections.emptyMap()); + client.currentNotificationEventId = 10; + HiveConnectorTransaction txn = newTxn(client); + txn.beginWrite(null, DB, TBL, ctx(false)); + txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", + Collections.singletonList("f1"), 100, 4))); + + txn.commit(); + + int acquired = client.calls.indexOf("acquireExclusiveTableLock:" + DB + "." + TBL); + int published = client.calls.indexOf("updateTableStatistics:" + DB + "." + TBL); + int released = client.calls.indexOf("releaseLock:77"); + Assertions.assertTrue(acquired >= 0 && acquired < published && published < released, + "the commit fence must cover every file/HMS publication action; calls=" + client.calls); + } + + private static HmsNotificationEvent event(long id, String type) { + return new HmsNotificationEvent(id, type, DB, TBL, "{}", "json-0.2", 0); + } + + @Test + public void testCommitRejectsPostPlanSchemaDdlBeforePublishing() throws TException { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table(false, Collections.emptyMap(), 10, + Arrays.asList(col("a", "int"), col("b", "int"))); + HiveConnectorTransaction txn = newTxn(client); + txn.beginWrite(null, DB, TBL, ctx(false)); + txn.addCommitData(serialize(pu("", TUpdateMode.APPEND, "s3://bucket/db/t", + Collections.singletonList("f1"), 100, 4))); + + client.table = table(false, Collections.emptyMap(), 10, + Arrays.asList(col("b", "int"), col("a", "int"))); + + Assertions.assertThrows(DorisConnectorException.class, txn::commit, + "a schema reorder after sink planning must abort before positional rows are published"); + Assertions.assertFalse(client.calls.contains("updateTableStatistics:" + DB + "." + TBL)); + } + // ─────────────────────────────── classification / NEW→APPEND downgrade ─────────────────────────────── @Test @@ -467,6 +540,8 @@ private static final class RecordingHmsClient implements HmsClient { private HmsTableInfo table; private boolean partitionExistsResult; private List partitions = Collections.emptyList(); + private long currentNotificationEventId; + private List notificationEvents = Collections.emptyList(); @Override public List listDatabases() { @@ -549,6 +624,33 @@ public boolean dropPartition(String dbName, String tableName, List parti return true; } + @Override + public long getCurrentNotificationEventId() { + calls.add("getCurrentNotificationEventId:" + currentNotificationEventId); + return currentNotificationEventId; + } + + @Override + public List getNextNotification(long lastEventId, int maxEvents) { + calls.add("getNextNotification:" + lastEventId); + return notificationEvents.stream() + .filter(event -> event.getEventId() > lastEventId) + .limit(maxEvents) + .collect(Collectors.toList()); + } + + @Override + public long acquireExclusiveTableLock(String queryId, String user, String dbName, + String tableName, long timeoutMs) { + calls.add("acquireExclusiveTableLock:" + dbName + "." + tableName); + return 77; + } + + @Override + public void releaseLock(long lockId) { + calls.add("releaseLock:" + lockId); + } + @Override public void close() { } diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java index fcb889c6b2df10..e27b2e74d919f6 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveWritePlanProviderTest.java @@ -24,6 +24,7 @@ import org.apache.doris.connector.spi.ConnectorBrokerAddress; import org.apache.doris.connector.spi.ConnectorColumn; import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; @@ -57,6 +58,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; /** * Pins {@link HiveWritePlanProvider#planWrite} for INSERT / INSERT OVERWRITE against legacy @@ -131,6 +133,32 @@ private static WriteHandle handle() { // ───────────────────────────── columns ───────────────────────────── + @Test + public void writeBindingPinsColumnsAndIdentityFromOneFreshGeneration() { + RecordingHmsClient client = new RecordingHmsClient(); + HmsTableInfo boundTable = tableBuilder() + .columns(Arrays.asList(col("a", "int"), col("b", "string"))) + .build(); + client.table = boundTable; + HiveWritePlanProvider provider = providerFor(client, new RecordingConnectorContext()); + WriteSession session = new WriteSession(null, Collections.emptyMap(), new TestStatementScope()); + + List columns = provider.getWriteColumns( + session, new HiveTableHandle(DB, TBL, HiveTableType.HIVE), Optional.empty()).orElseThrow(); + String identity = provider.getWriteMetadataIdentity( + session, new HiveTableHandle(DB, TBL, HiveTableType.HIVE)); + client.table = tableBuilder() + .columns(Arrays.asList(col("b", "string"), col("a", "int"))) + .build(); + + Assertions.assertEquals(Arrays.asList("a", "b"), + columns.stream().map(ConnectorColumn::getName).collect(java.util.stream.Collectors.toList())); + Assertions.assertEquals(HiveWriteMetadataSnapshot.of(boundTable, Collections.emptyMap()).getIdentity(), + identity); + Assertions.assertEquals(1, Collections.frequency(client.calls, "getTableFresh:" + DB + "." + TBL), + "columns and identity must share the same statement-pinned HMS generation"); + } + @Test public void planWriteEmitsRegularColumnsThenPartitionKeys() { // BE writes the data columns then maps the trailing partition keys; a wrong tag or order would write @@ -154,6 +182,135 @@ public void planWriteEmitsRegularColumnsThenPartitionKeys() { "partition keys must be tagged PARTITION_KEY and appended after the data columns"); } + @Test + public void planWriteRejectsSchemaReorderAfterBinding() { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = tableBuilder() + .columns(Arrays.asList(col("b", "int"), col("a", "int"))) + .build(); + WriteHandle boundHandle = handle() + .boundTargetColumns(Arrays.asList(col("a", "int"), col("b", "int"))); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(client, new RecordingConnectorContext(), boundHandle)); + + Assertions.assertTrue(ex.getMessage().contains("schema changed after the write was bound"), + "a live HMS reorder must fail before BE can index bound expressions by stale ordinals"); + } + + @Test + public void planWriteRejectsDataAndPartitionRoleChangeAfterBinding() { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = tableBuilder() + .columns(Arrays.asList(col("a", "int"), col("b", "int"))) + .partitionKeys(Collections.emptyList()) + .build(); + WriteHandle boundHandle = handle() + .boundWriteMetadataIdentity("bound-partitioned-schema"); + + Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(client, new RecordingConnectorContext(), boundHandle), + "the same flattened columns are unsafe when their data/partition roles changed"); + } + + @Test + public void planWriteUsesOneFreshTableGenerationForTransactionAndSink() { + RecordingHmsClient client = new RecordingHmsClient(); + client.table = tableBuilder().build(); + + planSink(client, new RecordingConnectorContext(), handle()); + + Assertions.assertEquals(1, Collections.frequency(client.calls, "getTableFresh:" + DB + "." + TBL), + "reloading after validation would let transaction state and sink columns straddle HMS schemas"); + Assertions.assertFalse(client.calls.contains("getTable:" + DB + "." + TBL), + "write planning must bypass a stale connector-side table cache"); + } + + @Test + public void writeMetadataIdentityIncludesRecursiveTypeShape() { + HmsTableInfo ints = tableBuilder() + .columns(Collections.singletonList(new ConnectorColumn("items", + ConnectorType.arrayOf(ConnectorType.of("INT")), "", true, null))) + .build(); + HmsTableInfo strings = tableBuilder() + .columns(Collections.singletonList(new ConnectorColumn("items", + ConnectorType.arrayOf(ConnectorType.of("STRING")), "", true, null))) + .build(); + + Assertions.assertNotEquals(HiveWritePlanProvider.writeMetadataIdentity(ints), + HiveWritePlanProvider.writeMetadataIdentity(strings), + "a nested leaf change must invalidate the bound physical layout"); + } + + @Test + public void writeMetadataIdentityIncludesEffectiveDefaults() { + ConnectorColumn oldDefault = new ConnectorColumn("c1", ConnectorType.of("INT"), "", true, "42"); + ConnectorColumn newDefault = new ConnectorColumn("c1", ConnectorType.of("INT"), "", true, "7"); + HmsTableInfo before = tableBuilder().columns(Collections.singletonList(oldDefault)).build(); + HmsTableInfo after = tableBuilder().columns(Collections.singletonList(newDefault)).build(); + + Assertions.assertNotEquals(HiveWritePlanProvider.writeMetadataIdentity(before), + HiveWritePlanProvider.writeMetadataIdentity(after), + "a default-only DDL changes values already materialized by binding"); + } + + @Test + public void writeMetadataIdentityIncludesSerdeThatShapesEffectiveColumns() { + HmsTableInfo csv = tableBuilder() + .serializationLib(HiveTextProperties.HIVE_OPEN_CSV_SERDE) + .build(); + HmsTableInfo lazy = tableBuilder().serializationLib(LAZY_SIMPLE_SERDE).build(); + + Assertions.assertNotEquals(HiveWritePlanProvider.writeMetadataIdentity(csv), + HiveWritePlanProvider.writeMetadataIdentity(lazy), + "OpenCSV and LazySimple bind different effective types for the same raw HMS columns"); + } + + @Test + public void planWriteRejectsNestedLeafChangeAfterBinding() { + HmsTableInfo before = tableBuilder() + .columns(Collections.singletonList(new ConnectorColumn("items", + ConnectorType.arrayOf(ConnectorType.of("INT")), "", true, null))) + .build(); + RecordingHmsClient client = new RecordingHmsClient(); + client.table = tableBuilder() + .columns(Collections.singletonList(new ConnectorColumn("items", + ConnectorType.arrayOf(ConnectorType.of("STRING")), "", true, null))) + .build(); + WriteHandle bound = handle().boundWriteMetadataIdentity( + HiveWriteMetadataSnapshot.of(before, Collections.emptyMap()).getIdentity()); + + Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(client, new RecordingConnectorContext(), bound)); + } + + @Test + public void planWriteRejectsOpenCsvToLazySimpleChangeAfterBinding() { + HmsTableInfo before = tableBuilder() + .serializationLib(HiveTextProperties.HIVE_OPEN_CSV_SERDE) + .build(); + RecordingHmsClient client = new RecordingHmsClient(); + client.table = tableBuilder().serializationLib(LAZY_SIMPLE_SERDE).build(); + WriteHandle bound = handle().boundWriteMetadataIdentity( + HiveWriteMetadataSnapshot.of(before, Collections.emptyMap()).getIdentity()); + + Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(client, new RecordingConnectorContext(), bound)); + } + + @Test + public void planWriteRejectsDefaultOnlyChangeAfterBinding() { + HmsTableInfo table = tableBuilder().build(); + RecordingHmsClient client = new RecordingHmsClient(); + client.table = table; + client.defaultValues = Collections.singletonMap("c1", "7"); + WriteHandle bound = handle().boundWriteMetadataIdentity( + HiveWriteMetadataSnapshot.of(table, Collections.singletonMap("c1", "42")).getIdentity()); + + Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(client, new RecordingConnectorContext(), bound)); + } + // ───────────────────────────── bucket ───────────────────────────── @Test @@ -579,6 +736,8 @@ public Optional toBackendProperties() { private static final class WriteHandle implements ConnectorWriteHandle { private final ConnectorTableHandle tableHandle; private boolean overwrite; + private List boundTargetColumns = Collections.emptyList(); + private String boundWriteMetadataIdentity; WriteHandle(ConnectorTableHandle tableHandle) { this.tableHandle = tableHandle; @@ -589,6 +748,16 @@ WriteHandle overwrite(boolean v) { return this; } + WriteHandle boundWriteMetadataIdentity(String identity) { + this.boundWriteMetadataIdentity = identity; + return this; + } + + WriteHandle boundTargetColumns(List columns) { + this.boundTargetColumns = columns; + return this; + } + @Override public ConnectorTableHandle getTableHandle() { return tableHandle; @@ -599,6 +768,11 @@ public List getColumns() { return Collections.emptyList(); } + @Override + public List getBoundTargetColumns() { + return boundTargetColumns; + } + @Override public boolean isOverwrite() { return overwrite; @@ -608,16 +782,28 @@ public boolean isOverwrite() { public Map getStaticPartitionSpec() { return Collections.emptyMap(); } + + @Override + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } } /** A session carrying the bound connector transaction and the session-variable overrides. */ private static final class WriteSession implements ConnectorSession { private final ConnectorTransaction txn; private final Map sessionProperties; + private final ConnectorStatementScope statementScope; WriteSession(ConnectorTransaction txn, Map sessionProperties) { + this(txn, sessionProperties, ConnectorStatementScope.NONE); + } + + WriteSession(ConnectorTransaction txn, Map sessionProperties, + ConnectorStatementScope statementScope) { this.txn = txn; this.sessionProperties = sessionProperties; + this.statementScope = statementScope; } @Override @@ -669,6 +855,21 @@ public T getProperty(String name, Class type) { public Map getCatalogProperties() { return Collections.emptyMap(); } + + @Override + public ConnectorStatementScope getStatementScope() { + return statementScope; + } + } + + private static final class TestStatementScope implements ConnectorStatementScope { + private final Map values = new HashMap<>(); + + @Override + @SuppressWarnings("unchecked") + public T computeIfAbsent(String key, Supplier loader) { + return (T) values.computeIfAbsent(key, ignored -> loader.get()); + } } /** Recording {@link HmsClient} fake returning canned table/partition metadata and recording the calls the @@ -676,6 +877,7 @@ public Map getCatalogProperties() { private static final class RecordingHmsClient implements HmsClient { private final List calls = new ArrayList<>(); private HmsTableInfo table; + private Map defaultValues = Collections.emptyMap(); private List partitionNames = Collections.emptyList(); private List partitions = Collections.emptyList(); @@ -708,9 +910,19 @@ public HmsTableInfo getTable(String dbName, String tableName) { return table; } + @Override + public HmsTableInfo getTableFresh(String dbName, String tableName) { + calls.add("getTableFresh:" + dbName + "." + tableName); + if (table == null) { + throw new UnsupportedOperationException("no canned table"); + } + return table; + } + @Override public Map getDefaultColumnValues(String dbName, String tableName) { - return Collections.emptyMap(); + calls.add("getDefaultColumnValues:" + dbName + "." + tableName); + return defaultValues; } @Override diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index 19d46822dc65ec..b9244157131de4 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -425,6 +425,22 @@ public void acquireSharedLock(String queryId, long txnId, String user, String db delegate.acquireSharedLock(queryId, txnId, user, dbName, tableName, partitionNames, timeoutMs); } + @Override + public long acquireExclusiveTableLock(String queryId, String user, String dbName, + String tableName, long timeoutMs) { + return delegate.acquireExclusiveTableLock(queryId, user, dbName, tableName, timeoutMs); + } + + @Override + public void releaseLock(long lockId) { + delegate.releaseLock(lockId); + } + + @Override + public void heartbeatLock(long lockId) { + delegate.heartbeatLock(lockId); + } + @Override public long getCurrentNotificationEventId() { return delegate.getCurrentNotificationEventId(); diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java index 1be6692974bd51..95c98ddd464016 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsClient.java @@ -388,6 +388,27 @@ default void acquireSharedLock(String queryId, long txnId, String user, String d throw new UnsupportedOperationException("acquireSharedLock is not supported by this client"); } + /** + * Acquire a standalone exclusive lock over a non-transactional Hive table. + * + * @return the HMS lock id, which the caller must release with {@link #releaseLock(long)} + */ + default long acquireExclusiveTableLock(String queryId, String user, String dbName, + String tableName, long timeoutMs) { + throw new UnsupportedOperationException( + "acquireExclusiveTableLock is not supported by this client"); + } + + /** Release a standalone HMS lock. */ + default void releaseLock(long lockId) { + throw new UnsupportedOperationException("releaseLock is not supported by this client"); + } + + /** Heartbeat a standalone HMS lock so long-running publication cannot lose its exclusion lease. */ + default void heartbeatLock(long lockId) { + throw new UnsupportedOperationException("heartbeatLock is not supported by this client"); + } + // ========== Phase 5: Metastore notification events (incremental metadata sync) ========== // // Optional: the incremental-metadata event feed. Only the production {@link ThriftHmsClient} diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsTableInfo.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsTableInfo.java index a037818f6b067f..fcfc66166e1917 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsTableInfo.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsTableInfo.java @@ -34,6 +34,8 @@ public final class HmsTableInfo { private final String dbName; private final String tableName; + private final String owner; + private final int createTime; private final String tableType; private final String location; private final String inputFormat; @@ -51,6 +53,8 @@ public final class HmsTableInfo { private HmsTableInfo(Builder builder) { this.dbName = Objects.requireNonNull(builder.dbName, "dbName"); this.tableName = Objects.requireNonNull(builder.tableName, "tableName"); + this.owner = builder.owner; + this.createTime = builder.createTime; this.tableType = builder.tableType; this.location = builder.location; this.inputFormat = builder.inputFormat; @@ -84,6 +88,16 @@ public String getTableName() { return tableName; } + /** HMS owner captured with the table generation. */ + public String getOwner() { + return owner; + } + + /** HMS creation time, used as the standard metastore table-incarnation signal. */ + public int getCreateTime() { + return createTime; + } + /** e.g. "MANAGED_TABLE", "EXTERNAL_TABLE", "VIRTUAL_VIEW". */ public String getTableType() { return tableType; @@ -165,6 +179,8 @@ public static Builder builder() { public static final class Builder { private String dbName; private String tableName; + private String owner; + private int createTime; private String tableType; private String location; private String inputFormat; @@ -192,6 +208,16 @@ public Builder tableName(String val) { return this; } + public Builder owner(String val) { + this.owner = val; + return this; + } + + public Builder createTime(int val) { + this.createTime = val; + return this; + } + public Builder tableType(String val) { this.tableType = val; return this; diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java index a0afe7f245471d..e2f7f3df9d8110 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/ThriftHmsClient.java @@ -538,11 +538,75 @@ public void acquireSharedLock(String queryId, long txnId, String user, String db } } + @Override + public long acquireExclusiveTableLock(String queryId, String user, String dbName, + String tableName, long timeoutMs) { + LockComponentBuilder componentBuilder = new LockComponentBuilder(); + componentBuilder.setExclusive(); + componentBuilder.setOperationType(DataOperationType.NO_TXN); + componentBuilder.setDbName(dbName); + componentBuilder.setTableName(tableName); + componentBuilder.setIsTransactional(false); + LockRequest lockRequest = new LockRequestBuilder(queryId) + .setUser(user) + .addLockComponent(componentBuilder.build()) + .build(); + LockResponse response = execute(client -> client.lock(lockRequest)); + long lockId = response.getLockid(); + long start = System.currentTimeMillis(); + try { + while (response.getState() == LockState.WAITING) { + if (System.currentTimeMillis() - start > timeoutMs) { + throw new HmsClientException("acquire exclusive table lock timeout for query " + + queryId + ", timeout(ms): " + timeoutMs); + } + response = execute(client -> client.checkLock(lockId)); + } + if (response.getState() != LockState.ACQUIRED) { + throw new HmsClientException( + "failed to acquire exclusive table lock, lock in state " + response.getState()); + } + return lockId; + } catch (RuntimeException e) { + // A standalone WAITING lock is not owned by an ACID transaction, so it must be removed explicitly. + try { + releaseLock(lockId); + } catch (RuntimeException cleanupFailure) { + e.addSuppressed(cleanupFailure); + } + throw e; + } + } + + @Override + public void releaseLock(long lockId) { + execute(client -> { + client.unlock(lockId); + return null; + }); + } + + @Override + public void heartbeatLock(long lockId) { + execute(client -> { + client.heartbeat(0, lockId); + return null; + }); + } + @Override public long getCurrentNotificationEventId() { return execute(client -> { - CurrentNotificationEventId id = client.getCurrentNotificationEventId(); - return id == null ? -1L : id.getEventId(); + try { + CurrentNotificationEventId id = client.getCurrentNotificationEventId(); + return id == null ? -1L : id.getEventId(); + } catch (TApplicationException e) { + // Some older metastores expose this optional RPC but reject it internally. The table lock and + // metadata identity remain the write fence; absence of the extra event watermark must not + // reject every write against those metastores. + LOG.debug("Hive metastore does not support notification watermarks, returning -1", e); + return -1L; + } }); } @@ -659,6 +723,8 @@ private HmsTableInfo convertTable(Table table) { HmsTableInfo.Builder builder = HmsTableInfo.builder() .dbName(table.getDbName()) .tableName(table.getTableName()) + .owner(table.getOwner()) + .createTime(table.getCreateTime()) .tableType(table.getTableType()) .parameters(table.getParameters()) // View text (null for a base table) — the hive VIEW SPI's isView signal + view-SQL source. diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java index 2d495293a5e013..3547df3202b1df 100644 --- a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/ThriftHmsClientWriteAcidTest.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import shade.doris.hive.org.apache.thrift.TApplicationException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; @@ -360,6 +361,53 @@ public void testAcquireSharedLockTimesOut() { client.acquireSharedLock("q", 5L, "u", "db", "t", Collections.emptyList(), -1L)); } + @Test + public void testExclusiveTableLockIsStandaloneAndExplicitlyReleased() { + RecordingClient fake = new RecordingClient() + .stub("lock", new LockResponse(9L, LockState.ACQUIRED)); + ThriftHmsClient client = newClient(fake); + + long lockId = client.acquireExclusiveTableLock("q", "u", "db", "t", 1000L); + client.heartbeatLock(lockId); + client.releaseLock(lockId); + + LockRequest request = (LockRequest) argsOf(fake, "lock")[0]; + LockComponent component = request.getComponent().get(0); + Assertions.assertEquals(9L, lockId); + Assertions.assertEquals(LockType.EXCLUSIVE, component.getType()); + Assertions.assertEquals(DataOperationType.NO_TXN, component.getOperationType()); + Assertions.assertFalse(component.isIsTransactional(), + "plain Hive publication must not masquerade as an ACID write lock"); + Assertions.assertEquals(0L, argsOf(fake, "heartbeat")[0]); + Assertions.assertEquals(9L, argsOf(fake, "heartbeat")[1]); + Assertions.assertEquals(9L, argsOf(fake, "unlock")[0]); + } + + @Test + public void testExclusiveTableLockTimeoutRemovesWaitingLock() { + RecordingClient fake = new RecordingClient() + .stub("lock", new LockResponse(10L, LockState.WAITING)) + .stub("checkLock", new LockResponse(10L, LockState.WAITING)); + ThriftHmsClient client = newClient(fake); + + Assertions.assertThrows(HmsClientException.class, + () -> client.acquireExclusiveTableLock("q", "u", "db", "t", -1L)); + + Assertions.assertEquals(10L, argsOf(fake, "unlock")[0], + "a standalone waiting lock has no transaction owner to clean it up"); + } + + @Test + public void testNotificationWatermarkFallsBackWhenMetastoreRejectsOptionalApi() { + RecordingClient fake = new RecordingClient().stub("getCurrentNotificationEventId", + new TApplicationException(TApplicationException.INTERNAL_ERROR, + "Internal error processing get_current_notificationEventId")); + ThriftHmsClient client = newClient(fake); + + Assertions.assertEquals(-1L, client.getCurrentNotificationEventId(), + "legacy metastores must disable the optional watermark instead of rejecting writes"); + } + // ---- harness -------------------------------------------------------------------------------- private static ThriftHmsClient newClient(RecordingClient handler) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java index dd1fc6ab38e979..9fb98d8f75411d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java @@ -112,6 +112,7 @@ public class IcebergConnectorMetadata implements ConnectorMetadata { // Internal sentinel property carrying a tag/branch ref name from resolveTimeTravel to applySnapshot (the // typed ConnectorMvccSnapshot has snapshotId/schemaId carriers but no ref field). NOT a BE scan option. static final String REF_PROPERTY = "iceberg.scan.ref"; + private static final String EMPTY_PARTITION_STYLE_PROPERTY = "iceberg.empty.partition.style"; // Iceberg v3 row-lineage hidden columns. Local literal copies of the Doris-side constants — the // connector cannot import fe-core. Column names mirror IcebergUtils.ICEBERG_ROW_ID_COL / @@ -550,7 +551,10 @@ private ConnectorTableSchema buildTableSchema(String tableName, Table table, Sch } } - return new ConnectorTableSchema(tableName, columns, "ICEBERG", tableProps); + // Capture the write identity from this exact Table object so the schema-cache generation and the + // later beginWrite fence cannot straddle a drop/recreate or metadata commit. + return new ConnectorTableSchema(tableName, columns, "ICEBERG", tableProps, + Collections.emptySet(), IcebergWritePlanProvider.writeMetadataIdentity(table)); } /** @@ -1702,7 +1706,7 @@ public Optional getSysTableHandle(ConnectorSession session IcebergTableHandle base = (IcebergTableHandle) baseTableHandle; return Optional.of(IcebergTableHandle.forSystemTable( base.getDbName(), base.getTableName(), sys, - base.getSnapshotId(), base.getRef(), base.getSchemaId())); + base.getSnapshotId(), base.getRef(), base.getSchemaId(), base.isSnapshotResolved())); } /** @@ -1856,7 +1860,8 @@ public boolean supportsCastPredicatePushdown(ConnectorSession session) { * remote PARTITIONS scan runs inside the FE-injected auth context. * *

The partition set + freshness are enumerated at the handle's pinned snapshot when present - * ({@code iceHandle.getSnapshotId() >= 0}), else the table's latest snapshot. The generic model (3/3) must + * ({@code iceHandle.getSnapshotId() >= 0}), remain empty for an explicitly resolved-empty handle, or else + * use the table's latest snapshot. The generic model (3/3) must * thread the query's pin onto the handle (via {@code applySnapshot} with {@code beginQuerySnapshot}'s * snapshot) before calling this, so the MTMV partition/freshness view stays consistent with the data-scan * pin — mirroring master, which routes enumeration, freshness and the scan through ONE snapshot cache value.

@@ -1875,10 +1880,11 @@ public Optional getMvccPartitionView( // function of the pinned MVCC coordinate (a new snapshot/schema yields a new key, never a stale hit). // The lookup sits INSIDE executeAuthenticated so a miss runs the loader (resolveTableForRead + the // remote PARTITIONS build) under the FE-injected auth scope; a hit returns without any remote call. A - // null cache (session=user / no-cache catalog) computes directly every call. -1 (empty table / unpinned) - // enumerates the current snapshot and caches a trivially-empty view (harmless; REFRESH re-pins). + // null cache (session=user / no-cache catalog) computes directly every call. A resolved-empty -1 + // bypasses cache A because its numeric key is otherwise indistinguishable from an unresolved latest + // read, even though only the former is a query-begin MVCC boundary. return context.executeAuthenticated(() -> { - if (mvccPartitionViewCache == null) { + if (mvccPartitionViewCache == null || iceHandle.isResolvedEmptySnapshot()) { return Optional.of(buildMvccPartitionViewUncached(session, iceHandle)); } ConnectorTableKey key = new ConnectorTableKey(iceHandle.getDbName(), @@ -1900,6 +1906,12 @@ public Optional getMvccPartitionView( */ private ConnectorMvccPartitionView buildMvccPartitionViewUncached( ConnectorSession session, IcebergTableHandle iceHandle) { + if (iceHandle.isResolvedEmptySnapshot()) { + // Data rows, partition style and freshness must describe one query-begin generation; neither a first + // append nor metadata-only spec evolution may be observed through the live table after this pin. + return IcebergPartitionUtils.buildResolvedEmptyMvccPartitionView( + iceHandle.getResolvedEmptyPartitionStyle()); + } Table table = resolveTableForRead(session, iceHandle); return IcebergPartitionUtils.buildMvccPartitionView(table, iceHandle.getSnapshotId(), TableIdentifier.of(iceHandle.getDbName(), iceHandle.getTableName()), partitionCache); @@ -1947,6 +1959,10 @@ public List listPartitionNames(ConnectorSession session, ConnectorTableH public List listPartitions(ConnectorSession session, ConnectorTableHandle handle, Optional filter) { IcebergTableHandle iceHandle = (IcebergTableHandle) handle; + if (iceHandle.isResolvedEmptySnapshot()) { + // LIST metadata must preserve the same empty generation as the scan even for non-RANGE specs. + return Collections.emptyList(); + } try { // PERF-06 cache A: memoize the BUILT partition-info list keyed by (db, table, snapshotId, schemaId). // The lookup sits INSIDE executeAuthenticated (a miss runs the remote build under the auth scope; a hit @@ -2013,8 +2029,12 @@ public Optional beginQuerySnapshot( IcebergLatestSnapshotCache.CachedSnapshot pin = latestSnapshotCache != null ? latestSnapshotCache.getOrLoad(id, () -> loadLatestSnapshotPin(session, iceHandle)) : loadLatestSnapshotPin(session, iceHandle); - return Optional.of( - ConnectorMvccSnapshot.builder().snapshotId(pin.snapshotId).schemaId(pin.schemaId).build()); + ConnectorMvccSnapshot.Builder snapshot = ConnectorMvccSnapshot.builder() + .snapshotId(pin.snapshotId).schemaId(pin.schemaId); + if (pin.snapshotId < 0) { + snapshot.property(EMPTY_PARTITION_STYLE_PROPERTY, pin.emptyPartitionStyle.name()); + } + return Optional.of(snapshot.build()); } /** @@ -2026,8 +2046,11 @@ private IcebergLatestSnapshotCache.CachedSnapshot loadLatestSnapshotPin( ConnectorSession session, IcebergTableHandle iceHandle) { Table table = loadTable(session, iceHandle); Snapshot current = table.currentSnapshot(); + ConnectorMvccPartitionView.Style emptyPartitionStyle = IcebergPartitionUtils.isValidRelatedTable(table) + ? ConnectorMvccPartitionView.Style.RANGE + : ConnectorMvccPartitionView.Style.UNPARTITIONED; return new IcebergLatestSnapshotCache.CachedSnapshot( - current == null ? -1L : current.snapshotId(), table.schema().schemaId()); + current == null ? -1L : current.snapshotId(), table.schema().schemaId(), emptyPartitionStyle); } /** @@ -2137,8 +2160,8 @@ private long parseTimestampMillis(ConnectorSession session, ConnectorTimeTravelS * Threads a resolved MVCC / time-travel pin onto the handle BEFORE the scan reads it (the generic * {@code PluginDrivenScanNode} calls this via {@code applyMvccSnapshotPin}). Reads the typed * {@code snapshotId}/{@code schemaId} and the {@code iceberg.scan.ref} property; an empty-table / query-begin - * latest pin ({@code snapshotId<0} and no ref) returns the handle UNCHANGED (read latest — a - * {@code useSnapshot(-1)} would be a non-existent snapshot; mirrors paimon's {@code -1} guard). + * latest pin ({@code snapshotId<0} and no ref) remains unpinned for scanning because + * {@code useSnapshot(-1)} would be invalid, but is recorded as an explicitly resolved empty snapshot. */ @Override public ConnectorTableHandle applySnapshot(ConnectorSession session, @@ -2149,10 +2172,10 @@ public ConnectorTableHandle applySnapshot(ConnectorSession session, } String ref = snapshot.getProperties().get(REF_PROPERTY); long snapshotId = snapshot.getSnapshotId(); - if (snapshotId < 0 && ref == null) { - return iceHandle; - } - return iceHandle.withSnapshot(snapshotId, ref, snapshot.getSchemaId()); + ConnectorMvccPartitionView.Style emptyPartitionStyle = ConnectorMvccPartitionView.Style.valueOf( + snapshot.getProperties().getOrDefault(EMPTY_PARTITION_STYLE_PROPERTY, + ConnectorMvccPartitionView.Style.UNPARTITIONED.name())); + return iceHandle.withSnapshot(snapshotId, ref, snapshot.getSchemaId(), emptyPartitionStyle); } /** diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java index 4c3f687c5f5bf4..21b301a3416490 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorTransaction.java @@ -219,11 +219,19 @@ public void beginWrite(ConnectorSession session, String db, String tableName, Ic Table loaded = IcebergStatementScope.sharedWritableTable(session, db, tableName, () -> catalogOps.loadTable(db, tableName)); this.table = loaded; + validateBoundWriteGeneration(ctx, loaded); if (writeSchemaContext != null) { writeSchemaContext.validateCurrentSchema(loaded, ctx.isOverwrite()); } applyBeginGuards(ctx, tableName); - this.transaction = openTransaction(loaded, ctx.isOverwrite()); + Transaction opened = openTransaction(loaded, ctx.isOverwrite()); + // BaseTable.newTransaction refreshes metadata. Fence that second load too, closing the + // drop/recreate race between the initial generation check and transaction construction. + validateBoundWriteGeneration(ctx, loaded); + if (writeSchemaContext != null) { + writeSchemaContext.validateCurrentSchema(loaded, ctx.isOverwrite()); + } + this.transaction = opened; return null; }); } catch (Exception e) { @@ -236,6 +244,17 @@ public void beginWrite(ConnectorSession session, String db, String tableName, Ic } } + private static void validateBoundWriteGeneration(IcebergWriteContext ctx, Table loaded) { + String boundIdentity = ctx.getBoundWriteMetadataIdentity(); + // Reject a same-name replacement before opening its SDK transaction; the replacement must never + // become the conflict baseline for a plan whose rows and defaults were bound against the old UUID. + if (boundIdentity != null + && !boundIdentity.equals(IcebergWritePlanProvider.writeMetadataIdentity(loaded))) { + throw new IllegalArgumentException( + "Iceberg write metadata changed after the write was bound; retry the statement"); + } + } + /** * Opens the SDK transaction for {@code loaded}. On a Kerberos catalog the table's {@link FileIO} is wrapped * in a plugin-side Kerberos {@code doAs} ({@link IcebergAuthenticatedFileIO}); otherwise this is byte-for-byte @@ -394,13 +413,14 @@ private void applyBeginGuards(IcebergWriteContext ctx, String tableName) { // scan used, S_read), threaded onto the write handle and carried on the ctx. The commit-time // removeDeletes (option D) re-derives from baseSnapshotId, and BE unions the scan-time (S_read) // old deletes into the new DV — anchoring both at S_read keeps supply and remove on one snapshot - // (no resurrection under a concurrent commit in the read->begin-write window). A -1 readSnapshotId - // (no pin: a caller without the threaded handle) falls back to the begin-time current snapshot. + // (no resurrection under a concurrent commit in the read->begin-write window). long pinnedReadSnapshot = ctx.getReadSnapshotId(); - // Keep both ternary arms boxed (Long): getSnapshotIdIfPresent returns null for an empty table - // (no snapshot), and a primitive arm would force-unbox that null into an NPE. - this.baseSnapshotId = pinnedReadSnapshot >= 0 - ? Long.valueOf(pinnedReadSnapshot) : getSnapshotIdIfPresent(table); + if (ctx.isReadSnapshotResolved()) { + // An explicitly empty read stays null so validation covers a concurrent first append. + this.baseSnapshotId = pinnedReadSnapshot >= 0 ? Long.valueOf(pinnedReadSnapshot) : null; + } else { + this.baseSnapshotId = getSnapshotIdIfPresent(table); + } if (table instanceof HasTableOperations) { int formatVersion = ((HasTableOperations) table).operations().current().formatVersion(); if (formatVersion < 2) { @@ -749,9 +769,11 @@ private Expression buildPartitionFilter(Map staticPartitions, Pa } List predicates = new ArrayList<>(); + Set unmatchedFields = new HashSet<>(staticPartitions.keySet()); for (PartitionField field : spec.fields()) { String partitionColName = field.name(); if (staticPartitions.containsKey(partitionColName)) { + unmatchedFields.remove(partitionColName); String partitionValueStr = staticPartitions.get(partitionColName); Types.NestedField sourceField = schema.findField(field.sourceId()); if (sourceField == null) { @@ -768,8 +790,10 @@ private Expression buildPartitionFilter(Map staticPartitions, Pa } } - if (predicates.isEmpty()) { - return Expressions.alwaysTrue(); + // A stale nonempty static spec must never widen into an always-true full-table overwrite. + if (!unmatchedFields.isEmpty()) { + throw new DorisConnectorException("Static partition field does not match the current Iceberg spec: " + + unmatchedFields); } Expression result = predicates.get(0); for (int i = 1; i < predicates.size(); i++) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java index 35b63c6aa8c8fc..8c98d57a46e7a7 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCacheEntry; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -35,13 +36,13 @@ * query-begin pin ({@link IcebergConnectorMetadata#beginQuerySnapshot}) reads the SAME snapshot until the * entry expires or is invalidated by {@code REFRESH TABLE}/{@code REFRESH CATALOG}. * - *

Value carries BOTH snapshotId and schemaId (the single iceberg-specific deviation from the paimon - * {@code long}-only mirror). {@code beginQuerySnapshot} pins the snapshot id and the LATEST schema - * id ({@code table.schema().schemaId()} — not {@code currentSnapshot().schemaId()}, mirroring legacy + *

Value carries snapshotId, schemaId and the resolved-empty partition style. + * {@code beginQuerySnapshot} pins the snapshot id and the LATEST schema id + * ({@code table.schema().schemaId()} — not {@code currentSnapshot().schemaId()}, mirroring legacy * {@code IcebergUtils.getLatestIcebergSnapshot}). A schema-only {@code ALTER} bumps the latest schema id - * without producing a new snapshot, so the two ids must be captured atomically — otherwise two live reads - * within one pin could observe a snapshotId/schemaId skew. The value type therefore ports legacy - * {@code IcebergSnapshot}'s {@code (snapshotId, schemaId)} shape. + * without producing a new snapshot, so the ids must be captured atomically. When the snapshot id is negative, + * the partition style must come from that same metadata generation; otherwise later spec evolution could make + * an empty scan expose live partition metadata. * *

Backed by the shared {@link MetaCacheEntry} framework (independent-copy meta-cache migration): a * contextual, access-TTL entry whose per-query loader is supplied at {@link #getOrLoad}. TTL is @@ -53,14 +54,21 @@ */ final class IcebergLatestSnapshotCache { - /** Immutable atomic pin = the latest snapshot id plus the latest schema id (port of legacy IcebergSnapshot). */ + /** Immutable atomic pin for the latest snapshot/schema and its resolved-empty partition style. */ static final class CachedSnapshot { final long snapshotId; final long schemaId; + final ConnectorMvccPartitionView.Style emptyPartitionStyle; CachedSnapshot(long snapshotId, long schemaId) { + this(snapshotId, schemaId, ConnectorMvccPartitionView.Style.UNPARTITIONED); + } + + CachedSnapshot(long snapshotId, long schemaId, + ConnectorMvccPartitionView.Style emptyPartitionStyle) { this.snapshotId = snapshotId; this.schemaId = schemaId; + this.emptyPartitionStyle = emptyPartitionStyle; } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index 92e02313ea62b7..a5ff4562e915cd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -534,6 +534,16 @@ static ConnectorMvccPartitionView buildMvccPartitionView(Table table, long pinne return buildMvccPartitionView(table, pinnedSnapshotId, null, null); } + /** Builds the style captured with the empty pin without consulting a later metadata generation. */ + static ConnectorMvccPartitionView buildResolvedEmptyMvccPartitionView( + ConnectorMvccPartitionView.Style style) { + if (style != ConnectorMvccPartitionView.Style.RANGE) { + return ConnectorMvccPartitionView.unpartitioned(); + } + return new ConnectorMvccPartitionView(ConnectorMvccPartitionView.Style.RANGE, + ConnectorMvccPartitionView.Freshness.SNAPSHOT_ID, Collections.emptyList(), 0L); + } + /** * Cache-aware overload (PERF-02): {@code id} + {@code cache} route the PARTITIONS scan through the * per-catalog {@link IcebergPartitionCache} keyed by {@code (id, resolvedSnapshotId)}. {@code cache == null} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index 024a036df11158..d306facec300e2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -437,7 +437,8 @@ public List planScan(ConnectorSession session, ConnectorScan public long streamingSplitEstimate(ConnectorSession session, ConnectorTableHandle handle, Optional filter, boolean countPushdown) { IcebergTableHandle iceHandle = (IcebergTableHandle) handle; - if (iceHandle.isSystemTable() || !sessionBool(session, ENABLE_EXTERNAL_TABLE_BATCH_MODE, true)) { + if (iceHandle.isResolvedEmptySnapshot() || iceHandle.isSystemTable() + || !sessionBool(session, ENABLE_EXTERNAL_TABLE_BATCH_MODE, true)) { return -1; } Table table = resolveTable(session, iceHandle); @@ -475,13 +476,19 @@ public long streamingSplitEstimate(ConnectorSession session, ConnectorTableHandl * a FIXED size ({@code file_split_size} if set, else {@code max_split_size} — NOT the per-table * {@link #determineTargetFileSplitSize} heuristic, which would force materializing every task), so * {@code planFiles()} streams without holding the full task list — the OOM protection. Bypasses the manifest - * cache (its planning materializes; legacy's lazy batch path only ran with the manifest cache off). Only - * called after {@link #streamingSplitEstimate} returned ≥ 0, so the snapshot/non-sys/v<3 gates already hold. + * cache (its planning materializes; legacy's lazy batch path only ran with the manifest cache off). Usually + * called after {@link #streamingSplitEstimate} returned ≥ 0; because MVCC pinning happens afterward, this + * method must independently preserve an explicitly empty pinned snapshot. */ @Override public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorTableHandle handle, List columns, Optional filter, long limit) { IcebergTableHandle iceHandle = (IcebergTableHandle) handle; + if (iceHandle.isResolvedEmptySnapshot()) { + // The batch decision is made before the engine pins MVCC; once pinned empty, streaming must + // preserve that boundary instead of interpreting Iceberg's sentinel as the latest snapshot. + return emptySplitSource(); + } Table table = resolveTable(session, iceHandle); TableScan scan = buildScan(table, iceHandle, filter, session); int formatVersion = getFormatVersion(table); @@ -499,6 +506,24 @@ public ConnectorSplitSource streamSplits(ConnectorSession session, ConnectorTabl orderedPartitionKeys, zone, uriNormalizer, sliceSize, iceHandle.getRewriteFileScope()); } + private static ConnectorSplitSource emptySplitSource() { + return new ConnectorSplitSource() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public ConnectorScanRange next() { + throw new NoSuchElementException(); + } + + @Override + public void close() { + } + }; + } + /** * The streaming source's whole-file enumeration, byte-offset-split at {@code sliceSize}. PERF-04 (C17): when * the manifest cache is enabled, read manifests THROUGH THE CACHE via the lazy {@link #cacheBackedFileScanTasks} @@ -613,6 +638,13 @@ private List planScanInternal( Optional filter, boolean countPushdown) { IcebergTableHandle iceHandle = (IcebergTableHandle) handle; + if (iceHandle.isResolvedEmptySnapshot() && !isSnapshotIndependentSystemTable(iceHandle)) { + // Iceberg has no snapshot id that can represent "before the first commit". Returning no ranges is + // the read-side MVCC fence; otherwise a refreshed Table would turn -1 into "latest" and expose a + // concurrent first append to MERGE after its anti-join had already decided the row was absent. Static + // metadata-history tables are exempt because their creation rows exist without a data snapshot. + return Collections.emptyList(); + } if (iceHandle.isSystemTable()) { // System tables take a metadata-table path, never the data-file path below (no count pushdown, no // data-file ranges) — mirrors legacy IcebergScanNode branching on isSystemTable. $position_deletes @@ -1143,6 +1175,17 @@ private static boolean supportsSnapshotSelection(IcebergTableHandle handle) { && type != MetadataTableType.ALL_ENTRIES; } + /** Metadata tables whose rows exist independently of a data snapshot. */ + private static boolean isSnapshotIndependentSystemTable(IcebergTableHandle handle) { + if (!handle.isSystemTable()) { + return false; + } + MetadataTableType type = MetadataTableType.from(handle.getSysTableName()); + // Only metadata_log_entries has a creation row before the first data snapshot; every other metadata + // table must preserve the resolved-empty fence or a concurrent first append becomes visible. + return type == MetadataTableType.METADATA_LOG_ENTRIES; + } + /** * The schema AS OF the handle's pinned schema id (for time-travel reads under schema evolution); the latest * schema when there is no pinned id or it is absent from {@code table.schemas()} (defensive — legacy diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableHandle.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableHandle.java index 64af1e29f207c4..530fa7cde107e3 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableHandle.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableHandle.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; import com.google.common.collect.ImmutableSet; @@ -34,11 +35,16 @@ * iceberg SDK applies time-travel through {@code TableScan.useSnapshot(id)} / {@code useRef(name)} rather * than a {@code Table.copy(properties)} option map: *

    - *
  • {@code snapshotId} ({@code -1} = none) — {@code FOR VERSION AS OF } / {@code FOR TIME AS OF}.
    • + *
  • {@code snapshotId} ({@code -1} = no concrete snapshot) — {@code FOR VERSION AS OF } / + * {@code FOR TIME AS OF}.
    • *
  • {@code ref} ({@code null} = none) — a tag/branch name; the scan pins by REF ({@code useRef}) so a * later commit to the tag/branch is honored (legacy parity).
    • *
  • {@code schemaId} ({@code -1} = latest) — the schema version AS OF the pin, so the field-id dictionary * and {@code getTableSchema(@snapshot)} read the historical schema.
    • + *
  • {@code snapshotResolved} distinguishes an unresolved latest read from a query-begin read that + * resolved to an empty table; both have {@code snapshotId=-1}, but only the latter is an MVCC boundary.
    • + *
  • {@code resolvedEmptyPartitionStyle} preserves the RANGE eligibility observed with that empty boundary, + * so a later partition-spec commit cannot change its metadata view.
    • *
* The handle is immutable: {@link #withSnapshot} returns a NEW handle (the pin is part of the handle * identity, so {@link #equals}/{@link #hashCode}/{@link #toString} include it). @@ -61,6 +67,8 @@ public class IcebergTableHandle implements ConnectorTableHandle { private final long snapshotId; private final String ref; private final long schemaId; + private final boolean snapshotResolved; + private final ConnectorMvccPartitionView.Style resolvedEmptyPartitionStyle; /** * Bare system-table name (no {@code "$"}), lower-cased by the caller @@ -97,16 +105,20 @@ public class IcebergTableHandle implements ConnectorTableHandle { private final boolean topnLazyMaterialize; public IcebergTableHandle(String dbName, String tableName) { - this(dbName, tableName, NO_PIN, null, NO_PIN, null, null, false); + this(dbName, tableName, NO_PIN, null, NO_PIN, false, + ConnectorMvccPartitionView.Style.UNPARTITIONED, null, null, false); } private IcebergTableHandle(String dbName, String tableName, long snapshotId, String ref, long schemaId, + boolean snapshotResolved, ConnectorMvccPartitionView.Style resolvedEmptyPartitionStyle, String sysTableName, Set rewriteFileScope, boolean topnLazyMaterialize) { this.dbName = dbName; this.tableName = tableName; this.snapshotId = snapshotId; this.ref = ref; this.schemaId = schemaId; + this.snapshotResolved = snapshotResolved; + this.resolvedEmptyPartitionStyle = resolvedEmptyPartitionStyle; this.sysTableName = sysTableName; this.rewriteFileScope = rewriteFileScope; this.topnLazyMaterialize = topnLazyMaterialize; @@ -121,7 +133,14 @@ private IcebergTableHandle(String dbName, String tableName, long snapshotId, Str */ public static IcebergTableHandle forSystemTable(String dbName, String tableName, String sysName, long snapshotId, String ref, long schemaId) { - return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, sysName, null, false); + return forSystemTable(dbName, tableName, sysName, snapshotId, ref, schemaId, + snapshotId >= 0 || ref != null); + } + + static IcebergTableHandle forSystemTable(String dbName, String tableName, String sysName, + long snapshotId, String ref, long schemaId, boolean snapshotResolved) { + return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, + snapshotResolved, ConnectorMvccPartitionView.Style.UNPARTITIONED, sysName, null, false); } public String getDbName() { @@ -147,6 +166,11 @@ public long getSchemaId() { return schemaId; } + /** Whether query-begin snapshot resolution ran, including an explicitly empty table ({@code -1}). */ + public boolean isSnapshotResolved() { + return snapshotResolved; + } + /** Bare system-table name (no {@code "$"}), or {@code null} for a normal data-table handle. */ public String getSysTableName() { return sysTableName; @@ -162,6 +186,16 @@ public boolean hasSnapshotPin() { return snapshotId >= 0 || ref != null; } + /** Whether snapshot resolution observed a table before its first snapshot was committed. */ + public boolean isResolvedEmptySnapshot() { + return snapshotResolved && snapshotId < 0 && ref == null; + } + + /** Partition style captured atomically with a resolved-empty query-begin snapshot. */ + ConnectorMvccPartitionView.Style getResolvedEmptyPartitionStyle() { + return resolvedEmptyPartitionStyle; + } + /** * The rewrite file scope (raw iceberg data-file paths the scan is restricted to), or {@code null} for a * normal full scan. See {@link #rewriteFileScope} and {@link #withRewriteFileScope}. @@ -180,11 +214,17 @@ public boolean isTopnLazyMaterialize() { * {@code PaimonTableHandle.withScanOptions}/{@code withBranch} but with iceberg's typed carriers. */ public IcebergTableHandle withSnapshot(long snapshotId, String ref, long schemaId) { + return withSnapshot(snapshotId, ref, schemaId, ConnectorMvccPartitionView.Style.UNPARTITIONED); + } + + IcebergTableHandle withSnapshot(long snapshotId, String ref, long schemaId, + ConnectorMvccPartitionView.Style resolvedEmptyPartitionStyle) { // sysTableName, rewriteFileScope and topnLazyMaterialize are preserved: threading a resolved // time-travel pin in must not degrade a sys handle (t$snapshots) into a normal data-table handle, // drop a rewrite scope, or drop the lazy-materialization signal. - return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, sysTableName, - rewriteFileScope, topnLazyMaterialize); + // A resolved empty table still needs a marker even though useSnapshot(-1) is invalid. + return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, true, + resolvedEmptyPartitionStyle, sysTableName, rewriteFileScope, topnLazyMaterialize); } /** @@ -196,8 +236,9 @@ public IcebergTableHandle withSnapshot(long snapshotId, String ref, long schemaI * The other carriers (snapshot/ref/schema/sys) are preserved. */ public IcebergTableHandle withRewriteFileScope(Set rawDataFilePaths) { - return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, sysTableName, - ImmutableSet.copyOf(rawDataFilePaths), topnLazyMaterialize); + return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, snapshotResolved, + resolvedEmptyPartitionStyle, sysTableName, ImmutableSet.copyOf(rawDataFilePaths), + topnLazyMaterialize); } /** @@ -205,8 +246,8 @@ public IcebergTableHandle withRewriteFileScope(Set rawDataFilePaths) { * {@link #topnLazyMaterialize}). The other carriers (snapshot/ref/schema/sys/rewriteScope) are preserved. */ public IcebergTableHandle withTopnLazyMaterialize(boolean topnLazyMaterialize) { - return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, sysTableName, - rewriteFileScope, topnLazyMaterialize); + return new IcebergTableHandle(dbName, tableName, snapshotId, ref, schemaId, snapshotResolved, + resolvedEmptyPartitionStyle, sysTableName, rewriteFileScope, topnLazyMaterialize); } @Override @@ -220,6 +261,8 @@ public boolean equals(Object o) { IcebergTableHandle that = (IcebergTableHandle) o; return snapshotId == that.snapshotId && schemaId == that.schemaId + && snapshotResolved == that.snapshotResolved + && resolvedEmptyPartitionStyle == that.resolvedEmptyPartitionStyle && topnLazyMaterialize == that.topnLazyMaterialize && Objects.equals(dbName, that.dbName) && Objects.equals(tableName, that.tableName) @@ -230,8 +273,8 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(dbName, tableName, snapshotId, ref, schemaId, sysTableName, rewriteFileScope, - topnLazyMaterialize); + return Objects.hash(dbName, tableName, snapshotId, ref, schemaId, snapshotResolved, + resolvedEmptyPartitionStyle, sysTableName, rewriteFileScope, topnLazyMaterialize); } @Override @@ -243,6 +286,8 @@ public String toString() { if (hasSnapshotPin()) { sb.append(", snapshotId=").append(snapshotId).append(", ref=").append(ref) .append(", schemaId=").append(schemaId); + } else if (snapshotResolved) { + sb.append(", snapshot=empty"); } if (rewriteFileScope != null) { sb.append(", rewriteFileScope=").append(rewriteFileScope.size()).append(" files"); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java index 2021e967017aff..9336efdcfdb60a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTypeMapping.java @@ -63,7 +63,7 @@ public static ConnectorType fromIcebergType(Type icebergType, Types.ListType list = (Types.ListType) icebergType; ConnectorType elemType = fromIcebergType( list.elementType(), enableMappingVarbinary, enableMappingTimestampTz); - return ConnectorType.arrayOf(elemType) + return ConnectorType.arrayOf(elemType, list.isElementOptional()) .withChildrenFieldIds(Collections.singletonList(list.elementId())); case MAP: // Carry key + value field-ids (legacy recurses into both via MapType.fields()). @@ -72,7 +72,7 @@ public static ConnectorType fromIcebergType(Type icebergType, map.keyType(), enableMappingVarbinary, enableMappingTimestampTz); ConnectorType valType = fromIcebergType( map.valueType(), enableMappingVarbinary, enableMappingTimestampTz); - return ConnectorType.mapOf(keyType, valType) + return ConnectorType.mapOf(keyType, valType, map.isValueOptional()) .withChildrenFieldIds(Arrays.asList(map.keyId(), map.valueId())); case STRUCT: // Carry each field's field-id, parallel to the field types (legacy recurses field-by-field). diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteContext.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteContext.java index 8ce93420fdc6c1..9167d21809ec97 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteContext.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWriteContext.java @@ -43,33 +43,62 @@ final class IcebergWriteContext { private final Optional branchName; private final long readSnapshotId; private final IcebergWriteSchemaContext writeSchemaContext; + private final boolean readSnapshotResolved; + private final String boundWriteMetadataIdentity; IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, Map staticPartitionValues, Optional branchName) { - this(writeOperation, overwrite, staticPartitionValues, branchName, -1L); + this(writeOperation, overwrite, staticPartitionValues, branchName, -1L, false, null, null); } IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, Map staticPartitionValues, Optional branchName, long readSnapshotId) { - this.writeOperation = writeOperation; - this.overwrite = overwrite; - this.staticPartitionValues = staticPartitionValues == null - ? Collections.emptyMap() : new HashMap<>(staticPartitionValues); - this.branchName = branchName == null ? Optional.empty() : branchName; - this.readSnapshotId = readSnapshotId; - this.writeSchemaContext = null; + this(writeOperation, overwrite, staticPartitionValues, branchName, + readSnapshotId, readSnapshotId >= 0, null, null); + } + + IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, + Map staticPartitionValues, Optional branchName, + long readSnapshotId, boolean readSnapshotResolved) { + this(writeOperation, overwrite, staticPartitionValues, branchName, + readSnapshotId, readSnapshotResolved, null, null); } IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, Map staticPartitionValues, Optional branchName, long readSnapshotId, IcebergWriteSchemaContext writeSchemaContext) { + this(writeOperation, overwrite, staticPartitionValues, branchName, + readSnapshotId, readSnapshotId >= 0, writeSchemaContext, null); + } + + IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, + Map staticPartitionValues, Optional branchName, + long readSnapshotId, boolean readSnapshotResolved, + IcebergWriteSchemaContext writeSchemaContext) { + this(writeOperation, overwrite, staticPartitionValues, branchName, + readSnapshotId, readSnapshotResolved, writeSchemaContext, null); + } + + IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, + Map staticPartitionValues, Optional branchName, + long readSnapshotId, boolean readSnapshotResolved, String boundWriteMetadataIdentity) { + this(writeOperation, overwrite, staticPartitionValues, branchName, + readSnapshotId, readSnapshotResolved, null, boundWriteMetadataIdentity); + } + + IcebergWriteContext(WriteOperation writeOperation, boolean overwrite, + Map staticPartitionValues, Optional branchName, + long readSnapshotId, boolean readSnapshotResolved, + IcebergWriteSchemaContext writeSchemaContext, String boundWriteMetadataIdentity) { this.writeOperation = writeOperation; this.overwrite = overwrite; this.staticPartitionValues = staticPartitionValues == null ? Collections.emptyMap() : new HashMap<>(staticPartitionValues); this.branchName = branchName == null ? Optional.empty() : branchName; this.readSnapshotId = readSnapshotId; + this.readSnapshotResolved = readSnapshotResolved; this.writeSchemaContext = writeSchemaContext; + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; } WriteOperation getWriteOperation() { @@ -95,7 +124,8 @@ Optional getBranchName() { /** * The statement's READ snapshot id (the MVCC pin the scan used, S_read), threaded from the write - * handle in {@code planWrite}; {@code -1} = no pin (the legacy fresh-current behavior). The + * handle in {@code planWrite}; {@code -1} means either no pin or an explicitly empty read, as + * distinguished by {@link #isReadSnapshotResolved()}. The * RowDelta path anchors {@code baseSnapshotId} at this snapshot so the commit-time removeDeletes * (option D) and the scan-time deletes BE unions into the new DV share one snapshot — see * {@link IcebergConnectorTransaction} [SHOULD-2] / Fix B. @@ -107,4 +137,13 @@ long getReadSnapshotId() { IcebergWriteSchemaContext getWriteSchemaContext() { return writeSchemaContext; } + + /** Whether the read snapshot was resolved, including a table with no snapshot yet. */ + boolean isReadSnapshotResolved() { + return readSnapshotResolved; + } + + String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java index 6594c2589d85c0..2d09fe753a7536 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java @@ -71,10 +71,13 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; /** * Write plan provider for iceberg INSERT / INSERT OVERWRITE. @@ -173,6 +176,8 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl IcebergWriteContext writeContext = buildWriteContext(handle, schemaContext); transaction.beginWrite(session, tableHandle.getDbName(), tableHandle.getTableName(), writeContext); Table table = transaction.getTable(); + validateBoundWriteMetadata(table, handle); + validateBoundWriteColumns(table, schemaContext, handle, writeContext.getWriteOperation()); // commit-bridge supply (S4 part 2): read the non-equality delete supply the scan seam accumulated into the // per-statement scope. DELETE/MERGE attach it to the sink so the BE OR-merges old deletes into the new @@ -231,6 +236,135 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl } } + private void validateBoundWriteColumns(Table table, IcebergWriteSchemaContext schemaContext, + ConnectorWriteHandle handle, WriteOperation writeOperation) { + List boundTargetColumns = handle.getBoundTargetColumns(); + if (!boundTargetColumns.isEmpty() + && (writeOperation == WriteOperation.REWRITE + || writeOperation == WriteOperation.UPDATE + || writeOperation == WriteOperation.MERGE)) { + long boundLineageColumns = boundTargetColumns.stream() + .filter(ConnectorColumn::isReservedPassthrough) + .count(); + long currentLineageColumns = IcebergWriterHelper.getFormatVersion(table) >= 3 ? 2 : 0; + // Format version changes the physical sink arity without changing table.schema(); the reserved + // columns are the bind-time witness that the output and v3 schema-json belong to one generation. + if (boundLineageColumns != currentLineageColumns) { + throw new DorisConnectorException( + "Iceberg write metadata changed after the write was bound; retry the statement"); + } + } + // V3 row-lineage columns are engine-generated metadata, not fields in table.schema(). Excluding + // their neutral marker keeps the comparison on the complete user schema for INSERT and MERGE. + List boundColumns = boundTargetColumns.stream() + .filter(column -> !column.isReservedPassthrough()) + .collect(Collectors.toList()); + if (writeOperation == WriteOperation.DELETE || boundColumns.isEmpty()) { + return; + } + // A branch write is deliberately bound to the branch-head schema, which can lag the current main + // schema. The schema context separately fences branch evolution, so comparing with table.schema() + // rejects valid branch writes without adding concurrency protection. + Schema writeSchema = schemaContext == null ? table.schema() : schemaContext.getSchema(); + List currentColumns = writeSchema.columns(); + boolean rowLevelWrite = writeOperation == WriteOperation.UPDATE || writeOperation == WriteOperation.MERGE; + boolean hasSyntheticRowId = rowLevelWrite && boundColumns.size() == currentColumns.size() + 1 + && DORIS_ICEBERG_ROWID_COL.equals(boundColumns.get(boundColumns.size() - 1).getName()); + if (boundColumns.size() != currentColumns.size() && !hasSyntheticRowId) { + throw new DorisConnectorException("Iceberg table schema changed after the write was bound; retry the " + + "statement with the latest schema"); + } + boolean enableVarbinary = Boolean.parseBoolean(properties.getOrDefault( + IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); + boolean enableTimestampTz = Boolean.parseBoolean(properties.getOrDefault( + IcebergConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + for (int i = 0; i < currentColumns.size(); ++i) { + NestedField current = currentColumns.get(i); + ConnectorColumn bound = boundColumns.get(i); + ConnectorType currentType = IcebergTypeMapping.fromIcebergType( + current.type(), enableVarbinary, enableTimestampTz); + String boundDefaultSql = bound.getDefaultValueSql(); + if ("NULL".equalsIgnoreCase(boundDefaultSql)) { + boundDefaultSql = null; + } + String currentDefaultSql = current.writeDefault() == null ? null + : IcebergWriteSchemaContext.toDorisSql(current.type(), current.writeDefault(), + enableVarbinary, enableTimestampTz); + // Do not compare top-level nullability: Doris widens Iceberg required columns in its read schema + // so evolution default-fill may yield NULL. Nested requiredness remains authoritative in + // sameBoundType, while current schema JSON enforces writes at the root. + if (!current.name().equalsIgnoreCase(bound.getName()) + || !sameBoundType(currentType, bound.getType()) + || (bound.getUniqueId() >= 0 && current.fieldId() != bound.getUniqueId()) + // Omitted columns and DEFAULT expressions were already materialized from this value at bind. + || !Objects.equals(boundDefaultSql, currentDefaultSql)) { + // BE maps write expressions to schema-json by ordinal, so accepting a reordered live + // schema here could silently place values under the wrong Iceberg field names. + throw new DorisConnectorException("Iceberg table schema changed after the write was bound; retry " + + "the statement with the latest schema"); + } + } + } + + private void validateBoundWriteMetadata(Table table, ConnectorWriteHandle handle) { + String boundIdentity = handle.getBoundWriteMetadataIdentity(); + if (boundIdentity == null) { + return; + } + // The FE sort/distribution and Iceberg file metadata must come from one generation; otherwise + // beginWrite's refresh can silently stamp files with a sort order or partition spec they did not use. + if (!boundIdentity.equals(writeMetadataIdentity(table))) { + throw new DorisConnectorException( + "Iceberg write metadata changed after the write was bound; retry the statement"); + } + } + + private static boolean sameBoundType(ConnectorType current, ConnectorType bound) { + String currentName = canonicalTypeName(current.getTypeName()); + String boundName = canonicalTypeName(bound.getTypeName()); + if (!currentName.equals(boundName) + || current.getChildren().size() != bound.getChildren().size() + || current.getFieldNames().size() != bound.getFieldNames().size()) { + return false; + } + if (hasMeaningfulTypeParameters(currentName) + && (current.getPrecision() != bound.getPrecision() || current.getScale() != bound.getScale())) { + return false; + } + for (int i = 0; i < current.getChildren().size(); i++) { + if (!current.getFieldNames().isEmpty() + && !current.getFieldNames().get(i).equalsIgnoreCase(bound.getFieldNames().get(i))) { + return false; + } + int boundFieldId = bound.getChildFieldId(i); + if ((boundFieldId >= 0 && current.getChildFieldId(i) != boundFieldId) + || current.isChildNullable(i) != bound.isChildNullable(i) + || !sameBoundType(current.getChildren().get(i), bound.getChildren().get(i))) { + // Nested field identity and requiredness are part of the write contract even though the SPI + // type's general equals() deliberately excludes them for non-write consumers. + return false; + } + } + return true; + } + + private static String canonicalTypeName(String typeName) { + String normalized = typeName.toUpperCase(Locale.ROOT); + // Doris chooses a physical DECIMAL width after conversion, while Iceberg exposes one logical decimal. + // Width aliases with the same precision/scale are one schema, not concurrent evolution. + return normalized.startsWith("DECIMAL") && !"DECIMALV2".equals(normalized) + ? "DECIMALV3" : normalized; + } + + private static boolean hasMeaningfulTypeParameters(String typeName) { + return typeName.startsWith("DECIMAL") + || "CHAR".equals(typeName) + || "VARCHAR".equals(typeName) + || "VARBINARY".equals(typeName) + || "DATETIMEV2".equals(typeName) + || "TIMESTAMPTZ".equals(typeName); + } + @Override public Optional> getWriteColumns(ConnectorSession session, ConnectorTableHandle tableHandle, Optional branchName) { @@ -265,6 +399,12 @@ public void appendExplainInfo(StringBuilder output, String prefix, @Override public List getWriteSortColumns(ConnectorSession session, ConnectorTableHandle tableHandle) { + return getWriteSortColumns(session, tableHandle, Collections.emptyList()); + } + + @Override + public List getWriteSortColumns(ConnectorSession session, + ConnectorTableHandle tableHandle, List boundTargetColumns) { IcebergTableHandle handle = (IcebergTableHandle) tableHandle; Optional active = IcebergStatementScope.activeWriteSchema( @@ -278,25 +418,103 @@ public List getWriteSortColumns(ConnectorSession sessi // unconditional setSortInfo inside the isSorted() branch even when no identity column resolves. return null; } - List columns = active.isPresent() - ? active.get().getSchema().columns() : table.schema().columns(); + Map positionsByFieldId = new HashMap<>(); + if (boundTargetColumns.isEmpty()) { + List currentColumns = active.isPresent() + ? active.get().getSchema().columns() : table.schema().columns(); + for (int i = 0; i < currentColumns.size(); i++) { + positionsByFieldId.put(currentColumns.get(i).fieldId(), i); + } + } else { + for (int i = 0; i < boundTargetColumns.size(); i++) { + positionsByFieldId.put(boundTargetColumns.get(i).getUniqueId(), i); + } + } List result = new ArrayList<>(); for (SortField sortField : sortOrder.fields()) { if (!sortField.transform().isIdentity()) { continue; } - for (int i = 0; i < columns.size(); i++) { - if (columns.get(i).fieldId() == sortField.sourceId()) { - result.add(new ConnectorWriteSortColumn(i, - sortField.direction() == SortDirection.ASC, - sortField.nullOrder() == NullOrder.NULLS_FIRST)); - break; - } + Integer position = positionsByFieldId.get(sortField.sourceId()); + if (position != null) { + // Resolve against the bound field id, never a newly refreshed live ordinal; otherwise + // schema reorder can sort one output expression using another column's ordering contract. + result.add(new ConnectorWriteSortColumn(position, + sortField.direction() == SortDirection.ASC, + sortField.nullOrder() == NullOrder.NULLS_FIRST)); } } return result; } + @Override + public String getWriteMetadataIdentity(ConnectorSession session, ConnectorTableHandle tableHandle) { + return writeMetadataIdentity(resolveTable(session, (IcebergTableHandle) tableHandle)); + } + + static String writeMetadataIdentity(Table table) { + StringBuilder identity = new StringBuilder(); + // UUID and schema id make this a table-generation fence, not just a physical-layout signature. They + // reject same-name recreation and every schema commit before a stale bound output can reach the sink. + appendMetadataToken(identity, "uuid"); + appendMetadataToken(identity, tableUuid(table)); + appendMetadataToken(identity, "schema"); + appendMetadataToken(identity, table.schema().schemaId()); + appendMetadataToken(identity, "format"); + appendMetadataToken(identity, IcebergWriterHelper.getFormatVersion(table)); + SortOrder sortOrder = table.sortOrder(); + appendMetadataToken(identity, "sort"); + // Preserve a deterministic fence for partial Table implementations instead of failing schema loads. + appendMetadataToken(identity, sortOrder == null ? null : sortOrder.orderId()); + if (sortOrder != null) { + for (SortField field : sortOrder.fields()) { + appendMetadataToken(identity, field.sourceId()); + appendMetadataToken(identity, field.transform()); + appendMetadataToken(identity, field.direction()); + appendMetadataToken(identity, field.nullOrder()); + } + } + PartitionSpec spec = table.spec(); + appendMetadataToken(identity, "spec"); + appendMetadataToken(identity, spec == null ? null : spec.specId()); + if (spec != null) { + for (PartitionField field : spec.fields()) { + appendMetadataToken(identity, field.sourceId()); + appendMetadataToken(identity, field.fieldId()); + appendMetadataToken(identity, field.name()); + appendMetadataToken(identity, field.transform()); + } + } + // Admission reads these properties before physical planning, while UPDATE is later encoded as MERGE. + // Fence all three contracts so a property-only refresh cannot open an unsupported RowDelta path. + appendMetadataToken(identity, "row-level-modes"); + appendMetadataToken(identity, normalizedMode(table, TableProperties.DELETE_MODE, + TableProperties.DELETE_MODE_DEFAULT)); + appendMetadataToken(identity, normalizedMode(table, TableProperties.UPDATE_MODE, + TableProperties.UPDATE_MODE_DEFAULT)); + appendMetadataToken(identity, normalizedMode(table, TableProperties.MERGE_MODE, + TableProperties.MERGE_MODE_DEFAULT)); + return identity.toString(); + } + + private static String normalizedMode(Table table, String property, String defaultMode) { + return table.properties().getOrDefault(property, defaultMode).toLowerCase(Locale.ROOT); + } + + private static Object tableUuid(Table table) { + try { + return table.uuid(); + } catch (UnsupportedOperationException e) { + // Non-BaseTable test doubles may not expose UUID; their remaining metadata still forms a fence. + return null; + } + } + + private static void appendMetadataToken(StringBuilder identity, Object value) { + String token = String.valueOf(value); + identity.append(token.length()).append(':').append(token); + } + @Override public ConnectorWritePartitionSpec getWritePartitioning(ConnectorSession session, ConnectorTableHandle tableHandle) { @@ -390,13 +608,16 @@ private IcebergWriteContext buildWriteContext( // Carry it on the op-context so beginWrite anchors the RowDelta baseSnapshotId at S_read, keeping // the commit-time removeDeletes (option D) and BE's scan-time DV union on one snapshot. -1 (no pin) // preserves the legacy begin-time current snapshot. - long readSnapshotId = handle.getTableHandle() instanceof IcebergTableHandle - ? ((IcebergTableHandle) handle.getTableHandle()).getSnapshotId() : -1L; + IcebergTableHandle readHandle = handle.getTableHandle() instanceof IcebergTableHandle + ? (IcebergTableHandle) handle.getTableHandle() : null; + long readSnapshotId = readHandle != null ? readHandle.getSnapshotId() : -1L; + boolean readSnapshotResolved = readHandle != null && readHandle.isSnapshotResolved(); // Branch-targeted INSERT (INSERT INTO tbl@branch): the branch is threaded from the generic insert // command context onto the write handle; beginWrite validates it against the table refs and points // the commit at the branch. Empty for a default-ref write. return new IcebergWriteContext(op, handle.isOverwrite(), handle.getStaticPartitionSpec(), - handle.getBranchName(), readSnapshotId, schemaContext); + handle.getBranchName(), readSnapshotId, readSnapshotResolved, + schemaContext, handle.getBoundWriteMetadataIdentity()); } private TIcebergTableSink buildSink(Table table, IcebergTableHandle tableHandle, diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java index a1afce8ed8111d..3aa8c9ea82564b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java @@ -335,16 +335,17 @@ public void applySnapshotThreadsRef() { } @Test - public void applySnapshotLatestPinLeavesHandleUnchanged() { + public void applySnapshotRecordsExplicitlyEmptyPinWithoutScanPin() { Fixture f = fixture(); IcebergConnectorMetadata md = metadataFor(f.table, new RecordingIcebergCatalogOps()); ConnectorTableHandle bare = handle(); - // null snapshot and an empty-table (-1, no ref) pin must both read latest (handle unchanged) — a - // useSnapshot(-1) would be a non-existent snapshot. + // A null snapshot leaves the handle untouched. An empty-table pin must remain unpinned for scanning + // because useSnapshot(-1) is invalid, while retaining that resolution for write conflict validation. Assertions.assertSame(bare, md.applySnapshot(null, bare, null)); IcebergTableHandle afterMinusOne = (IcebergTableHandle) md.applySnapshot(null, bare, ConnectorMvccSnapshot.builder().snapshotId(-1L).build()); Assertions.assertFalse(afterMinusOne.hasSnapshotPin()); + Assertions.assertTrue(afterMinusOne.isSnapshotResolved()); } // --------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java index 72a1850787c884..df241b1181952c 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java @@ -19,8 +19,10 @@ import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.spi.ConnectorPartitionInfo; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.spi.pushdown.ConnectorExpression; import org.apache.iceberg.DataFiles; @@ -197,6 +199,88 @@ public void getMvccPartitionViewNullCacheEnumeratesEveryCall() { Assertions.assertEquals(2, loadCount(ops), "a null (disabled) cache must re-enumerate every call"); } + @Test + public void resolvedEmptyPartitionViewIgnoresConcurrentFirstAppend() { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db1")); + PartitionSpec spec = PartitionSpec.builderFor(PARTITIONED_SCHEMA).day("ts").build(); + Table table = catalog.createTable(TableIdentifier.of("db1", "t1"), PARTITIONED_SCHEMA, spec); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = table; + IcebergConnectorMetadata md = metadataWithMvccCache(ops, null); + ConnectorMvccSnapshot emptyPin = md.beginQuerySnapshot(null, handle()).orElseThrow(); + + table.newAppend().appendFile( + dayFile(spec, "s3://b/db1/t1/concurrent.parquet", "ts_day=1970-04-11")).commit(); + ops.table = catalog.loadTable(TableIdentifier.of("db1", "t1")); + ConnectorTableHandle resolvedEmpty = md.applySnapshot(null, handle(), emptyPin); + + ConnectorMvccPartitionView view = md.getMvccPartitionView(null, resolvedEmpty).orElseThrow(); + + // Data rows and freshness must describe the same query-begin generation after the first append. + Assertions.assertEquals(ConnectorMvccPartitionView.Style.RANGE, view.getStyle()); + Assertions.assertTrue(view.getPartitions().isEmpty()); + Assertions.assertEquals(0L, view.getNewestUpdateMonotonicMarker()); + } + + @Test + public void resolvedEmptyIdentityPartitionIgnoresConcurrentFirstAppend() { + assertResolvedEmptyNonRangeIgnoresConcurrentFirstAppend( + PartitionSpec.builderFor(PARTITIONED_SCHEMA).identity("id").build(), "id=7"); + } + + @Test + public void resolvedEmptyBucketPartitionIgnoresConcurrentFirstAppend() { + assertResolvedEmptyNonRangeIgnoresConcurrentFirstAppend( + PartitionSpec.builderFor(PARTITIONED_SCHEMA).bucket("id", 8).build(), "id_bucket=0"); + } + + @Test + public void resolvedEmptyPartitionStyleSurvivesConcurrentSpecEvolution() { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db1")); + PartitionSpec daySpec = PartitionSpec.builderFor(PARTITIONED_SCHEMA).day("ts").build(); + Table table = catalog.createTable(TableIdentifier.of("db1", "t1"), PARTITIONED_SCHEMA, daySpec); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = table; + IcebergConnectorMetadata md = metadataWithMvccCache(ops, null); + ConnectorMvccSnapshot emptyPin = md.beginQuerySnapshot(null, handle()).orElseThrow(); + + table.updateSpec().removeField("ts_day").addField("id").commit(); + ops.table = catalog.loadTable(TableIdentifier.of("db1", "t1")); + ConnectorTableHandle pinnedHandle = md.applySnapshot(null, handle(), emptyPin); + ConnectorMvccPartitionView view = md.getMvccPartitionView(null, pinnedHandle).orElseThrow(); + + Assertions.assertEquals(ConnectorMvccPartitionView.Style.RANGE, view.getStyle(), + "an empty pin must retain the partition eligibility resolved at query begin"); + Assertions.assertTrue(view.getPartitions().isEmpty()); + } + + private static void assertResolvedEmptyNonRangeIgnoresConcurrentFirstAppend( + PartitionSpec spec, String partitionPath) { + InMemoryCatalog catalog = new InMemoryCatalog(); + catalog.initialize("test", Collections.emptyMap()); + catalog.createNamespace(Namespace.of("db1")); + Table table = catalog.createTable(TableIdentifier.of("db1", "t1"), PARTITIONED_SCHEMA, spec); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = table; + IcebergConnectorMetadata md = metadataWithMvccCache(ops, null); + ConnectorMvccSnapshot emptyPin = md.beginQuerySnapshot(null, handle()).orElseThrow(); + + table.newAppend().appendFile( + dayFile(spec, "s3://b/db1/t1/concurrent.parquet", partitionPath)).commit(); + ops.table = catalog.loadTable(TableIdentifier.of("db1", "t1")); + ConnectorTableHandle pinnedHandle = md.applySnapshot(null, handle(), emptyPin); + + ConnectorMvccPartitionView view = md.getMvccPartitionView(null, pinnedHandle).orElseThrow(); + Assertions.assertEquals(ConnectorMvccPartitionView.Style.UNPARTITIONED, view.getStyle()); + Assertions.assertTrue(view.getPartitions().isEmpty()); + Assertions.assertTrue(md.listPartitions(null, pinnedHandle, Optional.empty()).isEmpty(), + "partition listing must preserve the same empty generation as the data scan"); + } + // --------------------------------------------------------------------- // listPartitions // --------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java index 8d12d340009b0d..83a2303d95ac06 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java @@ -601,6 +601,9 @@ public void getTableSchemaParsesColumnsFromLoadedTable() { Assertions.assertEquals("INT", cols.get(0).getType().getTypeName()); Assertions.assertEquals("name", cols.get(1).getName()); Assertions.assertEquals("STRING", cols.get(1).getType().getTypeName()); + Assertions.assertEquals(IcebergWritePlanProvider.writeMetadataIdentity(ops.table), + schema.getWriteMetadataIdentity(), + "the bind-time schema and write fence must be derived from the exact same table load"); // WHY: legacy IcebergUtils.parseSchema builds EVERY column with isAllowNull=true regardless of // the Iceberg field's required/optional flag (rows can still read NULL under schema-evolution diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 5f4c2d1c0713b8..77c45d0c833f55 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.handle.WriteOperation; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.spi.pushdown.ConnectorAnd; import org.apache.doris.connector.spi.pushdown.ConnectorBetween; import org.apache.doris.connector.spi.pushdown.ConnectorColumnRef; @@ -28,6 +29,7 @@ import org.apache.doris.connector.spi.pushdown.ConnectorIsNull; import org.apache.doris.connector.spi.pushdown.ConnectorLiteral; import org.apache.doris.connector.spi.pushdown.ConnectorPredicate; +import org.apache.doris.connector.spi.scan.ConnectorScanRequest; import org.apache.doris.thrift.TFileContent; import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; @@ -57,6 +59,8 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -66,6 +70,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; /** * Pins {@link IcebergConnectorTransaction}: the T03 skeleton (single SDK transaction held through the @@ -655,6 +662,26 @@ public void overwriteStaticPartitionUsesRowFilter() { Assertions.assertEquals("1", snap.summary().get("added-data-files")); } + @Test + public void overwriteStaticPartitionRejectsUnmatchedPartitionField() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("region").build(); + Table table = catalog.createTable(id, PART_SCHEMA, spec, + props("write.format.default", "parquet")); + IcebergWriteContext boundContext = overwriteStaticCtx( + table, Collections.singletonMap("region", "us")); + table.updateSpec().renameField("region", "renamed_region").commit(); + IcebergConnectorTransaction txn = txnFor( + opsReturning(catalog.loadTable(id)), new RecordingConnectorContext()); + + DorisConnectorException ex = Assertions.assertThrows( + DorisConnectorException.class, + () -> txn.beginWrite(SESSION, "db1", "t1", boundContext)); + Assertions.assertTrue(ex.getMessage().contains("partition spec changed"), + "a stale static overwrite must fail at begin before it can degrade to an always-true filter"); + } + @Test public void deleteWritesRowDeltaDeleteFiles() { InMemoryCatalog catalog = freshCatalog(); @@ -732,6 +759,38 @@ public void rollbackAndCloseAreNoOps() { }); } + @Test + public void beginWriteRejectsReplacementExposedByTransactionRefresh() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table original = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("format-version", "2")); + String originalIdentity = IcebergWritePlanProvider.writeMetadataIdentity(original); + catalog.dropTable(id, false); + Table replacement = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("format-version", "2")); + AtomicReference delegate = new AtomicReference<>(original); + Table refreshingTable = (Table) Proxy.newProxyInstance(Table.class.getClassLoader(), + new Class[] {Table.class}, (proxy, method, args) -> { + if ("newTransaction".equals(method.getName())) { + delegate.set(replacement); + } + try { + return method.invoke(delegate.get(), args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + IcebergConnectorTransaction txn = txnFor(opsReturning(refreshingTable), new RecordingConnectorContext()); + IcebergWriteContext ctx = new IcebergWriteContext(WriteOperation.INSERT, false, + Collections.emptyMap(), Optional.empty(), -1L, false, originalIdentity); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> txn.beginWrite(SESSION, "db1", "t1", ctx)); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "a replacement exposed by newTransaction refresh must not become the write baseline"); + } + // ─────────────────── commit-time conflict-detection validation suite (T05) ─────────────────── @Test @@ -758,6 +817,65 @@ public void deleteDetectsConcurrentDataFileConflict() { "a concurrent data-file append since the base snapshot must be detected as a conflict"); } + @Test + public void mergeFromResolvedEmptySnapshotRejectsConcurrentFirstAppend() throws Exception { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table empty = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + props("format-version", "2", "write.format.default", "parquet")); + RecordingIcebergCatalogOps ops = opsReturning(empty); + RecordingConnectorContext context = new RecordingConnectorContext(); + IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, Collections.emptyMap(), context); + + CountDownLatch mergeReadResolved = new CountDownLatch(1); + CountDownLatch concurrentInsertCommitted = new CountDownLatch(1); + AtomicReference insertFailure = new AtomicReference<>(); + Thread concurrentInsert = new Thread(() -> { + try { + if (!mergeReadResolved.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("MERGE read barrier was not reached"); + } + catalog.loadTable(id).newAppend() + .appendFile(dataFile(PartitionSpec.unpartitioned(), + "s3://b/db1/t1/concurrent.parquet", 1L)) + .commit(); + } catch (Throwable t) { + insertFailure.set(t); + } finally { + concurrentInsertCommitted.countDown(); + } + }, "iceberg-concurrent-first-append"); + concurrentInsert.start(); + + ConnectorMvccSnapshot snapshot = metadata.beginQuerySnapshot(null, + new IcebergTableHandle("db1", "t1")).orElseThrow(AssertionError::new); + IcebergTableHandle emptyRead = (IcebergTableHandle) metadata.applySnapshot( + null, new IcebergTableHandle("db1", "t1"), snapshot); + mergeReadResolved.countDown(); + Assertions.assertTrue(concurrentInsertCommitted.await(10, TimeUnit.SECONDS)); + concurrentInsert.join(); + Assertions.assertNull(insertFailure.get(), "the concurrent INSERT must commit at the barrier"); + + ops.table = catalog.loadTable(id); + IcebergScanPlanProvider scanProvider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + Assertions.assertTrue(scanProvider.planScan(null, + ConnectorScanRequest.builder(emptyRead, Collections.emptyList()).build()).isEmpty(), + "MERGE must keep reading the empty snapshot after the concurrent INSERT"); + + IcebergConnectorTransaction merge = txnFor(ops, context); + merge.beginWrite(SESSION, "db1", "t1", new IcebergWriteContext( + WriteOperation.MERGE, false, Collections.emptyMap(), Optional.empty(), + emptyRead.getSnapshotId(), emptyRead.isSnapshotResolved())); + merge.addCommitData(commitBytes(dataFileItem("s3://b/db1/t1/merge.parquet", 1L, 1024L))); + + Assertions.assertThrows(DorisConnectorException.class, merge::commit, + "RowDelta must validate from table creation and reject the first concurrent append"); + List committedFiles = currentDataFiles(catalog.loadTable(id)); + Assertions.assertEquals(1, committedFiles.size(), "the failed MERGE must not add a duplicate row file"); + Assertions.assertEquals("s3://b/db1/t1/concurrent.parquet", + committedFiles.get(0).path().toString()); + } + @Test public void deletePassesValidationSuiteWhenNoConcurrentChange() { InMemoryCatalog catalog = freshCatalog(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 426f89c7d427b6..80e0da6816a103 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -1417,6 +1417,97 @@ public void planScanPinnedToOlderSnapshotReadsOnlyThatSnapshotsFiles() { Assertions.assertTrue(pinned.get(0).getPath().get().endsWith("f1.parquet")); } + @Test + public void planScanResolvedEmptySnapshotIgnoresConcurrentFirstAppend() { + // MERGE may resolve its read snapshot while the target has no snapshots, then race with the first append. + // The -1 marker is a real MVCC boundary: treating it as "latest" lets MERGE miss the new row and insert a + // duplicate, so both the ordinary scan and COUNT pushdown must remain empty after the concurrent commit. + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + IcebergTableHandle emptyRead = new IcebergTableHandle("db1", "t1") + .withSnapshot(-1L, null, table.schema().schemaId()); + + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = + new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + + List rows = provider.planScan(null, + ConnectorScanRequest.builder(emptyRead, Collections.emptyList()).build()); + List count = provider.planScan(null, + ConnectorScanRequest.builder(emptyRead, Collections.emptyList()) + .requiredPartitions(Collections.emptyList()).countPushdown(true).build()); + + Assertions.assertTrue(rows.isEmpty(), "an explicitly empty read must not see the first concurrent append"); + Assertions.assertTrue(count.isEmpty(), "COUNT over an explicitly empty read must remain zero"); + } + + @Test + public void streamSplitsResolvedEmptySnapshotIgnoresConcurrentFirstAppend() throws IOException { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + IcebergTableHandle emptyRead = new IcebergTableHandle("db1", "t1") + .withSnapshot(-1L, null, table.schema().schemaId()); + IcebergScanPlanProvider provider = + new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); + + Assertions.assertEquals(1, provider.streamingSplitEstimate(batchSession(1, true), + new IcebergTableHandle("db1", "t1"), Optional.empty(), false), + "the pre-pin batch decision must observe the concurrent first append"); + List ranges = drain(provider.streamSplits(emptySession(), + emptyRead, Collections.emptyList(), Optional.empty(), -1L)); + Assertions.assertTrue(ranges.isEmpty(), + "the post-pin streaming path must preserve the explicitly empty read boundary"); + } + + @Test + public void resolvedEmptyMetadataLogEntriesStillReturnsCreationEntry() throws Exception { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + IcebergScanPlanProvider provider = + new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergTableHandle handle = IcebergTableHandle.forSystemTable( + "db1", "t1", "metadata_log_entries", -1L, null, table.schema().schemaId(), true); + + List ranges = provider.planScan(null, + ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); + + Assertions.assertEquals(1L, countSerializedSplitRows(ranges), + "metadata history exists before the first data snapshot and must not be hidden by the data fence"); + } + + @Test + public void resolvedEmptySnapshotDerivedMetadataTablesIgnoreConcurrentFirstAppend() { + for (String systemTable : Arrays.asList("snapshots", "history", "refs")) { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + IcebergTableHandle emptyRead = IcebergTableHandle.forSystemTable( + "db1", "t1", systemTable, -1L, null, table.schema().schemaId(), true); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = + new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + + Assertions.assertTrue(provider.planScan(null, + ConnectorScanRequest.builder(emptyRead, Collections.emptyList()).build()).isEmpty(), + systemTable + " must preserve the resolved-empty boundary across the first append"); + } + } + + @Test + public void resolvedEmptyAllFilesDoesNotExposeConcurrentFirstAppend() { + Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); + IcebergTableHandle emptyAllFiles = IcebergTableHandle.forSystemTable( + "db1", "t1", "all_files", -1L, null, table.schema().schemaId(), true); + table.newAppend().appendFile( + dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); + IcebergScanPlanProvider provider = + new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + + Assertions.assertTrue(provider.planScan(null, + ConnectorScanRequest.builder(emptyAllFiles, Collections.emptyList()).build()).isEmpty(), + "snapshot-derived history must preserve the empty boundary across a concurrent first append"); + } + @Test public void planScanPinnedToTagReadsViaUseRefNotSnapshotId() { // The handle carries BOTH a ref (tag1 -> S1) AND the LATEST snapshot id (s2). The scan must pin by REF diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java index cc431d2814e8c9..342c97f91d6caa 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java @@ -27,6 +27,7 @@ import org.apache.doris.connector.spi.handle.ConnectorTransaction; import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.handle.WriteOperation; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccSnapshot; import org.apache.doris.connector.spi.write.ConnectorSinkPlan; import org.apache.doris.connector.spi.write.ConnectorWritePartitionField; import org.apache.doris.connector.spi.write.ConnectorWritePartitionSpec; @@ -54,6 +55,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.NullOrder; import org.apache.iceberg.PartitionSpec; @@ -76,6 +79,7 @@ import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -225,7 +229,15 @@ private static WriteSession sessionFor(Table table, RecordingConnectorContext ct private static TIcebergTableSink planSink(Table table, RecordingConnectorContext ctx, ConnectorWriteHandle handle) { - ConnectorSinkPlan plan = providerFor(table, ctx).planWrite(sessionFor(table, ctx), handle); + return planSink(table, ctx, handle, NON_REST_PROPS); + } + + private static TIcebergTableSink planSink(Table table, RecordingConnectorContext ctx, + ConnectorWriteHandle handle, Map properties) { + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = table; + ConnectorSinkPlan plan = new IcebergWritePlanProvider(properties, ops, ctx) + .planWrite(sessionFor(table, ctx), handle); Assertions.assertEquals(TDataSinkType.ICEBERG_TABLE_SINK, plan.getDataSink().getType()); return plan.getDataSink().getIcebergTableSink(); } @@ -318,6 +330,27 @@ public void branchWritePinsBranchHeadSchema() { Assertions.assertDoesNotThrow(() -> writeContext.validateCurrentSchema(table, false)); } + @Test + public void planWriteValidatesBoundColumnsAgainstPinnedBranchSchema() { + InMemoryCatalog catalog = freshCatalog(); + Table table = unpartitionedUnsortedTable(catalog); + table.newAppend().commit(); + table.manageSnapshots().createBranch("old_schema", table.currentSnapshot().snapshotId()).commit(); + table.updateSchema().renameColumn("name", "renamed_name").commit(); + RecordingConnectorContext context = contextWithStorage(); + IcebergWritePlanProvider provider = providerFor(table, context); + WriteSession session = sessionFor(table, context); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); + List branchColumns = provider.getWriteColumns( + session, tableHandle, Optional.of("old_schema")).orElseThrow(AssertionError::new); + + ConnectorSinkPlan plan = Assertions.assertDoesNotThrow(() -> provider.planWrite(session, + new WriteHandle(tableHandle).branch("old_schema").boundTargetColumns(branchColumns))); + + Assertions.assertTrue(plan.getDataSink().getIcebergTableSink().getSchemaJson().contains("name")); + Assertions.assertFalse(plan.getDataSink().getIcebergTableSink().getSchemaJson().contains("renamed_name")); + } + @Test public void branchWriteRejectsCurrentRequiredFieldAbsentWithoutDefault() { InMemoryCatalog catalog = freshCatalog(); @@ -394,6 +427,162 @@ public void planWriteBuildsInsertSinkWithTableDerivedFields() { Assertions.assertFalse(sink.isSetStaticPartitionValues()); } + @Test + public void planWriteValidatesFullBoundSchemaForPartialInsert() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + List fullBoundSchema = Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null) + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId())); + WriteHandle handle = new WriteHandle(new IcebergTableHandle("db1", "t2")) + .columns(Collections.singletonList(fullBoundSchema.get(0))) + .boundTargetColumns(fullBoundSchema); + + TIcebergTableSink sink = planSink(table, contextWithStorage(), handle); + + Assertions.assertEquals(SchemaParser.toJson(table.schema()), sink.getSchemaJson()); + } + + @Test + public void planWriteAcceptsCanonicalEquivalentScalarDefaults() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + List boundSchema = Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT", 0, 0), "", false, null) + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING", 0, 0), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId())); + + // Doris scalar defaults are an encoding detail, not an Iceberg schema change. Rejecting this exact + // production conversion shape broke every ordinary Iceberg INSERT in external regression. + Assertions.assertDoesNotThrow(() -> planSink(table, contextWithStorage(), + new WriteHandle(new IcebergTableHandle("db1", "t2")).boundTargetColumns(boundSchema))); + } + + @Test + public void planWriteAcceptsReadFacingTopLevelNullability() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + List boundSchema = Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", true, null) + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId())); + + // Iceberg exposes required top-level fields as nullable Doris scan columns so schema-evolution + // default fill can still produce NULL. The write validator must not treat that read contract as drift. + Assertions.assertDoesNotThrow(() -> planSink(table, contextWithStorage(), + new WriteHandle(new IcebergTableHandle("db1", "t2")).boundTargetColumns(boundSchema))); + } + + @Test + public void planWriteAcceptsDorisPhysicalScalarAliases() { + Schema schema = new Schema( + Types.NestedField.required(1, "amount", Types.DecimalType.of(12, 2)), + Types.NestedField.optional(2, "event_time", Types.TimestampType.withoutZone()), + Types.NestedField.optional(3, "payload", Types.BinaryType.get())); + InMemoryCatalog catalog = freshCatalog(); + Table table = catalog.createTable(TableIdentifier.of("db1", "scalars"), schema, + PartitionSpec.unpartitioned()); + List boundSchema = Arrays.asList( + new ConnectorColumn("amount", ConnectorType.of("DECIMAL64", 12, 2), "", false, null) + .withUniqueId(1), + new ConnectorColumn("event_time", ConnectorType.of("DATETIMEV2", 6, 0), "", true, null) + .withUniqueId(2), + new ConnectorColumn("payload", ConnectorType.of("VARBINARY"), "", true, null) + .withUniqueId(3)); + + // Doris physical decimal widths and unbounded binary encoding must not look like schema evolution. + Map properties = new HashMap<>(NON_REST_PROPS); + properties.put(IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "true"); + Assertions.assertDoesNotThrow(() -> planSink(table, contextWithStorage(), + new WriteHandle(new IcebergTableHandle("db1", "scalars")).boundTargetColumns(boundSchema), + properties)); + } + + @Test + public void planWriteRejectsRecreatedNestedFieldWithSameShape() { + Schema nestedSchema = new Schema(Types.NestedField.optional(1, "payload", + Types.StructType.of(Types.NestedField.optional(2, "value", Types.FixedType.ofLength(4))))); + InMemoryCatalog catalog = freshCatalog(); + Table table = catalog.createTable(TableIdentifier.of("db1", "nested"), nestedSchema, + PartitionSpec.unpartitioned()); + ConnectorType boundType = ConnectorType.structOf( + Collections.singletonList("value"), + Collections.singletonList(ConnectorType.of("CHAR", 4, 0)), + Collections.singletonList(true), Collections.singletonList(null)) + .withChildrenFieldIds(Collections.singletonList(2)); + ConnectorColumn bound = new ConnectorColumn("payload", boundType, "", true, null) + .withUniqueId(1); + + table.updateSchema().deleteColumn("payload.value").commit(); + table.updateSchema().addColumn("payload", "value", Types.FixedType.ofLength(4)).commit(); + + // The replacement has the same name/type/ordinal but a new nested field id. Accepting it would write + // the old bound payload under a different Iceberg identity. + Assertions.assertThrows(DorisConnectorException.class, () -> planSink(table, contextWithStorage(), + new WriteHandle(new IcebergTableHandle("db1", "nested")) + .boundTargetColumns(Collections.singletonList(bound)))); + } + + @Test + public void planWriteIgnoresReservedRowLineageColumnsDuringSchemaValidation() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + List fullBoundSchema = Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null) + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId()), + new ConnectorColumn("_row_id", ConnectorType.of("BIGINT"), "", true, null) + .invisible().reservedPassthrough(), + new ConnectorColumn("_last_updated_sequence_number", ConnectorType.of("BIGINT"), "", true, null) + .invisible().reservedPassthrough()); + WriteHandle handle = new WriteHandle(new IcebergTableHandle("db1", "t2")) + .columns(Collections.singletonList(fullBoundSchema.get(0))) + .boundTargetColumns(fullBoundSchema); + + Assertions.assertDoesNotThrow(() -> planSink(table, contextWithStorage(), handle)); + } + + @Test + public void planWriteRejectsRecreatedColumnWithSameNameAndType() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + WriteHandle handle = new WriteHandle(new IcebergTableHandle("db1", "t2")).columns(Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null) + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId()))); + table.updateSchema().deleteColumn("name").commit(); + table.updateSchema().addColumn("name", Types.StringType.get()).commit(); + + Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(table, contextWithStorage(), handle)); + } + + @Test + public void planWriteRejectsSchemaReorderedAfterBinding() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + WriteHandle handle = new WriteHandle(new IcebergTableHandle("db1", "t2")).columns(Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null))); + table.updateSchema().moveFirst("name").commit(); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(table, contextWithStorage(), handle)); + Assertions.assertTrue(ex.getMessage().contains("schema changed")); + } + + @Test + public void planWriteRejectsColumnTypeChangedAfterBinding() { + Table table = unpartitionedUnsortedTable(freshCatalog()); + WriteHandle handle = new WriteHandle(new IcebergTableHandle("db1", "t2")).columns(Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null))); + table.updateSchema().updateColumn("id", Types.LongType.get()).commit(); + + Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(table, contextWithStorage(), handle)); + } + // ───────────────────────────── REWRITE: compaction sink (TIcebergTableSink) ───────────────────────────── // // WHY: post-cutover rewrite_data_files reuses the INSERT TIcebergTableSink dialect with two deltas vs @@ -432,6 +621,35 @@ public void planWriteRewriteFv3AppendsRowLineageSchema() { "fv3 rewrite schema-json must include the row-lineage _last_updated_sequence_number field"); } + @Test + public void planWriteRewriteRejectsRequestScopedRowLocatorForV2AndV3() { + assertRewriteRejectsRequestScopedRowLocator(unpartitionedUnsortedTable(freshCatalog())); + assertRewriteRejectsRequestScopedRowLocator(formatVersionThreeTable(freshCatalog())); + } + + private static void assertRewriteRejectsRequestScopedRowLocator(Table table) { + RecordingConnectorContext context = contextWithStorage(); + IcebergWritePlanProvider provider = providerFor(table, context); + WriteSession session = sessionFor(table, context); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", + IcebergWriterHelper.getFormatVersion(table) >= 3 ? "tv3" : "t2"); + List boundColumns = new ArrayList<>(boundDataColumns(table)); + if (IcebergWriterHelper.getFormatVersion(table) >= 3) { + boundColumns.add(new ConnectorColumn("_row_id", ConnectorType.of("BIGINT"), "", true, null) + .invisible().reservedPassthrough()); + boundColumns.add(new ConnectorColumn( + "_last_updated_sequence_number", ConnectorType.of("BIGINT"), "", true, null) + .invisible().reservedPassthrough()); + } + boundColumns.add(provider.getSyntheticWriteColumns(session, tableHandle).get(0)); + + DorisConnectorException exception = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, new WriteHandle(tableHandle) + .boundTargetColumns(boundColumns) + .writeOperation(WriteOperation.REWRITE))); + Assertions.assertTrue(exception.getMessage().contains("schema changed")); + } + @Test public void planWriteRewriteRejectsOverwrite() { // REWRITE is a compaction, never a user INSERT OVERWRITE; the BE writer rejects the pairing, so the @@ -734,6 +952,23 @@ public void getWriteSortColumnsForSortedTableMapsIdentityFields() { Assertions.assertTrue(cols.get(0).isNullsFirst()); } + @Test + public void getWriteSortColumnsUsesBoundFieldIdentityInsteadOfLiveOrdinal() { + Table table = partitionedSortedTable(freshCatalog()); + List reversedBoundSchema = Arrays.asList( + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId()), + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null) + .withUniqueId(table.schema().findField("id").fieldId())); + + List cols = providerFor(table, contextWithStorage()) + .getWriteSortColumns(sessionFor(table, contextWithStorage()), + new IcebergTableHandle("db1", "t1"), reversedBoundSchema); + + // Sort positions index the already-bound output. A live ordinal would incorrectly select name here. + Assertions.assertEquals(1, cols.get(0).getColumnIndex()); + } + @Test public void getWriteSortColumnsNullForUnsortedTable() { // null == "no write sort order" (legacy gates setSortInfo on isSorted()) -> the engine emits no @@ -768,6 +1003,226 @@ public void getWriteSortColumnsNonNullEmptyForSortOrderWithoutIdentityColumns() Assertions.assertTrue(cols.isEmpty(), "no identity column resolves -> empty list -> empty TSortInfo"); } + @Test + public void planWriteRejectsSortOrderEvolutionAfterPhysicalShaping() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t2"); + Table plannedTable = unpartitionedUnsortedTable(catalog); + RecordingConnectorContext ctx = contextWithStorage(); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = plannedTable; + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); + String boundMetadataIdentity = provider.getWriteMetadataIdentity(session, tableHandle); + Assertions.assertNull(provider.getWriteSortColumns(session, tableHandle), + "the physical plan must be shaped as unsorted at S0"); + + catalog.loadTable(id).replaceSortOrder().asc("name").commit(); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, + new WriteHandle(tableHandle).boundWriteMetadataIdentity(boundMetadataIdentity))); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "a post-bind sort-order change must fail before files can be labeled with the refreshed order"); + } + + @Test + public void planWriteRejectsPartitionFieldRenameAfterStaticOverwriteBinding() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "t1"); + Table plannedTable = catalog.createTable(id, SCHEMA, + PartitionSpec.builderFor(SCHEMA).identity("id").build()); + RecordingConnectorContext ctx = contextWithStorage(); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = plannedTable; + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t1"); + String boundMetadataIdentity = provider.getWriteMetadataIdentity(session, tableHandle); + + catalog.loadTable(id).updateSpec().renameField("id", "renamed_id").commit(); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, new WriteHandle(tableHandle) + .overwrite(true) + .writeContext(Collections.singletonMap("id", "7")) + .boundWriteMetadataIdentity(boundMetadataIdentity))); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "a post-bind partition-field rename must fail before the stale static spec reaches commit"); + } + + @Test + public void planWriteRejectsDeleteModeEvolution() { + assertRowLevelModeEvolutionRejected(TableProperties.DELETE_MODE, WriteOperation.DELETE); + } + + @Test + public void planWriteRejectsUpdateModeEvolutionAfterUpdateIsEncodedAsMerge() { + assertRowLevelModeEvolutionRejected(TableProperties.UPDATE_MODE, WriteOperation.MERGE); + } + + @Test + public void planWriteRejectsMergeModeEvolution() { + assertRowLevelModeEvolutionRejected(TableProperties.MERGE_MODE, WriteOperation.MERGE); + } + + private static void assertRowLevelModeEvolutionRejected(String modeProperty, WriteOperation sinkOperation) { + InMemoryCatalog catalog = freshCatalog(); + Table table = unpartitionedUnsortedTable(catalog); + RecordingConnectorContext ctx = contextWithStorage(); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = table; + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); + table.updateProperties().set(modeProperty, "merge-on-read").commit(); + String boundMetadataIdentity = provider.getWriteMetadataIdentity(session, tableHandle); + + table.updateProperties().set(modeProperty, "copy-on-write").commit(); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, new WriteHandle(tableHandle) + .boundWriteMetadataIdentity(boundMetadataIdentity) + .writeOperation(sinkOperation))); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "row-level mode changes must invalidate the generation that admitted the statement"); + } + + @Test + public void writeMetadataIdentityChangesWithFormatVersion() { + InMemoryCatalog catalog = freshCatalog(); + Table table = unpartitionedUnsortedTable(catalog); + RecordingConnectorContext ctx = contextWithStorage(); + IcebergWritePlanProvider provider = providerFor(table, ctx); + WriteSession session = sessionFor(table, ctx); + IcebergTableHandle handle = new IcebergTableHandle("db1", "t2"); + String v2Identity = provider.getWriteMetadataIdentity(session, handle); + + table.updateProperties().set(TableProperties.FORMAT_VERSION, "3").commit(); + + Assertions.assertNotEquals(v2Identity, provider.getWriteMetadataIdentity(session, handle), + "format version changes the physical write schema and must change the generation fence"); + } + + @Test + public void planWriteRejectsDropRecreateBeforeFirstWritableTableLoad() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier identifier = TableIdentifier.of("db1", "t2"); + Table original = unpartitionedUnsortedTable(catalog); + IcebergTableHandle handle = new IcebergTableHandle("db1", "t2"); + // BindSink got U0's schema and identity from one load. Construct the write provider now, but do not + // resolve its sharedWritableTable until after the same-name replacement below. + String originalIdentity = IcebergWritePlanProvider.writeMetadataIdentity(original); + RecordingConnectorContext ctx = contextWithStorage(); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = original; + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); + + catalog.dropTable(identifier, false); + Table recreated = unpartitionedUnsortedTable(catalog); + ops.table = recreated; + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, + new WriteHandle(handle).boundWriteMetadataIdentity(originalIdentity))); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "the first writable-table load must reject U1 instead of moving U0's conflict baseline"); + } + + @Test + public void planWriteRejectsWriteDefaultEvolution() { + InMemoryCatalog catalog = freshCatalog(); + Table table = unpartitionedUnsortedTable(catalog); + table.updateSchema().updateColumnDefault("id", Literal.of(42)).commit(); + List boundColumns = Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null) + .withDefaultValueSql("42") + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId())); + + table.updateSchema().updateColumnDefault("id", Literal.of(7)).commit(); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> planSink(table, contextWithStorage(), + new WriteHandle(new IcebergTableHandle("db1", "t2")) + .boundTargetColumns(boundColumns))); + Assertions.assertTrue(ex.getMessage().contains("schema changed"), + "a statement must retry instead of writing a value materialized from the stale default"); + } + + @Test + public void planWriteAcceptsPinnedRequestScopedDefaults() { + InMemoryCatalog catalog = freshCatalog(); + Table table = formatVersionThreeTable(catalog); + table.updateSchema().updateColumnDefault("id", Literal.of(42)).commit(); + RecordingConnectorContext context = contextWithStorage(); + IcebergWritePlanProvider provider = providerFor(table, context); + WriteSession session = sessionFor(table, context); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "tv3"); + List boundColumns = provider.getWriteColumns( + session, tableHandle, Optional.empty()).orElseThrow(AssertionError::new); + + TIcebergTableSink sink = Assertions.assertDoesNotThrow(() -> provider.planWrite(session, + new WriteHandle(tableHandle).boundTargetColumns(boundColumns))) + .getDataSink().getIcebergTableSink(); + + Assertions.assertTrue(sink.getSchemaJson().contains("\"write-default\":42")); + } + + @Test + public void planRewriteRejectsV2BoundSchemaAtV3Planning() { + InMemoryCatalog catalog = freshCatalog(); + Table table = unpartitionedUnsortedTable(catalog); + List v2BoundSchema = boundDataColumns(table); + table.updateProperties().set(TableProperties.FORMAT_VERSION, "3").commit(); + RecordingConnectorContext ctx = contextWithStorage(); + IcebergWritePlanProvider provider = providerFor(table, ctx); + WriteSession session = sessionFor(table, ctx); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, new WriteHandle(tableHandle) + .boundTargetColumns(v2BoundSchema) + .boundWriteMetadataIdentity(provider.getWriteMetadataIdentity(session, tableHandle)) + .writeOperation(WriteOperation.REWRITE))); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "a v3 rewrite sink must not consume output bound without row-lineage columns"); + } + + @Test + public void planMergeRejectsV2BoundSchemaAtV3Planning() { + InMemoryCatalog catalog = freshCatalog(); + Table table = unpartitionedUnsortedTable(catalog); + RecordingConnectorContext ctx = contextWithStorage(); + IcebergWritePlanProvider provider = providerFor(table, ctx); + WriteSession session = sessionFor(table, ctx); + IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); + List v2DataColumns = boundDataColumns(table); + List v2BoundSchemaWithRowId = Arrays.asList( + v2DataColumns.get(0), v2DataColumns.get(1), + provider.getSyntheticWriteColumns(session, tableHandle).get(0)); + table.updateProperties().set(TableProperties.FORMAT_VERSION, "3").commit(); + + DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, + () -> provider.planWrite(session, new WriteHandle(tableHandle) + .boundTargetColumns(v2BoundSchemaWithRowId) + .boundWriteMetadataIdentity(provider.getWriteMetadataIdentity(session, tableHandle)) + .writeOperation(WriteOperation.MERGE))); + Assertions.assertTrue(ex.getMessage().contains("write metadata changed"), + "a v3 merge sink must not advertise lineage columns absent from the bound output"); + } + + private static List boundDataColumns(Table table) { + return Arrays.asList( + new ConnectorColumn("id", ConnectorType.of("INT"), "", false, null) + .withUniqueId(table.schema().findField("id").fieldId()), + new ConnectorColumn("name", ConnectorType.of("STRING"), "", true, null) + .withUniqueId(table.schema().findField("name").fieldId())); + } + // ───────────────────────────── getWritePartitioning (connector declares, ② C3b-core) ───────────────────────────── // // WHY: post-flip the iceberg merge-write distribution (DistributionSpecMerge) is built fe-core-side, but @@ -1115,6 +1570,38 @@ public void planWriteThreadsPinnedReadSnapshotFromHandleToTransaction() { "planWrite must thread the handle's pinned read snapshot into beginWrite as baseSnapshotId"); } + @Test + public void planMergePreservesExplicitlyEmptyReadAcrossConcurrentFirstAppend() { + InMemoryCatalog catalog = freshCatalog(); + TableIdentifier id = TableIdentifier.of("db1", "tv2"); + Table empty = catalog.createTable(id, SCHEMA, PartitionSpec.unpartitioned(), + Collections.singletonMap("format-version", "2")); + RecordingConnectorContext ctx = contextWithStorage(); + RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); + ops.table = empty; + IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, Collections.emptyMap(), ctx); + ConnectorMvccSnapshot emptySnapshot = metadata.beginQuerySnapshot(null, + new IcebergTableHandle("db1", "tv2")).orElseThrow(AssertionError::new); + IcebergTableHandle emptyPinnedHandle = (IcebergTableHandle) metadata.applySnapshot( + null, new IcebergTableHandle("db1", "tv2"), emptySnapshot); + + empty.newAppend().appendFile(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("s3://bucket/db1/tv2/concurrent.parquet") + .withFileSizeInBytes(100) + .withRecordCount(1) + .withFormat(FileFormat.PARQUET) + .build()).commit(); + ops.table = catalog.loadTable(id); + Assertions.assertNotNull(ops.table.currentSnapshot(), "the concurrent append must create S1"); + + IcebergConnectorTransaction txn = new IcebergConnectorTransaction(42L, ops, ctx); + providerFor(ops.table, ctx).planWrite(new WriteSession(txn), + new WriteHandle(emptyPinnedHandle).writeOperation(WriteOperation.MERGE)); + + Assertions.assertNull(txn.getBaseSnapshotId(), + "an explicitly empty read must leave RowDelta validation unbounded across the first append"); + } + // ───────────────────────────── MERGE sink (TIcebergMergeSink) ───────────────────────────── // // WHY: UPDATE and MERGE both write the TIcebergMergeSink dialect. Two parity traps vs the table/delete @@ -1274,6 +1761,9 @@ private static final class WriteHandle implements ConnectorWriteHandle { private WriteOperation writeOperation = WriteOperation.INSERT; private Optional branchName = Optional.empty(); private boolean requireMergeCardinalityCheck; + private List columns = Collections.emptyList(); + private List boundTargetColumns; + private String boundWriteMetadataIdentity; WriteHandle(ConnectorTableHandle tableHandle) { this.tableHandle = tableHandle; @@ -1284,6 +1774,31 @@ WriteHandle branch(String v) { return this; } + WriteHandle columns(List v) { + this.columns = v; + return this; + } + + WriteHandle boundTargetColumns(List v) { + this.boundTargetColumns = v; + return this; + } + + WriteHandle boundWriteMetadataIdentity(String v) { + this.boundWriteMetadataIdentity = v; + return this; + } + + @Override + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + + @Override + public List getBoundTargetColumns() { + return boundTargetColumns == null ? columns : boundTargetColumns; + } + @Override public Optional getBranchName() { return branchName; @@ -1331,7 +1846,7 @@ public ConnectorTableHandle getTableHandle() { @Override public List getColumns() { - return Collections.emptyList(); + return columns; } @Override diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableSchema.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableSchema.java index 0fc926b01be581..b8e82dac3b3e9e 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableSchema.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableSchema.java @@ -106,13 +106,16 @@ public final class ConnectorTableSchema { private final String tableFormatType; private final Map properties; private final Set tableCapabilities; + // Opaque connector generation captured by the same remote load as columns. Write binding carries this + // value forward so a later planning load can validate rather than silently replace the bind-time baseline. + private final String writeMetadataIdentity; /** For a connector whose tables all have the same capabilities — the per-table set is empty. */ public ConnectorTableSchema(String tableName, List columns, String tableFormatType, Map properties) { - this(tableName, columns, tableFormatType, properties, Collections.emptySet()); + this(tableName, columns, tableFormatType, properties, Collections.emptySet(), null); } /** For a connector that refines its capabilities per table — see {@link #getTableCapabilities()}. */ @@ -121,6 +124,18 @@ public ConnectorTableSchema(String tableName, String tableFormatType, Map properties, Set tableCapabilities) { + this(tableName, columns, tableFormatType, properties, tableCapabilities, null); + } + + /** + * Builds a schema with the connector's opaque write generation captured from the same table load. + */ + public ConnectorTableSchema(String tableName, + List columns, + String tableFormatType, + Map properties, + Set tableCapabilities, + String writeMetadataIdentity) { this.tableName = Objects.requireNonNull(tableName, "tableName"); this.columns = columns == null ? Collections.emptyList() @@ -132,6 +147,7 @@ public ConnectorTableSchema(String tableName, this.tableCapabilities = tableCapabilities == null || tableCapabilities.isEmpty() ? Collections.emptySet() : Collections.unmodifiableSet(EnumSet.copyOf(tableCapabilities)); + this.writeMetadataIdentity = writeMetadataIdentity; } public String getTableName() { @@ -170,6 +186,11 @@ public Set getTableCapabilities() { return tableCapabilities; } + /** Opaque write generation captured together with {@link #getColumns()}, or {@code null}. */ + public String getWriteMetadataIdentity() { + return writeMetadataIdentity; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -183,12 +204,14 @@ public boolean equals(Object o) { && columns.equals(that.columns) && Objects.equals(tableFormatType, that.tableFormatType) && properties.equals(that.properties) - && tableCapabilities.equals(that.tableCapabilities); + && tableCapabilities.equals(that.tableCapabilities) + && Objects.equals(writeMetadataIdentity, that.writeMetadataIdentity); } @Override public int hashCode() { - return Objects.hash(tableName, columns, tableFormatType, properties, tableCapabilities); + return Objects.hash(tableName, columns, tableFormatType, properties, tableCapabilities, + writeMetadataIdentity); } @Override diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorWriteHandle.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorWriteHandle.java index 3987325ae56949..1855fbd75bcee3 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorWriteHandle.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorWriteHandle.java @@ -41,6 +41,18 @@ public interface ConnectorWriteHandle { /** The columns being written, ordered to match the INSERT column list. */ List getColumns(); + /** + * The complete target schema captured when this write was bound, in target-schema order. + * + *

This is deliberately separate from {@link #getColumns()}: an INSERT column list and static + * partitions can make the write list a subset even though schema-drift validation must compare the + * complete bound schema. The default preserves compatibility for handles that already carry a full + * write list.

+ */ + default List getBoundTargetColumns() { + return getColumns(); + } + /** Whether this is an INSERT OVERWRITE. */ boolean isOverwrite(); @@ -84,6 +96,11 @@ default TSortInfo getSortInfo() { return null; } + /** Metadata identity captured when the engine bound the physical write plan, or {@code null}. */ + default String getBoundWriteMetadataIdentity() { + return null; + } + /** * Whether the statement behind this write is a SQL {@code MERGE INTO} whose cardinality rule the sink * must enforce: a target row matched by more than one source row is an error, and the connector's BE diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/write/ConnectorWritePlanProvider.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/write/ConnectorWritePlanProvider.java index 9d493405cff653..27a795f22d9236 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/write/ConnectorWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/write/ConnectorWritePlanProvider.java @@ -118,6 +118,25 @@ default List getWriteSortColumns(ConnectorSession sess return null; } + /** + * Resolves write-sort positions against the bind-time target schema. Connectors with stable field + * identities should override this form; the default preserves existing name/ordinal behavior. + */ + default List getWriteSortColumns(ConnectorSession session, + ConnectorTableHandle tableHandle, List boundTargetColumns) { + return getWriteSortColumns(session, tableHandle); + } + + /** + * Returns an opaque identity for metadata that shapes the physical write plan. The engine captures it + * while binding the sink and returns it through {@link ConnectorWriteHandle}; connectors can reject the + * write if a later metadata refresh would make that physical plan stale. Default: {@code null} when the + * connector has no such metadata fence. + */ + default String getWriteMetadataIdentity(ConnectorSession session, ConnectorTableHandle tableHandle) { + return null; + } + /** * Declares the target's write-time partitioning, in an engine-neutral form, so the engine can reproduce * the connector's write distribution (the iceberg merge-write {@code DistributionSpecMerge}) without diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index cfe2fc607999af..015ba4aaa4cea6 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.spi; import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; +import org.apache.doris.connector.spi.handle.ConnectorWriteHandle; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; import org.junit.jupiter.api.Assertions; @@ -71,10 +72,9 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException Assertions.assertNotNull(in, "missing connector plugin API version resource"); version.load(in); } - // ConnectorWritePlanProvider and ConnectorColumnHandle gained public default methods in this - // surface revision. A plugin built against major 3 must be refused rather than silently run - // against an expanded contract it did not compile against. - Assertions.assertEquals("4.0", version.getProperty("api.version")); + // Write binding gained generation and full-target-schema methods in this surface revision. A plugin + // built against major 4 must be refused rather than run against a contract it did not compile against. + Assertions.assertEquals("5.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ @@ -83,6 +83,8 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException ConnectorContext.class, Connector.class, ConnectorColumnHandle.class, + ConnectorTableSchema.class, + ConnectorWriteHandle.class, ConnectorWritePlanProvider.class, org.apache.doris.extension.spi.Plugin.class, org.apache.doris.extension.spi.PluginFactory.class, diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index 7e2854758a24e1..328bde2cfd2008 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -42,14 +42,35 @@ org.apache.doris.connector.spi.ConnectorProvider#providesEventSource():boolean org.apache.doris.connector.spi.ConnectorProvider#supports(java.lang.String,java.util.Map):boolean org.apache.doris.connector.spi.ConnectorProvider#validateProperties(java.util.Map):void org.apache.doris.connector.spi.ConnectorProvider#validatePropertiesForUpdate(java.util.Map,java.util.Map):void +org.apache.doris.connector.spi.ConnectorTableSchema#equals(java.lang.Object):boolean +org.apache.doris.connector.spi.ConnectorTableSchema#getColumns():java.util.List +org.apache.doris.connector.spi.ConnectorTableSchema#getProperties():java.util.Map +org.apache.doris.connector.spi.ConnectorTableSchema#getTableCapabilities():java.util.Set +org.apache.doris.connector.spi.ConnectorTableSchema#getTableFormatType():java.lang.String +org.apache.doris.connector.spi.ConnectorTableSchema#getTableName():java.lang.String +org.apache.doris.connector.spi.ConnectorTableSchema#getWriteMetadataIdentity():java.lang.String +org.apache.doris.connector.spi.ConnectorTableSchema#hashCode():int +org.apache.doris.connector.spi.ConnectorTableSchema#toString():java.lang.String org.apache.doris.connector.spi.handle.ConnectorColumnHandle#equals(java.lang.Object):boolean org.apache.doris.connector.spi.handle.ConnectorColumnHandle#hashCode():int org.apache.doris.connector.spi.handle.ConnectorColumnHandle#withProjectedFieldIds(java.util.Set):org.apache.doris.connector.spi.handle.ConnectorColumnHandle +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getBoundTargetColumns():java.util.List +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getBoundWriteMetadataIdentity():java.lang.String +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getBranchName():java.util.Optional +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getColumns():java.util.List +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getSortInfo():org.apache.doris.thrift.TSortInfo +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getStaticPartitionSpec():java.util.Map +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getTableHandle():org.apache.doris.connector.spi.handle.ConnectorTableHandle +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#getWriteOperation():org.apache.doris.connector.spi.handle.WriteOperation +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#isOverwrite():boolean +org.apache.doris.connector.spi.handle.ConnectorWriteHandle#isRequireMergeCardinalityCheck():boolean org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#appendExplainInfo(java.lang.StringBuilder,java.lang.String,org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorWriteHandle):void org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#getSyntheticWriteColumns(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorTableHandle):java.util.List org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#getWriteColumns(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorTableHandle,java.util.Optional):java.util.Optional +org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#getWriteMetadataIdentity(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorTableHandle):java.lang.String org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#getWritePartitioning(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorTableHandle):org.apache.doris.connector.spi.write.ConnectorWritePartitionSpec org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#getWriteSortColumns(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorTableHandle):java.util.List +org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#getWriteSortColumns(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorTableHandle,java.util.List):java.util.List org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#planWrite(org.apache.doris.connector.spi.ConnectorSession,org.apache.doris.connector.spi.handle.ConnectorWriteHandle):org.apache.doris.connector.spi.write.ConnectorSinkPlan org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#requiresFullSchemaWriteOrder():boolean org.apache.doris.connector.spi.write.ConnectorWritePlanProvider#requiresMaterializeStaticPartitionValues():boolean diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 9ac781a1e38015..dc9ab08118d1f1 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all four families). --> - 4.0 + 5.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 7578685a771d64..0b0f7d67277f54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SqlModeHelper; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -89,7 +90,8 @@ public static ProcResult createResult(List schema, Set bfColumns String extraStr = StringUtils.join(extras, ","); List rowList = Lists.newArrayList(column.getDisplayName(), - column.getOriginType().hideVersionForVersionColumn(true, showNestedComment), + column.getOriginType().hideVersionForVersionColumn( + true, showNestedComment, SqlModeHelper.hasNoBackSlashEscapes()), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), column.getDefaultValue() == null diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java index a81f27720c10dc..62598eec708bef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java @@ -98,11 +98,10 @@ public static Column convertColumn(ConnectorColumn cc) { if (cc.isReservedPassthrough()) { column.setReservedPassthrough(true); } - // Stamp the nested (STRUCT/ARRAY/MAP) child column tree with the per-field ids the connector carried - // on the ConnectorType (iceberg), mirroring legacy IcebergUtils.updateIcebergColumnUniqueId's - // recursive set. The BE field-id scan path matches a pruned nested leaf by id; a -1 leaf is skipped - // and returns NULL. Inert for connectors that don't carry field ids (getChildFieldId returns -1). - applyNestedFieldIds(column, cc.getType()); + // Stamp the nested (STRUCT/ARRAY/MAP) child column tree with connector field metadata. ArrayType + // cannot retain required element semantics itself, so the child Column is the canonical round-trip + // carrier; field ids are also needed by the BE nested-field scan path. + applyNestedFieldMetadata(column, cc.getType()); return column; } @@ -111,9 +110,10 @@ public static Column convertColumn(ConnectorColumn cc) { * per-child field ids carried on {@code type} ({@link ConnectorType#getChildFieldId(int)}). The Doris * child column order built by {@code Column.createChildrenColumn} matches the {@link ConnectorType} * children order (array element / map key,value / struct fields-in-order), so a parallel walk aligns them. - * Only sets a child whose carried id is {@code >= 0}, leaving others at the default -1. + * Nullability is always copied because ARRAY/MAP type objects cannot consistently retain it. A field id is + * set only when the connector carries one ({@code >= 0}), leaving other ids at the default -1. */ - private static void applyNestedFieldIds(Column column, ConnectorType type) { + private static void applyNestedFieldMetadata(Column column, ConnectorType type) { List childColumns = column.getChildren(); if (childColumns == null || childColumns.isEmpty()) { return; @@ -122,11 +122,12 @@ private static void applyNestedFieldIds(Column column, ConnectorType type) { int n = Math.min(childColumns.size(), childTypes.size()); for (int i = 0; i < n; i++) { Column childColumn = childColumns.get(i); + childColumn.setIsAllowNull(type.isChildNullable(i)); int childFieldId = type.getChildFieldId(i); if (childFieldId >= 0) { childColumn.setUniqueId(childFieldId); } - applyNestedFieldIds(childColumn, childTypes.get(i)); + applyNestedFieldMetadata(childColumn, childTypes.get(i)); } } @@ -149,8 +150,10 @@ public static List toConnectorColumns(List columns) { * and the connector could not tell an aggregated/auto-inc column apart from a plain one.

*/ public static ConnectorColumn toConnectorColumn(Column col) { - ConnectorType connectorType = toConnectorType(col.getType()); - return new ConnectorColumn( + // Preserve the complete field-id tree when a bound write schema crosses the connector boundary. + // Iceberg field ids, rather than nested names, are the stable identity across schema evolution. + ConnectorType connectorType = toConnectorType(col); + ConnectorColumn result = new ConnectorColumn( col.getName(), connectorType, col.getComment(), @@ -162,7 +165,43 @@ public static ConnectorColumn toConnectorColumn(Column col) { // Thread the #65329 "specified" markers so a connector's nested MODIFY COLUMN can honor // omit-preserves-metadata (an omitted NULL/NOT NULL never widens the field; an omitted COMMENT // keeps the current doc). Inert for connectors / paths that don't read them. - .withSpecified(col.isNullableSpecified(), col.isCommentSpecified()); + .withSpecified(col.isNullableSpecified(), col.isCommentSpecified()) + // The connector SQL is transient write metadata, distinct from Column.defaultValue. Losing it + // on the return trip makes an unchanged remote default look like concurrent schema evolution. + .withDefaultValueSql(col.getConnectorDefaultValueSql()); + if (!col.isVisible()) { + result = result.invisible(); + } + if (col.getUniqueId() >= 0) { + result = result.withUniqueId(col.getUniqueId()); + } + if (col.isReservedPassthrough()) { + result = result.reservedPassthrough(); + } + return result; + } + + private static ConnectorType toConnectorType(Column column) { + ConnectorType type = toConnectorType(column.getType()); + List childColumns = column.getChildren(); + if (childColumns == null || childColumns.size() != type.getChildren().size()) { + return type; + } + List childTypes = new ArrayList<>(childColumns.size()); + List childIds = new ArrayList<>(childColumns.size()); + List childNullable = new ArrayList<>(childColumns.size()); + List childComments = new ArrayList<>(childColumns.size()); + List childCommentSpecified = new ArrayList<>(childColumns.size()); + for (int i = 0; i < childColumns.size(); i++) { + Column child = childColumns.get(i); + childTypes.add(toConnectorType(child)); + childIds.add(child.getUniqueId()); + childNullable.add(child.isAllowNull()); + childComments.add(type.getChildComment(i)); + childCommentSpecified.add(type.isChildCommentSpecified(i)); + } + return new ConnectorType(type.getTypeName(), type.getPrecision(), type.getScale(), childTypes, + type.getFieldNames(), childNullable, childComments, childIds, childCommentSpecified); } /** @@ -208,15 +247,28 @@ public static ConnectorType toConnectorType(Type dorisType) { // CHAR/VARCHAR store their length in `len`, not `precision`; encode it // into the ConnectorType precision field (matching convertScalarType and // the connector type convention) so CREATE TABLE requests keep the length. + if (primitiveType == PrimitiveType.VARBINARY + && scalar.getLength() == ScalarType.MAX_VARBINARY_LENGTH) { + // Doris materializes an unbounded connector VARBINARY with the maximum internal length. + // Collapse it on the reverse path so an unchanged Iceberg BINARY schema compares equal. + return ConnectorType.of(primitiveType.toString()); + } if (primitiveType == PrimitiveType.CHAR - || primitiveType == PrimitiveType.VARCHAR) { + || primitiveType == PrimitiveType.VARCHAR + || primitiveType == PrimitiveType.VARBINARY) { return ConnectorType.of(primitiveType.toString(), scalar.getLength(), 0); } - return ConnectorType.of( - primitiveType.toString(), - scalar.getScalarPrecision(), - scalar.getScalarScale()); + if (primitiveType.isDecimalV3Type() || primitiveType == PrimitiveType.DECIMALV2) { + return ConnectorType.of(primitiveType.toString(), + scalar.getScalarPrecision(), scalar.getScalarScale()); + } + if (primitiveType == PrimitiveType.DATETIMEV2 || primitiveType == PrimitiveType.TIMESTAMPTZ) { + return ConnectorType.of(primitiveType.toString(), scalar.getScalarScale(), 0); + } + // Parameterless Doris scalars expose 0/0 through ScalarType, while connector schemas use -1/-1. + // Canonicalizing them prevents an unchanged schema from being mistaken for concurrent drift. + return ConnectorType.of(primitiveType.toString()); } else { return ConnectorType.of(dorisType.toString(), -1, -1); } @@ -253,7 +305,8 @@ private static Type convertMapType(ConnectorType ct) { if (children.size() < 2) { return new MapType(Type.NULL, Type.NULL); } - return new MapType(convertType(children.get(0)), convertType(children.get(1))); + return new MapType(convertType(children.get(0)), convertType(children.get(1)), + ct.isChildNullable(0), ct.isChildNullable(1)); } private static Type convertStructType(ConnectorType ct) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTable.java index fd16f374819e96..dfb60caf43e695 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTable.java @@ -178,6 +178,11 @@ private PluginDrivenMvccSnapshot materializeLatest( metadata.getMvccPartitionView(session, pinnedHandle); if (viewOpt.isPresent()) { ConnectorMvccPartitionView view = viewOpt.get(); + if (connectorSnapshot.getSnapshotId() < 0) { + // A negative query-begin pin is the connector's resolved-empty generation. Falling through to + // a live LIST read would mix partitions committed after the data scan's empty boundary. + return buildFromRangeView(connectorSnapshot, view); + } // A non-RANGE (UNPARTITIONED) view is the connector's "not RANGE / not MTMV-range-eligible" verdict // (iceberg's range view only covers single time-transform specs). If the table nonetheless declares // partition columns (e.g. iceberg identity / bucket / truncate partitions), enumerate them via the diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index f01cb520eb2ffc..795c602e08d868 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -569,7 +569,8 @@ protected PluginDrivenSchemaCacheValue toSchemaCacheValue(ConnectorMetadata meta } } return new PluginDrivenSchemaCacheValue(columns, partitionColumns, partitionColumnRemoteNames, - tableSchema.getProperties(), tableSchema.getTableCapabilities()); + tableSchema.getProperties(), tableSchema.getTableCapabilities(), + tableSchema.getWriteMetadataIdentity()); } @Override @@ -735,6 +736,61 @@ private List appendSyntheticWriteColumns(List schema) { return result; } + /** Immutable write-facing views captured from one schema-cache generation. */ + public static final class WriteSchemaSnapshot { + private final List baseSchema; + private final List fullSchema; + private final List partitionColumns; + private final String writeMetadataIdentity; + + private WriteSchemaSnapshot(List baseSchema, List fullSchema, + List partitionColumns, String writeMetadataIdentity) { + this.baseSchema = Collections.unmodifiableList(new ArrayList<>(baseSchema)); + this.fullSchema = Collections.unmodifiableList(new ArrayList<>(fullSchema)); + this.partitionColumns = Collections.unmodifiableList(new ArrayList<>(partitionColumns)); + this.writeMetadataIdentity = writeMetadataIdentity; + } + + public List getBaseSchema() { + return baseSchema; + } + + public List getFullSchema() { + return fullSchema; + } + + public List getPartitionColumns() { + return partitionColumns; + } + + public String getWriteMetadataIdentity() { + return writeMetadataIdentity; + } + } + + /** + * Captures schema and partition identities from one cache value for write binding. Reading them through + * separate table APIs can straddle a concurrent refresh and make the planner hash an older output by a + * newer column ordinal. + */ + public WriteSchemaSnapshot getWriteSchemaSnapshot() { + makeSureInitialized(); + PluginDrivenSchemaCacheValue value = getSchemaCacheValue(Optional.empty()) + .map(PluginDrivenSchemaCacheValue.class::cast) + .orElseGet(() -> new PluginDrivenSchemaCacheValue( + Collections.emptyList(), Collections.emptyList(), Collections.emptyList())); + List baseSchema = value.getSchema(); + String writeMetadataIdentity = value.getWriteMetadataIdentity(); + ConnectContext ctx = ConnectContext.get(); + if (ctx != null && ctx.getStatementContext() != null) { + writeMetadataIdentity = ctx.getStatementContext() + .getConnectorWriteMetadataIdentity(getId()) + .orElse(writeMetadataIdentity); + } + return new WriteSchemaSnapshot(baseSchema, appendSyntheticWriteColumns(baseSchema), + value.getPartitionColumns(), writeMetadataIdentity); + } + /** * Fetches the connector's declared synthetic write columns for this table, in engine-neutral form. * Degrades to an empty list on any miss (non-plugin catalog, a read-only connector with no write-plan @@ -802,8 +858,17 @@ public Optional> resolveWriteColumns(Optional branchName) { ClassLoader previous = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(provider.getClass().getClassLoader()); - return provider.getWriteColumns(session, handle.get(), branchName) - .map(ConnectorColumnConverter::convertColumns); + Optional> connectorColumns = + provider.getWriteColumns(session, handle.get(), branchName); + if (!connectorColumns.isPresent()) { + return Optional.empty(); + } + String identity = provider.getWriteMetadataIdentity(session, handle.get()); + ConnectContext ctx = ConnectContext.get(); + if (identity != null && ctx != null && ctx.getStatementContext() != null) { + ctx.getStatementContext().setConnectorWriteMetadataIdentity(getId(), identity); + } + return connectorColumns.map(ConnectorColumnConverter::convertColumns); } finally { Thread.currentThread().setContextClassLoader(previous); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java index 725d23c2c14999..f89662035e2bd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java @@ -60,25 +60,34 @@ public class PluginDrivenSchemaCacheValue extends SchemaCacheValue { // rationale as tableProperties: the ConnectorTableSchema is transient, so the schema cache is the // carrier. Empty for every connector that does not refine per table. private final Set tableCapabilities; + private final String writeMetadataIdentity; public PluginDrivenSchemaCacheValue(List schema, List partitionColumns, List partitionColumnRemoteNames) { - this(schema, partitionColumns, partitionColumnRemoteNames, Collections.emptyMap()); + this(schema, partitionColumns, partitionColumnRemoteNames, Collections.emptyMap(), + Collections.emptySet(), null); } public PluginDrivenSchemaCacheValue(List schema, List partitionColumns, List partitionColumnRemoteNames, Map tableProperties) { - this(schema, partitionColumns, partitionColumnRemoteNames, tableProperties, Collections.emptySet()); + this(schema, partitionColumns, partitionColumnRemoteNames, tableProperties, Collections.emptySet(), null); } public PluginDrivenSchemaCacheValue(List schema, List partitionColumns, List partitionColumnRemoteNames, Map tableProperties, Set tableCapabilities) { + this(schema, partitionColumns, partitionColumnRemoteNames, tableProperties, tableCapabilities, null); + } + + public PluginDrivenSchemaCacheValue(List schema, List partitionColumns, + List partitionColumnRemoteNames, Map tableProperties, + Set tableCapabilities, String writeMetadataIdentity) { super(schema); this.partitionColumns = partitionColumns; this.partitionColumnRemoteNames = partitionColumnRemoteNames; this.tableProperties = tableProperties == null ? Collections.emptyMap() : tableProperties; this.tableCapabilities = tableCapabilities == null ? Collections.emptySet() : tableCapabilities; + this.writeMetadataIdentity = writeMetadataIdentity; } public List getPartitionColumns() { @@ -96,4 +105,8 @@ public Map getTableProperties() { public Set getTableCapabilities() { return tableCapabilities; } + + public String getWriteMetadataIdentity() { + return writeMetadataIdentity; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 2e687fa24e8284..ae37ee567fed47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -250,6 +250,9 @@ public enum TableFrom { // intentionally separate from the cached table schema used by DESCRIBE/SHOW CREATE, and cleared together // with the connector statement scope when a prepared StatementContext starts its next execution. private final Map> connectorWriteSchemas = new HashMap<>(); + // The identity is captured by the same provider call chain as connectorWriteSchemas. Keeping it alongside + // the columns prevents bind-time expressions and the later sink fence from naming different generations. + private final Map connectorWriteMetadataIdentities = new HashMap<>(); // for create view support in nereids // key is the start and end position of the sql substring that needs to be @@ -463,6 +466,14 @@ public Optional> getConnectorWriteSchema(long tableId) { return Optional.ofNullable(connectorWriteSchemas.get(tableId)); } + public void setConnectorWriteMetadataIdentity(long tableId, String identity) { + connectorWriteMetadataIdentities.put(tableId, identity); + } + + public Optional getConnectorWriteMetadataIdentity(long tableId) { + return Optional.ofNullable(connectorWriteMetadataIdentities.get(tableId)); + } + public void setTables(Map, TableIf> tables) { this.tables.clear(); this.tables.putAll(tables); @@ -696,6 +707,7 @@ public synchronized void resetConnectorStatementScope() { } this.connectorStatementScope = null; this.connectorWriteSchemas.clear(); + this.connectorWriteMetadataIdentities.clear(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 5af8be9c147bbf..099d6a1edb3a2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -555,7 +555,8 @@ public PlanFragment visitPhysicalExternalRowLevelDeleteSink( // TIcebergDeleteSink dialect. No output-expr / materialized-name loop is needed: the row id reaches // BE as the __DORIS_ICEBERG_ROWID_COL__ block column (a real hidden column), and viceberg_delete_sink // resolves it by block-name, not by output-expr name. - rootFragment.setSink(buildPluginRowLevelDmlSink(deleteSink, WriteOperation.DELETE, false)); + rootFragment.setSink(buildPluginRowLevelDmlSink(deleteSink, WriteOperation.DELETE, false, + deleteSink.getBoundWriteMetadataIdentity())); return rootFragment; } @@ -592,7 +593,7 @@ public PlanFragment visitPhysicalExternalRowLevelMergeSink( // SQL MERGE INTO carries the cardinality requirement onto the write handle; UPDATE shares this // sink dialect but has no such rule, so it threads false (see PhysicalExternalRowLevelMergeSink). rootFragment.setSink(buildPluginRowLevelDmlSink(mergeSink, WriteOperation.MERGE, - mergeSink.isRequireMergeCardinalityCheck())); + mergeSink.isRequireMergeCardinalityCheck(), mergeSink.getBoundWriteMetadataIdentity())); return rootFragment; } @@ -609,7 +610,7 @@ public PlanFragment visitPhysicalExternalRowLevelMergeSink( */ private PluginDrivenTableSink buildPluginRowLevelDmlSink( PhysicalBaseExternalTableSink sink, WriteOperation writeOperation, - boolean requireMergeCardinalityCheck) { + boolean requireMergeCardinalityCheck, String boundWriteMetadataIdentity) { PluginDrivenExternalTable targetTable = (PluginDrivenExternalTable) sink.getTargetTable(); PluginDrivenExternalCatalog catalog = (PluginDrivenExternalCatalog) targetTable.getCatalog(); @@ -621,9 +622,7 @@ private PluginDrivenTableSink buildPluginRowLevelDmlSink( // __DORIS_ICEBERG_ROWID_COL__ STRUCT, and the target may hold ARRAY/MAP/STRUCT data columns. // Naming only the tag would drop the children and yield a childless (invalid) complex type. List connectorColumns = sink.getCols().stream() - .map(col -> new ConnectorColumn(col.getName(), - ConnectorColumnConverter.toConnectorType(col.getType()), - null, col.isAllowNull(), null)) + .map(PhysicalPlanTranslator::toWriteConnectorColumn) .collect(java.util.stream.Collectors.toList()); // Resolve the table handle first so BOTH the write-admission gate and the write provider are chosen @@ -648,11 +647,11 @@ private PluginDrivenTableSink buildPluginRowLevelDmlSink( } providerTableHandle = PluginDrivenScanNode.applyMvccSnapshotPin( metadata, connSession, providerTableHandle, MvccUtil.getSnapshotFromContext(targetTable)); - // writeSortInfo == null: a row-level DML has no engine-resolved write sort (MERGE's sort lives in the // connector's TIcebergMergeSink.sort_fields, DELETE is unsorted). return new PluginDrivenTableSink(targetTable, writePlanProvider, connSession, - providerTableHandle, connectorColumns, null, writeOperation, requireMergeCardinalityCheck); + providerTableHandle, connectorColumns, connectorColumns, null, writeOperation, + requireMergeCardinalityCheck, boundWriteMetadataIdentity); } @Override @@ -676,10 +675,19 @@ public PlanFragment visitPhysicalConnectorTableSink( // converted (see the row-level DML arm): a bare primitive tag drops an ARRAY/MAP/STRUCT // column's children and yields a childless, invalid complex type. List connectorColumns = connectorTableSink.getCols().stream() - .map(col -> new ConnectorColumn(col.getName(), - ConnectorColumnConverter.toConnectorType(col.getType()), - null, col.isAllowNull(), null)) + .map(PhysicalPlanTranslator::toWriteConnectorColumn) + .collect(java.util.stream.Collectors.toList()); + List boundTargetColumns = connectorTableSink.getBoundTargetSchema().stream() + .map(PhysicalPlanTranslator::toWriteConnectorColumn) .collect(java.util.stream.Collectors.toList()); + // Sort ordinals are consumed against the sink output. BindSink puts positional writes in physical + // bound-schema order, while name-mapped writes keep user order. Preserve that coordinate space so + // partial/static INSERTs cannot sort another slot. + List boundOutputColumns = targetTable.requiresFullSchemaWriteOrder() + ? connectorTableSink.getBoundTargetSchema().stream() + .map(PhysicalPlanTranslator::toWriteConnectorColumn) + .collect(java.util.stream.Collectors.toList()) + : connectorColumns; // Every write-capable connector builds its own opaque TDataSink via its write-plan // provider (jdbc / maxcompute / iceberg). A connector whose declared write operations do @@ -712,12 +720,16 @@ public PlanFragment visitPhysicalConnectorTableSink( providerTableHandle = PluginDrivenScanNode.applyMvccSnapshotPin( metadata, connSession, providerTableHandle, MvccUtil.getSnapshotFromContext(targetTable)); + // Preserve the generation captured from the exact remote table load that supplied the bound schema. + // A live lookup here would silently move the fence after a concurrent drop/recreate. + String boundWriteMetadataIdentity = connectorTableSink.getBoundWriteMetadataIdentity(); + // The connector declares its write-sort columns (e.g. an iceberg WRITE ORDERED BY) as positions // into the sink's full-schema output; the engine resolves them to bound slots and builds the // TSortInfo here (the connector's planWrite has no bound exprs). Empty for connectors with no // write sort (jdbc/maxcompute) -> null, byte-identical unsorted sink. TSortInfo writeSortInfo = buildConnectorWriteSortInfo( - writePlanProvider.getWriteSortColumns(connSession, providerTableHandle), + writePlanProvider.getWriteSortColumns(connSession, providerTableHandle, boundOutputColumns), connectorTableSink, context); // A distributed rewrite_data_files INSERT-SELECT threads WriteOperation.REWRITE so the connector's @@ -726,13 +738,22 @@ public PlanFragment visitPhysicalConnectorTableSink( // an instanceof Iceberg. Ordinary connector INSERTs keep WriteOperation.INSERT (byte-identical). WriteOperation writeOperation = connectorTableSink.isRewrite() ? WriteOperation.REWRITE : WriteOperation.INSERT; + // The write list can omit explicit/static-partition columns, but schema-drift validation must + // retain the complete generation captured by BindSink instead of comparing that subset. PluginDrivenTableSink providerSink = new PluginDrivenTableSink(targetTable, - writePlanProvider, connSession, providerTableHandle, connectorColumns, writeSortInfo, - writeOperation); + writePlanProvider, connSession, providerTableHandle, connectorColumns, + boundTargetColumns, writeSortInfo, writeOperation, false, + boundWriteMetadataIdentity); rootFragment.setSink(providerSink); return rootFragment; } + private static ConnectorColumn toWriteConnectorColumn(Column column) { + // Use the shared recursive conversion so write validation receives nested field identities as well + // as the root id; rebuilding only the root silently accepted drop-and-recreate nested fields. + return ConnectorColumnConverter.toConnectorColumn(column); + } + private TSortInfo buildConnectorWriteSortInfo(List sortColumns, PhysicalConnectorTableSink connectorTableSink, PlanTranslatorContext context) { // null == no write sort order -> no TSortInfo (jdbc/maxcompute, unsorted iceberg). A non-null diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 47deb5da9f6791..06fe765f004a50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -665,11 +665,12 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { * *

An INSERT may read an older version of the same connector table. The no-arg external-table schema * lookup consults that statement-level ambient pin, but a sink is not that source reference and must bind - * against the latest write schema. Non-connector tables retain their existing lookup.

+ * against one coherent latest write-schema generation. Non-connector tables retain their existing lookup.

*/ private static List sinkTargetFullSchema(TableIf table) { if (table instanceof PluginDrivenExternalTable) { - return ((PluginDrivenExternalTable) table).getFullSchema(Optional.empty()); + // Schema, synthetic columns, and write identity must come from the same cache value. + return ((PluginDrivenExternalTable) table).getWriteSchemaSnapshot().getFullSchema(); } return table.getFullSchema(); } @@ -795,9 +796,10 @@ private Plan bindConnectorTableSink(MatchingContext targetWriteSchema = ctx.cascadesContext.getStatementContext() + PluginDrivenExternalTable.WriteSchemaSnapshot targetMetadata = table.getWriteSchemaSnapshot(); + List resolvedTargetSchema = ctx.cascadesContext.getStatementContext() .getConnectorWriteSchema(table.getId()) - .orElseGet(() -> sinkTargetFullSchema(table)); + .orElseGet(targetMetadata::getFullSchema); // Static-partition columns (e.g. MaxCompute `PARTITION(pt='x')`) carry their value via the // static partition spec rather than the query output, so they are excluded from the bound @@ -818,7 +820,7 @@ private Plan bindConnectorTableSink(MatchingContext targetWriteSchema = resolvedTargetSchema.stream() + .filter(column -> isConnectorSinkWriteColumn(column, sink.isRewrite())) + .collect(ImmutableList.toImmutableList()); + if (sink.isRewrite()) { + List rewriteOutputs = selectConnectorRewriteOutputs( + targetWriteSchema, child.getOutput()); + if (!rewriteOutputs.equals(child.getOutput())) { + child = new LogicalProject<>(rewriteOutputs, child); + } + } List bindColumns = selectConnectorSinkBindColumns( table, targetWriteSchema, sink.getColNames(), staticPartitionColNames, sink.isRewrite()); LogicalConnectorTableSink boundSink = new LogicalConnectorTableSink<>( database, table, + targetWriteSchema, + targetMetadata.getPartitionColumns(), + targetMetadata.getWriteMetadataIdentity(), bindColumns, child.getOutput().stream() .map(NamedExpression.class::cast) @@ -879,21 +894,8 @@ private Plan bindConnectorTableSink(MatchingContext outputWriteSchema = sink.isRewrite() - ? targetWriteSchema - : targetWriteSchema.stream() - .filter(Column::isVisible) - .collect(ImmutableList.toImmutableList()); LogicalProject fullOutputProject = - getOutputProjectByCoercion(outputWriteSchema, child, columnToOutput); + getOutputProjectByCoercion(targetWriteSchema, child, columnToOutput); return boundSink.withChildAndUpdateOutput(fullOutputProject); } // Name-mapped connector tables (JDBC / ES): keep columns in user-specified order because the @@ -914,9 +916,9 @@ private Plan bindConnectorTableSink(MatchingContext{@code staticPartitionColNames} must already be canonicalized by * {@link #canonicalStaticPartitionColNames}, so both the exclusion filter and the explicit-column-list @@ -936,7 +938,7 @@ static List selectConnectorSinkBindColumns(PluginDrivenExternalTable tab if (colNames.isEmpty()) { return targetSchema.stream() .filter(col -> !staticPartitionColNames.contains(col.getName())) - .filter(col -> isRewrite || col.isVisible()) + .filter(col -> isConnectorSinkWriteColumn(col, isRewrite)) .collect(ImmutableList.toImmutableList()); } return colNames.stream().map(cn -> { @@ -962,13 +964,7 @@ static List selectConnectorSinkBindColumns(PluginDrivenExternalTable tab throw new AnalysisException(String.format( "column %s is a static partition column, should not be in the insert column list", cn)); } - // Reject explicitly naming an engine-managed invisible column (e.g. iceberg v3 row-lineage - // _row_id / _last_updated_sequence_number) in an ordinary INSERT: the user never supplies - // its value. RETAINED for a rewrite (rewrite_data_files reads/rewrites full rows, preserving - // the engine-managed values), mirroring the isVisible/isRewrite split of the empty-colNames - // branch above. Uses only Column.isVisible(), so no source-specific code enters the generic - // SPI path (replaces the retired legacy source-specific iceberg row-lineage guard). - if (!isRewrite && !column.isVisible()) { + if (!isConnectorSinkWriteColumn(column, isRewrite)) { throw new AnalysisException(String.format( "Cannot specify invisible column '%s' in INSERT statement", cn)); } @@ -983,6 +979,33 @@ private static Column findColumn(List schema, String name) { .orElse(null); } + private static boolean isConnectorSinkWriteColumn(Column column, boolean isRewrite) { + // Hidden request-scoped scan columns are not sink fields; only connector-declared persistent + // passthrough columns may survive a rewrite and change its physical arity. + return column.isVisible() || (isRewrite && column.isReservedPassthrough()); + } + + @VisibleForTesting + static List selectConnectorRewriteOutputs( + List writeSchema, List sourceOutputs) { + Map outputByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + for (NamedExpression output : sourceOutputs) { + if (outputByName.put(output.getName(), output) != null) { + throw new AnalysisException("Duplicate column in connector rewrite source: " + output.getName()); + } + } + // Source scans can expose request-scoped hidden columns under show-hidden. Selecting by the physical + // write schema keeps source output, bound schema, and BE sink arity on one shared invariant. + return writeSchema.stream().map(column -> { + NamedExpression output = outputByName.get(column.getName()); + if (output == null) { + throw new AnalysisException("Column " + column.getName() + + " is missing from connector rewrite source"); + } + return output; + }).collect(ImmutableList.toImmutableList()); + } + /** * Build column-to-output mapping for connector table sinks. * Maps each user-specified column to the corresponding child output expression diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalConnectorTableSinkToPhysicalConnectorTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalConnectorTableSinkToPhysicalConnectorTableSink.java index d58ae8f40963fb..ff32bddfa139a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalConnectorTableSinkToPhysicalConnectorTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalConnectorTableSinkToPhysicalConnectorTableSink.java @@ -36,6 +36,9 @@ public Rule build() { return new PhysicalConnectorTableSink<>( sink.getDatabase(), sink.getTargetTable(), + sink.getBoundTargetSchema(), + sink.getBoundPartitionColumns(), + sink.getBoundWriteMetadataIdentity(), sink.getCols(), sink.getOutputExprs(), Optional.empty(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelDeleteSinkToPhysicalExternalRowLevelDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelDeleteSinkToPhysicalExternalRowLevelDeleteSink.java index eb5e662adc153e..69215b683d0a67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelDeleteSinkToPhysicalExternalRowLevelDeleteSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelDeleteSinkToPhysicalExternalRowLevelDeleteSink.java @@ -37,6 +37,7 @@ public Rule build() { return new PhysicalExternalRowLevelDeleteSink<>( sink.getDatabase(), sink.getTargetTable(), + sink.getBoundWriteMetadataIdentity(), sink.getCols(), sink.getOutputExprs(), Optional.empty(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelMergeSinkToPhysicalExternalRowLevelMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelMergeSinkToPhysicalExternalRowLevelMergeSink.java index c00d9e7fd619a6..4d84d6ebda60e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelMergeSinkToPhysicalExternalRowLevelMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalExternalRowLevelMergeSinkToPhysicalExternalRowLevelMergeSink.java @@ -37,6 +37,7 @@ public Rule build() { return new PhysicalExternalRowLevelMergeSink<>( sink.getDatabase(), sink.getTargetTable(), + sink.getBoundWriteMetadataIdentity(), sink.getCols(), sink.getOutputExprs(), sink.isRequireMergeCardinalityCheck(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java index b0ab4725908f5d..c27b5cc94c9a54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.StructField; @@ -108,7 +109,8 @@ public FunctionSignature customSignature() { // A named struct has the same value-nullability contract as struct(...); keeping // the field nullable here would reject safe casts into required target fields. structFields.add(new StructField(nameLiteral.getStringValue(), - children.get(i + 1).getDataType(), children.get(i + 1).nullable(), "")); + children.get(i + 1).getDataType(), + StructLiteral.computeFieldNullable(children.get(i + 1)), "")); } return FunctionSignature.ret(new StructType(structFields.build())) .args(children.stream().map(ExpressionTrait::getDataType).toArray(DataType[]::new)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java index 3104e490fb3414..53d1044cb7efc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteral.java @@ -158,9 +158,16 @@ public static StructType computeDataType(List fields) { ImmutableList.Builder structFields = ImmutableList.builder(); for (int i = 0; i < fields.size(); i++) { Expression field = fields.get(i); - // Preserve literal nullability so a known non-null value can satisfy a required nested field. - structFields.add(new StructField(COL_PREFIX + (i + 1), field.getDataType(), field.nullable(), "")); + structFields.add(new StructField(COL_PREFIX + (i + 1), field.getDataType(), + computeFieldNullable(field), "")); } return new StructType(structFields.build()); } + + /** Infer field nullability for struct constructors. */ + public static boolean computeFieldNullable(Expression field) { + // Strict cast changes failure behavior, not the physical result column. Preserve the cast's + // nullable type so FunctionStruct never inserts ColumnNullable into a required child column. + return field.nullable(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelDeletePlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelDeletePlanBuilder.java index b07d9faa2f7273..1decb589794a9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelDeletePlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelDeletePlanBuilder.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -85,6 +86,10 @@ public ExternalRowLevelDeletePlanBuilder( // package-visible: the generic RowLevelDmlCommand shell delegates synthesis here. LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuery, ExternalTable icebergTable) { + // The row shape and conflict fence must come from one cache generation; otherwise a replacement + // between planning and beginWrite could become the new baseline instead of aborting the stale DELETE. + PluginDrivenExternalTable.WriteSchemaSnapshot writeSchema = + ((PluginDrivenExternalTable) icebergTable).getWriteSchemaSnapshot(); LogicalPlan queryPlan = buildPositionDeletePlan(ctx, logicalQuery, icebergTable); // Convert output to NamedExpression list @@ -103,6 +108,7 @@ LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuery, LogicalExternalRowLevelDeleteSink deleteSink = new LogicalExternalRowLevelDeleteSink<>( (ExternalDatabase) icebergTable.getDatabase(), icebergTable, + writeSchema.getWriteMetadataIdentity(), ConnectorWriteSchemaUtils.pinAndGet(ctx, icebergTable), // cols outputExprs, // outputExprs Optional.empty(), // groupExpression diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilder.java index 853d2275f467cd..b3ac202bd4d330 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilder.java @@ -22,6 +22,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -394,6 +395,10 @@ private LogicalPlan buildMergeProjectPlan(ConnectContext ctx, ExternalTable iceb // package-visible: the generic RowLevelDmlCommand shell delegates synthesis here. LogicalPlan buildMergePlan(ConnectContext ctx, ExternalTable icebergTable) { LogicalPlan projectPlan = buildMergeProjectPlan(ctx, icebergTable); + // Project synthesis pins both the request-scoped writer columns and their identity. Read the snapshot + // afterwards so the sink fence cannot retain the older cached identity while carrying newer columns. + PluginDrivenExternalTable.WriteSchemaSnapshot writeSchema = + ((PluginDrivenExternalTable) icebergTable).getWriteSchemaSnapshot(); List outputExprs; if (!RowLevelDmlRowIdUtils.hasUnboundPlan(projectPlan)) { @@ -409,6 +414,7 @@ LogicalPlan buildMergePlan(ConnectContext ctx, ExternalTable icebergTable) { return new LogicalExternalRowLevelMergeSink<>( (ExternalDatabase) icebergTable.getDatabase(), icebergTable, + writeSchema.getWriteMetadataIdentity(), ConnectorWriteSchemaUtils.pinAndGet(ctx, icebergTable), outputExprs, true, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilder.java index 59686f0012171c..d0c045df9efe54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilder.java @@ -21,6 +21,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -112,6 +113,10 @@ LogicalPlan buildMergePlan(ConnectContext ctx, LogicalPlan logicalQuery, ? tableAlias : Util.getTempTableDisplayName(icebergTable.getName()); List writeColumns = ConnectorWriteSchemaUtils.pinAndGet(ctx, icebergTable); + // pinAndGet records the identity paired with these request-scoped columns. Capture the sink snapshot + // afterwards so a cached identity can never fence output bound from a newer connector generation. + PluginDrivenExternalTable.WriteSchemaSnapshot writeSchema = + ((PluginDrivenExternalTable) icebergTable).getWriteSchemaSnapshot(); List resolvedAssignments = assignments.stream() .map(assignment -> (EqualTo) assignment.withChildren(ImmutableList.of( assignment.left(), @@ -136,6 +141,7 @@ LogicalPlan buildMergePlan(ConnectContext ctx, LogicalPlan logicalQuery, return new LogicalExternalRowLevelMergeSink<>( (ExternalDatabase) icebergTable.getDatabase(), icebergTable, + writeSchema.getWriteMetadataIdentity(), writeColumns, outputExprs, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalConnectorTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalConnectorTableSink.java index d1ec0d63041bb6..396d6466beba4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalConnectorTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalConnectorTableSink.java @@ -47,6 +47,11 @@ public class LogicalConnectorTableSink extends LogicalT // bound data sink private final ExternalDatabase database; private final ExternalTable targetTable; + // Preserve the single schema generation captured by BindSink; later phases must not re-read a + // newer schema and accidentally move the baseline used for ordinal write validation. + private final List boundTargetSchema; + private final List boundPartitionColumns; + private final String boundWriteMetadataIdentity; private final DMLCommandType dmlCommandType; // Rewrite (compaction) marker, carried from UnboundConnectorTableSink.isRewrite so the physical sink // can force single-node GATHER output for a rewrite_data_files INSERT-SELECT. Part of plan identity @@ -58,6 +63,42 @@ public class LogicalConnectorTableSink extends LogicalT */ public LogicalConnectorTableSink(ExternalDatabase database, ExternalTable targetTable, + List boundTargetSchema, + List cols, + List outputExprs, + DMLCommandType dmlCommandType, + boolean rewrite, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + this(database, targetTable, boundTargetSchema, ImmutableList.of(), cols, outputExprs, + dmlCommandType, rewrite, groupExpression, logicalProperties, child); + } + + /** + * Builds a connector sink with target and partition columns captured from one schema generation. + */ + public LogicalConnectorTableSink(ExternalDatabase database, + ExternalTable targetTable, + List boundTargetSchema, + List boundPartitionColumns, + List cols, + List outputExprs, + DMLCommandType dmlCommandType, + boolean rewrite, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + this(database, targetTable, boundTargetSchema, boundPartitionColumns, null, cols, outputExprs, + dmlCommandType, rewrite, groupExpression, logicalProperties, child); + } + + /** Builds a connector sink with its opaque write generation captured by the same schema load. */ + public LogicalConnectorTableSink(ExternalDatabase database, + ExternalTable targetTable, + List boundTargetSchema, + List boundPartitionColumns, + String boundWriteMetadataIdentity, List cols, List outputExprs, DMLCommandType dmlCommandType, @@ -68,6 +109,9 @@ public LogicalConnectorTableSink(ExternalDatabase database, super(PlanType.LOGICAL_CONNECTOR_TABLE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); this.database = Objects.requireNonNull(database, "database != null in LogicalConnectorTableSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalConnectorTableSink"); + this.boundTargetSchema = ImmutableList.copyOf(boundTargetSchema); + this.boundPartitionColumns = ImmutableList.copyOf(boundPartitionColumns); + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; this.dmlCommandType = dmlCommandType; this.rewrite = rewrite; } @@ -78,7 +122,8 @@ public Plan withChildAndUpdateOutput(Plan child) { .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); return AbstractPlan.copyWithSameId(this, () -> - new LogicalConnectorTableSink<>(database, targetTable, cols, output, + new LogicalConnectorTableSink<>(database, targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, output, dmlCommandType, rewrite, Optional.empty(), Optional.empty(), child)); } @@ -86,13 +131,15 @@ public Plan withChildAndUpdateOutput(Plan child) { public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalConnectorTableSink only accepts one child"); return AbstractPlan.copyWithSameId(this, () -> - new LogicalConnectorTableSink<>(database, targetTable, cols, outputExprs, + new LogicalConnectorTableSink<>(database, targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, outputExprs, dmlCommandType, rewrite, Optional.empty(), Optional.empty(), children.get(0))); } public LogicalConnectorTableSink withOutputExprs(List outputExprs) { return AbstractPlan.copyWithSameId(this, () -> - new LogicalConnectorTableSink<>(database, targetTable, cols, outputExprs, + new LogicalConnectorTableSink<>(database, targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, outputExprs, dmlCommandType, rewrite, Optional.empty(), Optional.empty(), child())); } @@ -104,6 +151,18 @@ public ExternalTable getTargetTable() { return targetTable; } + public List getBoundTargetSchema() { + return boundTargetSchema; + } + + public List getBoundPartitionColumns() { + return boundPartitionColumns; + } + + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + public DMLCommandType getDmlCommandType() { return dmlCommandType; } @@ -127,12 +186,17 @@ public boolean equals(Object o) { return dmlCommandType == that.dmlCommandType && rewrite == that.rewrite && Objects.equals(database, that.database) - && Objects.equals(targetTable, that.targetTable) && Objects.equals(cols, that.cols); + && Objects.equals(targetTable, that.targetTable) + && Objects.equals(boundTargetSchema, that.boundTargetSchema) + && Objects.equals(boundPartitionColumns, that.boundPartitionColumns) + && Objects.equals(boundWriteMetadataIdentity, that.boundWriteMetadataIdentity) + && Objects.equals(cols, that.cols); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols, dmlCommandType, rewrite); + return Objects.hash(super.hashCode(), database, targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, dmlCommandType, rewrite); } @Override @@ -141,6 +205,8 @@ public String toString() { "outputExprs", outputExprs, "database", database.getFullName(), "targetTable", targetTable.getName(), + "boundTargetSchema", boundTargetSchema, + "boundPartitionColumns", boundPartitionColumns, "cols", cols, "dmlCommandType", dmlCommandType, "rewrite", rewrite @@ -155,7 +221,8 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return AbstractPlan.copyWithSameId(this, () -> - new LogicalConnectorTableSink<>(database, targetTable, cols, outputExprs, + new LogicalConnectorTableSink<>(database, targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, outputExprs, dmlCommandType, rewrite, groupExpression, Optional.of(getLogicalProperties()), child())); } @@ -163,7 +230,8 @@ public Plan withGroupExpression(Optional groupExpression) { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return AbstractPlan.copyWithSameId(this, () -> - new LogicalConnectorTableSink<>(database, targetTable, cols, outputExprs, + new LogicalConnectorTableSink<>(database, targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, outputExprs, dmlCommandType, rewrite, groupExpression, logicalProperties, children.get(0))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelDeleteSink.java index 2b3f28c85abbcf..18ca37bb9b6dc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelDeleteSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelDeleteSink.java @@ -45,6 +45,7 @@ public class LogicalExternalRowLevelDeleteSink extends implements Sink, PropagateFuncDeps { private final ExternalDatabase database; private final ExternalTable targetTable; + private final String boundWriteMetadataIdentity; /** * Constructor. @@ -61,31 +62,48 @@ public LogicalExternalRowLevelDeleteSink(ExternalDatabase database, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { + this(database, targetTable, null, cols, outputExprs, groupExpression, logicalProperties, child); + } + + /** Builds a row-level sink bound to the same remote generation as its target columns. */ + public LogicalExternalRowLevelDeleteSink(ExternalDatabase database, + ExternalTable targetTable, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { super(PlanType.LOGICAL_EXTERNAL_ROW_LEVEL_DELETE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); this.database = Objects.requireNonNull(database, "database != null in LogicalExternalRowLevelDeleteSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalExternalRowLevelDeleteSink"); + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; } + /** Rebuilds the sink after its child has been rewritten and adopts the child's bound output. */ public Plan withChildAndUpdateOutput(Plan child) { List output = child.getOutput().stream() .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); - return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, cols, output, + return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, output, Optional.empty(), Optional.empty(), child); } @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalExternalRowLevelDeleteSink only accepts one child"); - return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, Optional.empty(), Optional.empty(), children.get(0)); } public LogicalExternalRowLevelDeleteSink withOutputExprs(List outputExprs) { - return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, Optional.empty(), Optional.empty(), child()); } @@ -97,6 +115,10 @@ public ExternalTable getTargetTable() { return targetTable; } + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -111,12 +133,13 @@ public boolean equals(Object o) { LogicalExternalRowLevelDeleteSink that = (LogicalExternalRowLevelDeleteSink) o; return Objects.equals(database, that.database) && Objects.equals(targetTable, that.targetTable) + && Objects.equals(boundWriteMetadataIdentity, that.boundWriteMetadataIdentity) && Objects.equals(cols, that.cols); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols); + return Objects.hash(super.hashCode(), database, targetTable, boundWriteMetadataIdentity, cols); } @Override @@ -136,14 +159,16 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelDeleteSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelMergeSink.java index d2e55779ef5b48..768a284c217a86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRowLevelMergeSink.java @@ -45,6 +45,7 @@ public class LogicalExternalRowLevelMergeSink extends L implements Sink, PropagateFuncDeps { private final ExternalDatabase database; private final ExternalTable targetTable; + private final String boundWriteMetadataIdentity; // True for SQL MERGE INTO, false for UPDATE. MERGE must reject a target row matched by more than one // source row (SQL cardinality rule), which the BE sink can only do when the plan keeps the merge // distribution; UPDATE has no such rule. Read by RequestPropertyDeriver (which otherwise drops the @@ -67,12 +68,27 @@ public LogicalExternalRowLevelMergeSink(ExternalDatabase database, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { + this(database, targetTable, null, cols, outputExprs, requireMergeCardinalityCheck, + groupExpression, logicalProperties, child); + } + + /** Builds a row-level sink bound to the same remote generation as its target columns. */ + public LogicalExternalRowLevelMergeSink(ExternalDatabase database, + ExternalTable targetTable, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + boolean requireMergeCardinalityCheck, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { super(PlanType.LOGICAL_EXTERNAL_ROW_LEVEL_MERGE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); this.database = Objects.requireNonNull(database, "database != null in LogicalExternalRowLevelMergeSink"); this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalExternalRowLevelMergeSink"); + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; } @@ -80,19 +96,21 @@ public Plan withChildAndUpdateOutput(Plan child) { List output = child.getOutput().stream() .map(NamedExpression.class::cast) .collect(ImmutableList.toImmutableList()); - return new LogicalExternalRowLevelMergeSink<>(database, targetTable, cols, output, + return new LogicalExternalRowLevelMergeSink<>(database, targetTable, boundWriteMetadataIdentity, cols, output, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), child); } @Override public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "LogicalExternalRowLevelMergeSink only accepts one child"); - return new LogicalExternalRowLevelMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelMergeSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), children.get(0)); } public LogicalExternalRowLevelMergeSink withOutputExprs(List outputExprs) { - return new LogicalExternalRowLevelMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelMergeSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, requireMergeCardinalityCheck, Optional.empty(), Optional.empty(), child()); } @@ -104,6 +122,10 @@ public ExternalTable getTargetTable() { return targetTable; } + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + public boolean isRequireMergeCardinalityCheck() { return requireMergeCardinalityCheck; } @@ -122,13 +144,15 @@ public boolean equals(Object o) { LogicalExternalRowLevelMergeSink that = (LogicalExternalRowLevelMergeSink) o; return Objects.equals(database, that.database) && Objects.equals(targetTable, that.targetTable) + && Objects.equals(boundWriteMetadataIdentity, that.boundWriteMetadataIdentity) && Objects.equals(cols, that.cols) && requireMergeCardinalityCheck == that.requireMergeCardinalityCheck; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), database, targetTable, cols, requireMergeCardinalityCheck); + return Objects.hash(super.hashCode(), database, targetTable, boundWriteMetadataIdentity, cols, + requireMergeCardinalityCheck); } @Override @@ -148,14 +172,16 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { - return new LogicalExternalRowLevelMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelMergeSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, requireMergeCardinalityCheck, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalExternalRowLevelMergeSink<>(database, targetTable, cols, outputExprs, + return new LogicalExternalRowLevelMergeSink<>(database, targetTable, boundWriteMetadataIdentity, + cols, outputExprs, requireMergeCardinalityCheck, groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java index 277c6986274bf5..aac40e428b0c18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSink.java @@ -35,8 +35,11 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.statistics.Statistics; +import com.google.common.collect.ImmutableList; + import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -46,6 +49,10 @@ */ public class PhysicalConnectorTableSink extends PhysicalBaseExternalTableSink { + private final List boundTargetSchema; + private final List boundPartitionColumns; + private final String boundWriteMetadataIdentity; + // Rewrite (compaction) marker, threaded from LogicalConnectorTableSink.isRewrite. When set, // getRequirePhysicalProperties() short-circuits to GATHER (single writer) so a rewrite_data_files // INSERT-SELECT controls its output file count even on a partitioned table — the override must win @@ -58,13 +65,45 @@ public class PhysicalConnectorTableSink extends Physica */ public PhysicalConnectorTableSink(ExternalDatabase database, ExternalTable targetTable, + List boundTargetSchema, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + boolean isRewrite, + CHILD_TYPE child) { + this(database, targetTable, boundTargetSchema, ImmutableList.of(), cols, outputExprs, + groupExpression, logicalProperties, isRewrite, child); + } + + public PhysicalConnectorTableSink(ExternalDatabase database, + ExternalTable targetTable, + List boundTargetSchema, + List boundPartitionColumns, List cols, List outputExprs, Optional groupExpression, LogicalProperties logicalProperties, boolean isRewrite, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, groupExpression, logicalProperties, + this(database, targetTable, boundTargetSchema, boundPartitionColumns, null, cols, outputExprs, + groupExpression, logicalProperties, isRewrite, child); + } + + /** Builds a physical sink with the write generation captured during sink binding. */ + public PhysicalConnectorTableSink(ExternalDatabase database, + ExternalTable targetTable, + List boundTargetSchema, + List boundPartitionColumns, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + boolean isRewrite, + CHILD_TYPE child) { + this(database, targetTable, boundTargetSchema, boundPartitionColumns, boundWriteMetadataIdentity, + cols, outputExprs, groupExpression, logicalProperties, PhysicalProperties.GATHER, null, isRewrite, child); } @@ -73,6 +112,41 @@ public PhysicalConnectorTableSink(ExternalDatabase database, */ public PhysicalConnectorTableSink(ExternalDatabase database, ExternalTable targetTable, + List boundTargetSchema, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + boolean isRewrite, + CHILD_TYPE child) { + this(database, targetTable, boundTargetSchema, ImmutableList.of(), cols, outputExprs, + groupExpression, logicalProperties, physicalProperties, statistics, isRewrite, child); + } + + public PhysicalConnectorTableSink(ExternalDatabase database, + ExternalTable targetTable, + List boundTargetSchema, + List boundPartitionColumns, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + boolean isRewrite, + CHILD_TYPE child) { + this(database, targetTable, boundTargetSchema, boundPartitionColumns, null, cols, outputExprs, + groupExpression, logicalProperties, physicalProperties, statistics, isRewrite, child); + } + + /** Builds a physical sink with the write generation captured during sink binding. */ + public PhysicalConnectorTableSink(ExternalDatabase database, + ExternalTable targetTable, + List boundTargetSchema, + List boundPartitionColumns, + String boundWriteMetadataIdentity, List cols, List outputExprs, Optional groupExpression, @@ -83,14 +157,19 @@ public PhysicalConnectorTableSink(ExternalDatabase database, CHILD_TYPE child) { super(PlanType.PHYSICAL_CONNECTOR_TABLE_SINK, database, targetTable, cols, outputExprs, groupExpression, logicalProperties, physicalProperties, statistics, child); + this.boundTargetSchema = ImmutableList.copyOf(boundTargetSchema); + this.boundPartitionColumns = ImmutableList.copyOf(boundPartitionColumns); + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; this.isRewrite = isRewrite; } @Override public Plan withChildren(List children) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalConnectorTableSink<>( - (ExternalDatabase) database, (ExternalTable) targetTable, cols, outputExprs, groupExpression, - getLogicalProperties(), physicalProperties, statistics, isRewrite, children.get(0))); + (ExternalDatabase) database, (ExternalTable) targetTable, boundTargetSchema, + boundPartitionColumns, boundWriteMetadataIdentity, cols, + outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, + isRewrite, children.get(0))); } @Override @@ -101,23 +180,63 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalConnectorTableSink<>( - (ExternalDatabase) database, (ExternalTable) targetTable, cols, outputExprs, - groupExpression, getLogicalProperties(), isRewrite, child())); + (ExternalDatabase) database, (ExternalTable) targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, + outputExprs, groupExpression, getLogicalProperties(), isRewrite, child())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalConnectorTableSink<>( - (ExternalDatabase) database, (ExternalTable) targetTable, cols, outputExprs, - groupExpression, logicalProperties.get(), isRewrite, children.get(0))); + (ExternalDatabase) database, (ExternalTable) targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, + outputExprs, groupExpression, logicalProperties.get(), isRewrite, children.get(0))); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalConnectorTableSink<>( - (ExternalDatabase) database, (ExternalTable) targetTable, cols, outputExprs, - groupExpression, getLogicalProperties(), physicalProperties, statistics, isRewrite, child())); + (ExternalDatabase) database, (ExternalTable) targetTable, boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, cols, + outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, + isRewrite, child())); + } + + public List getBoundTargetSchema() { + return boundTargetSchema; + } + + public List getBoundPartitionColumns() { + return boundPartitionColumns; + } + + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + PhysicalConnectorTableSink that = (PhysicalConnectorTableSink) o; + return isRewrite == that.isRewrite + && Objects.equals(boundTargetSchema, that.boundTargetSchema) + && Objects.equals(boundPartitionColumns, that.boundPartitionColumns) + && Objects.equals(boundWriteMetadataIdentity, that.boundWriteMetadataIdentity); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), boundTargetSchema, boundPartitionColumns, + boundWriteMetadataIdentity, isRewrite); } /** @@ -152,7 +271,7 @@ public boolean isRewrite() { * {@code BindSink.bindConnectorTableSink} projects the child to full-schema order (any * unmentioned / static-partition columns filled in), exactly like legacy {@code bindMaxComputeTableSink}, * because the BE writer strips the trailing partition columns by position. So {@code child().getOutput()} - * is aligned 1:1 with {@code targetTable.getFullSchema()}, while {@code cols} excludes the static + * is aligned 1:1 with {@code boundTargetSchema}, while {@code cols} excludes the static * partition columns and may be in a different (user-specified) order. Partition columns are therefore * located by their position in the full schema. (An earlier revision indexed by {@code cols}, which * mislocated the dynamic column whenever {@code cols} order diverged from the full schema — the @@ -172,7 +291,7 @@ public PhysicalProperties getRequirePhysicalProperties() { PluginDrivenExternalTable table = (PluginDrivenExternalTable) targetTable; if (table.requirePartitionLocalSortOnWrite()) { - Set partitionNames = table.getPartitionColumns().stream() + Set partitionNames = boundPartitionColumns.stream() .map(Column::getName) .collect(Collectors.toSet()); if (!partitionNames.isEmpty()) { @@ -192,7 +311,7 @@ public PhysicalProperties getRequirePhysicalProperties() { // by the correct (dynamic) column in the partial-static case. Mirrors legacy // PhysicalMaxComputeTableSink. List columnIdx = new ArrayList<>(); - List fullSchema = targetTable.getFullSchema(); + List fullSchema = boundTargetSchema; for (int i = 0; i < fullSchema.size(); i++) { if (partitionNames.contains(fullSchema.get(i).getName())) { columnIdx.add(i); @@ -219,7 +338,7 @@ public PhysicalProperties getRequirePhysicalProperties() { } if (table.requirePartitionHashOnWrite()) { - Set partitionNames = table.getPartitionColumns().stream() + Set partitionNames = boundPartitionColumns.stream() .map(Column::getName) .collect(Collectors.toSet()); if (!partitionNames.isEmpty()) { @@ -230,7 +349,7 @@ public PhysicalProperties getRequirePhysicalProperties() { // Index by full-schema position, which is aligned 1:1 with child output because a connector // declaring requiresPartitionHashWrite also declares requiresFullSchemaWriteOrder. List columnIdx = new ArrayList<>(); - List fullSchema = targetTable.getFullSchema(); + List fullSchema = boundTargetSchema; for (int i = 0; i < fullSchema.size(); i++) { if (partitionNames.contains(fullSchema.get(i).getName())) { columnIdx.add(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelDeleteSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelDeleteSink.java index d3015501113fa7..45b8e93782d6d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelDeleteSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelDeleteSink.java @@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -45,6 +46,7 @@ */ public class PhysicalExternalRowLevelDeleteSink extends PhysicalBaseExternalTableSink { + private final String boundWriteMetadataIdentity; /** * Constructor @@ -56,10 +58,23 @@ public PhysicalExternalRowLevelDeleteSink(ExternalDatabase database, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, groupExpression, logicalProperties, + this(database, targetTable, null, cols, outputExprs, groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); } + /** Builds a row-level sink with the write generation captured during logical planning. */ + public PhysicalExternalRowLevelDeleteSink(ExternalDatabase database, + ExternalTable targetTable, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + CHILD_TYPE child) { + this(database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, groupExpression, + logicalProperties, PhysicalProperties.GATHER, null, child); + } + /** * Constructor */ @@ -72,15 +87,35 @@ public PhysicalExternalRowLevelDeleteSink(ExternalDatabase database, PhysicalProperties physicalProperties, Statistics statistics, CHILD_TYPE child) { + this(database, targetTable, null, cols, outputExprs, groupExpression, logicalProperties, + physicalProperties, statistics, child); + } + + /** Builds a row-level sink with the write generation captured during logical planning. */ + public PhysicalExternalRowLevelDeleteSink(ExternalDatabase database, + ExternalTable targetTable, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + CHILD_TYPE child) { super(PlanType.PHYSICAL_EXTERNAL_ROW_LEVEL_DELETE_SINK, database, targetTable, cols, outputExprs, groupExpression, logicalProperties, physicalProperties, statistics, child); + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; + } + + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; } @Override public Plan withChildren(List children) { return new PhysicalExternalRowLevelDeleteSink<>( database, targetTable, - cols, outputExprs, groupExpression, + boundWriteMetadataIdentity, cols, outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -92,7 +127,7 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return new PhysicalExternalRowLevelDeleteSink<>( - database, targetTable, cols, outputExprs, + database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, groupExpression, getLogicalProperties(), child()); } @@ -100,14 +135,14 @@ public Plan withGroupExpression(Optional groupExpression) { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalExternalRowLevelDeleteSink<>( - database, targetTable, cols, outputExprs, + database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalExternalRowLevelDeleteSink<>( - database, targetTable, cols, outputExprs, + database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); } @@ -119,12 +154,16 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - return super.equals(o); + if (!super.equals(o)) { + return false; + } + PhysicalExternalRowLevelDeleteSink that = (PhysicalExternalRowLevelDeleteSink) o; + return Objects.equals(boundWriteMetadataIdentity, that.boundWriteMetadataIdentity); } @Override public int hashCode() { - return super.hashCode(); + return Objects.hash(super.hashCode(), boundWriteMetadataIdentity); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSink.java index 9634c33e2458f6..292cf34fd1d80e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSink.java @@ -47,11 +47,13 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Physical Iceberg Merge Sink for UPDATE operations. @@ -59,6 +61,7 @@ */ public class PhysicalExternalRowLevelMergeSink extends PhysicalBaseExternalTableSink { + private final String boundWriteMetadataIdentity; // True for SQL MERGE INTO, false for UPDATE; see LogicalExternalRowLevelMergeSink. private final boolean requireMergeCardinalityCheck; @@ -73,10 +76,25 @@ public PhysicalExternalRowLevelMergeSink(ExternalDatabase database, Optional groupExpression, LogicalProperties logicalProperties, CHILD_TYPE child) { - this(database, targetTable, cols, outputExprs, requireMergeCardinalityCheck, + this(database, targetTable, null, cols, outputExprs, requireMergeCardinalityCheck, groupExpression, logicalProperties, PhysicalProperties.GATHER, null, child); } + /** Builds a row-level sink with the write generation captured during logical planning. */ + public PhysicalExternalRowLevelMergeSink(ExternalDatabase database, + ExternalTable targetTable, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + boolean requireMergeCardinalityCheck, + Optional groupExpression, + LogicalProperties logicalProperties, + CHILD_TYPE child) { + this(database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, + requireMergeCardinalityCheck, groupExpression, logicalProperties, + PhysicalProperties.GATHER, null, child); + } + /** * Constructor */ @@ -90,11 +108,32 @@ public PhysicalExternalRowLevelMergeSink(ExternalDatabase database, PhysicalProperties physicalProperties, Statistics statistics, CHILD_TYPE child) { + this(database, targetTable, null, cols, outputExprs, requireMergeCardinalityCheck, + groupExpression, logicalProperties, physicalProperties, statistics, child); + } + + /** Builds a row-level sink with the write generation captured during logical planning. */ + public PhysicalExternalRowLevelMergeSink(ExternalDatabase database, + ExternalTable targetTable, + String boundWriteMetadataIdentity, + List cols, + List outputExprs, + boolean requireMergeCardinalityCheck, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + CHILD_TYPE child) { super(PlanType.PHYSICAL_EXTERNAL_ROW_LEVEL_MERGE_SINK, database, targetTable, cols, outputExprs, groupExpression, logicalProperties, physicalProperties, statistics, child); + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; } + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + public boolean isRequireMergeCardinalityCheck() { return requireMergeCardinalityCheck; } @@ -103,7 +142,7 @@ public boolean isRequireMergeCardinalityCheck() { public Plan withChildren(List children) { return new PhysicalExternalRowLevelMergeSink<>( database, targetTable, - cols, outputExprs, requireMergeCardinalityCheck, groupExpression, + boundWriteMetadataIdentity, cols, outputExprs, requireMergeCardinalityCheck, groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -115,7 +154,7 @@ public R accept(PlanVisitor visitor, C context) { @Override public Plan withGroupExpression(Optional groupExpression) { return new PhysicalExternalRowLevelMergeSink<>( - database, targetTable, cols, outputExprs, + database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, requireMergeCardinalityCheck, groupExpression, getLogicalProperties(), child()); } @@ -123,14 +162,14 @@ public Plan withGroupExpression(Optional groupExpression) { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalExternalRowLevelMergeSink<>( - database, targetTable, cols, outputExprs, + database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, requireMergeCardinalityCheck, groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalExternalRowLevelMergeSink<>( - database, targetTable, cols, outputExprs, requireMergeCardinalityCheck, + database, targetTable, boundWriteMetadataIdentity, cols, outputExprs, requireMergeCardinalityCheck, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); } @@ -146,12 +185,13 @@ public boolean equals(Object o) { return false; } PhysicalExternalRowLevelMergeSink that = (PhysicalExternalRowLevelMergeSink) o; - return requireMergeCardinalityCheck == that.requireMergeCardinalityCheck; + return requireMergeCardinalityCheck == that.requireMergeCardinalityCheck + && Objects.equals(boundWriteMetadataIdentity, that.boundWriteMetadataIdentity); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), requireMergeCardinalityCheck); + return Objects.hash(super.hashCode(), boundWriteMetadataIdentity, requireMergeCardinalityCheck); } /** @@ -197,18 +237,20 @@ public PhysicalProperties getRequirePhysicalProperties() { List insertPartitionExprIds = new ArrayList<>(); List insertPartitionFields = new ArrayList<>(); Integer partitionSpecId = null; - List partitionColumns = targetTable.getPartitionColumns(Optional.empty()); Map columnExprIdMap = buildColumnExprIdMap(outputSlots, nameToExprId); + Map columnIdToExprId = buildColumnIdExprIdMap(outputSlots); boolean insertExprsOk = false; - if (!partitionColumns.isEmpty()) { - insertExprsOk = buildInsertPartitionExprIds(insertPartitionExprIds, partitionColumns, columnExprIdMap); - } InsertPartitionFieldResult fieldResult = getIcebergPartitioning( - insertPartitionFields, targetTable, columnExprIdMap); + insertPartitionFields, targetTable, columnExprIdMap, columnIdToExprId); boolean insertFieldsOk = fieldResult.success; boolean hasNonIdentity = fieldResult.hasNonIdentity; if (insertFieldsOk) { partitionSpecId = fieldResult.partitionSpecId; + insertPartitionFields.stream() + .filter(field -> "identity".equals(field.getTransform())) + .map(DistributionSpecMerge.MergePartitionField::getSourceExprId) + .forEach(insertPartitionExprIds::add); + insertExprsOk = !insertPartitionExprIds.isEmpty(); } boolean insertRandom = !(insertExprsOk || insertFieldsOk); @@ -230,20 +272,6 @@ public PhysicalProperties getRequirePhysicalProperties() { partitionSpecId)); } - private boolean buildInsertPartitionExprIds(List insertPartitionExprIds, - List partitionColumns, - Map columnExprIdMap) { - for (Column column : partitionColumns) { - ExprId exprId = columnExprIdMap.get(column.getName()); - if (exprId == null) { - insertPartitionExprIds.clear(); - return false; - } - insertPartitionExprIds.add(exprId); - } - return insertPartitionExprIds.size() == partitionColumns.size(); - } - private Map buildColumnExprIdMap(List outputSlots, Map nameToExprId) { List visibleColumns = new ArrayList<>(); @@ -263,6 +291,23 @@ private Map buildColumnExprIdMap(List outputSlots, return nameToExprId; } + private Map buildColumnIdExprIdMap(List outputSlots) { + Map result = new HashMap<>(); + List visibleColumns = cols.stream() + .filter(Column::isVisible) + .collect(Collectors.toList()); + List dataSlots = getDataSlots(outputSlots); + if (visibleColumns.size() != dataSlots.size()) { + return result; + } + for (int i = 0; i < visibleColumns.size(); i++) { + if (visibleColumns.get(i).getUniqueId() >= 0) { + result.put(visibleColumns.get(i).getUniqueId(), dataSlots.get(i).getExprId()); + } + } + return result; + } + private List getDataSlots(List outputSlots) { List dataSlots = new ArrayList<>(); for (Slot slot : outputSlots) { @@ -288,9 +333,10 @@ private List getDataSlots(List outputSlots) { private InsertPartitionFieldResult getIcebergPartitioning( List insertPartitionFields, ExternalTable table, - Map columnExprIdMap) { + Map columnExprIdMap, + Map columnIdToExprId) { return buildInsertPartitionFieldsFromConnector( - insertPartitionFields, (PluginDrivenExternalTable) table, columnExprIdMap); + insertPartitionFields, (PluginDrivenExternalTable) table, columnExprIdMap, columnIdToExprId); } /** @@ -304,7 +350,8 @@ private InsertPartitionFieldResult getIcebergPartitioning( private InsertPartitionFieldResult buildInsertPartitionFieldsFromConnector( List insertPartitionFields, PluginDrivenExternalTable table, - Map columnExprIdMap) { + Map columnExprIdMap, + Map columnIdToExprId) { PluginDrivenExternalCatalog catalog = (PluginDrivenExternalCatalog) table.getCatalog(); Connector connector = catalog.getConnector(); ConnectorSession session = catalog.buildConnectorSession(); @@ -322,7 +369,7 @@ private InsertPartitionFieldResult buildInsertPartitionFieldsFromConnector( return new InsertPartitionFieldResult(false, false, null); } ConnectorWritePartitionSpec spec = writePlanProvider.getWritePartitioning(session, handle); - return reconstructPartitionFields(insertPartitionFields, spec, columnExprIdMap); + return reconstructPartitionFields(insertPartitionFields, spec, columnExprIdMap, columnIdToExprId); } /** @@ -349,6 +396,15 @@ static InsertPartitionFieldResult reconstructPartitionFields( List insertPartitionFields, ConnectorWritePartitionSpec spec, Map columnExprIdMap) { + return reconstructPartitionFields(insertPartitionFields, spec, columnExprIdMap, + java.util.Collections.emptyMap()); + } + + static InsertPartitionFieldResult reconstructPartitionFields( + List insertPartitionFields, + ConnectorWritePartitionSpec spec, + Map columnExprIdMap, + Map columnIdToExprId) { if (spec == null) { return new InsertPartitionFieldResult(false, false, null); } @@ -366,7 +422,11 @@ static InsertPartitionFieldResult reconstructPartitionFields( insertPartitionFields.clear(); return new InsertPartitionFieldResult(false, hasNonIdentity, spec.getSpecId()); } - ExprId exprId = columnExprIdMap.get(sourceColumnName); + // Prefer the stable source field id carried by the bind-time schema. A same-name replacement + // must not inherit the old output expression after concurrent Iceberg schema evolution. + ExprId exprId = columnIdToExprId.isEmpty() + ? columnExprIdMap.get(sourceColumnName) + : columnIdToExprId.get(field.getSourceId()); if (exprId == null) { insertPartitionFields.clear(); return new InsertPartitionFieldResult(false, hasNonIdentity, spec.getSpecId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java index 5d87bc08f5f56d..0e59cc2c6c1248 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.util; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.qe.SqlModeHelper; /** @@ -89,8 +90,6 @@ public static String parseStringLiteral(String text) { * Quote a value as a STRING_LITERAL that can be parsed under the current SQL mode. */ public static String quoteStringLiteral(String value) { - String escaped = SqlModeHelper.hasNoBackSlashEscapes() - ? value : value.replace("\\", "\\\\"); - return "\"" + escaped.replace("\"", "\"\"") + "\""; + return SqlUtils.quoteStringLiteral(value, SqlModeHelper.hasNoBackSlashEscapes()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java index e9e539ea018e2f..79e53cd3f02968 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PluginDrivenTableSink.java @@ -33,6 +33,8 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TSortInfo; +import com.google.common.collect.ImmutableList; + import java.util.Collections; import java.util.EnumSet; import java.util.List; @@ -58,11 +60,14 @@ public class PluginDrivenTableSink extends BaseExternalTableDataSink { private final ConnectorSession connectorSession; private final ConnectorTableHandle tableHandle; private final List connectorColumns; + private final List boundTargetColumns; // The engine-built BE sort instruction for a connector that declares write-sort columns (iceberg // WRITE ORDERED BY); null when the target needs no write sort. The connector cannot build it (the // bound output exprs live only here), so the translator resolves the connector's declared sort // columns against the sink output and hands the TSortInfo here to thread onto the write handle. private final TSortInfo writeSortInfo; + // Opaque connector metadata generation captured before the engine shaped the physical write. + private final String boundWriteMetadataIdentity; // The DML write operation this sink performs. A plain INSERT sink keeps the default INSERT (the // connector promotes it to OVERWRITE from the handle's isOverwrite() flag); the row-level DML // translator arms (DELETE / UPDATE / MERGE) pass the operation here so the connector's planWrite @@ -121,13 +126,42 @@ public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, ConnectorTableHandle tableHandle, List connectorColumns, TSortInfo writeSortInfo, WriteOperation writeOperation, boolean requireMergeCardinalityCheck) { + this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, + connectorColumns, writeSortInfo, writeOperation, requireMergeCardinalityCheck); + } + + /** + * Plan-provider mode with the write subset and complete bind-time target schema carried separately. + */ + public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, + ConnectorTableHandle tableHandle, List connectorColumns, + List boundTargetColumns, TSortInfo writeSortInfo, + WriteOperation writeOperation, boolean requireMergeCardinalityCheck) { + this(targetTable, writePlanProvider, connectorSession, tableHandle, connectorColumns, + boundTargetColumns, writeSortInfo, writeOperation, requireMergeCardinalityCheck, null); + } + + /** + * Plan-provider mode with the connector metadata generation that shaped this physical write. + */ + public PluginDrivenTableSink(PluginDrivenExternalTable targetTable, + ConnectorWritePlanProvider writePlanProvider, ConnectorSession connectorSession, + ConnectorTableHandle tableHandle, List connectorColumns, + List boundTargetColumns, TSortInfo writeSortInfo, + WriteOperation writeOperation, boolean requireMergeCardinalityCheck, + String boundWriteMetadataIdentity) { super(); this.targetTable = targetTable; this.writePlanProvider = writePlanProvider; this.connectorSession = connectorSession; this.tableHandle = tableHandle; this.connectorColumns = connectorColumns; + // Keep this immutable bind-time snapshot distinct from the write subset. Re-reading the live + // table here would move the conflict-detection baseline and could miss ordinal schema drift. + this.boundTargetColumns = ImmutableList.copyOf(boundTargetColumns); this.writeSortInfo = writeSortInfo; + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; this.writeOperation = writeOperation == null ? WriteOperation.INSERT : writeOperation; this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; } @@ -161,8 +195,8 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) { // source-agnostic. This runs before the write plan is bound (planWrite has not run yet for an // EXPLAIN), so the connector derives the detail from the write handle. ConnectorWriteHandle handle = new PluginDrivenWriteHandle( - tableHandle, connectorColumns, false, Collections.emptyMap(), null, Optional.empty(), - writeOperation, requireMergeCardinalityCheck); + tableHandle, connectorColumns, boundTargetColumns, false, Collections.emptyMap(), null, + null, Optional.empty(), writeOperation, requireMergeCardinalityCheck); writePlanProvider.appendExplainInfo(sb, prefix, connectorSession, handle); return sb.toString(); } @@ -188,8 +222,8 @@ public void bindDataSink(Optional insertCtx) branchName = ctx.getBranchName(); } ConnectorWriteHandle handle = new PluginDrivenWriteHandle( - tableHandle, connectorColumns, overwrite, writeContext, writeSortInfo, branchName, - writeOperation, requireMergeCardinalityCheck); + tableHandle, connectorColumns, boundTargetColumns, overwrite, writeContext, writeSortInfo, + boundWriteMetadataIdentity, branchName, writeOperation, requireMergeCardinalityCheck); ConnectorSinkPlan sinkPlan = writePlanProvider.planWrite(connectorSession, handle); this.tDataSink = sinkPlan.getDataSink(); } @@ -205,22 +239,27 @@ public PluginDrivenExternalTable getTargetTable() { private static final class PluginDrivenWriteHandle implements ConnectorWriteHandle { private final ConnectorTableHandle tableHandle; private final List columns; + private final List boundTargetColumns; private final boolean overwrite; private final Map writeContext; private final TSortInfo sortInfo; + private final String boundWriteMetadataIdentity; private final Optional branchName; private final WriteOperation writeOperation; private final boolean requireMergeCardinalityCheck; private PluginDrivenWriteHandle(ConnectorTableHandle tableHandle, List columns, - boolean overwrite, Map writeContext, TSortInfo sortInfo, + List boundTargetColumns, boolean overwrite, + Map writeContext, TSortInfo sortInfo, String boundWriteMetadataIdentity, Optional branchName, WriteOperation writeOperation, boolean requireMergeCardinalityCheck) { this.tableHandle = tableHandle; this.columns = columns; + this.boundTargetColumns = boundTargetColumns; this.overwrite = overwrite; this.writeContext = writeContext; this.sortInfo = sortInfo; + this.boundWriteMetadataIdentity = boundWriteMetadataIdentity; this.branchName = branchName == null ? Optional.empty() : branchName; this.writeOperation = writeOperation == null ? WriteOperation.INSERT : writeOperation; this.requireMergeCardinalityCheck = requireMergeCardinalityCheck; @@ -236,6 +275,11 @@ public TSortInfo getSortInfo() { return sortInfo; } + @Override + public String getBoundWriteMetadataIdentity() { + return boundWriteMetadataIdentity; + } + @Override public Optional getBranchName() { return branchName; @@ -251,6 +295,11 @@ public List getColumns() { return columns; } + @Override + public List getBoundTargetColumns() { + return boundTargetColumns; + } + @Override public boolean isOverwrite() { return overwrite; diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java index 16d360d3adbb3f..c65d6795d3ed3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexSchemaProcNodeTest.java @@ -27,10 +27,13 @@ import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.qe.SqlModeHelper; import com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.List; @@ -82,9 +85,9 @@ public void testCreateResultPreservesNestedRequirednessWithAndWithoutComments() Lists.newArrayList(IndexSchemaProcNode.COMMENT_COLUMN_TITLE)) .getRows().get(0).get(1); Assert.assertTrue(typeWithComments.contains( - "required_value:int not null comment 'required-comment'")); + "required_value:int not null comment \"required-comment\"")); Assert.assertTrue(typeWithComments.contains( - "optional_value:int comment 'optional-comment'")); + "optional_value:int comment \"optional-comment\"")); String typeWithoutComments = IndexSchemaProcNode.createResult( Lists.newArrayList(column), null, Lists.newArrayList()) @@ -93,4 +96,21 @@ public void testCreateResultPreservesNestedRequirednessWithAndWithoutComments() Assert.assertFalse(typeWithoutComments.contains("required-comment")); Assert.assertFalse(typeWithoutComments.contains("optional-comment")); } + + @Test + public void testCreateResultQuotesNestedCommentsAsSqlLiterals() { + StructType structType = new StructType( + new StructField("value", Type.INT, "owner's \\path", true)); + Column column = new Column("info", structType, true, null, true, "", "top-level-comment"); + + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(false); + String displayedType = IndexSchemaProcNode.createResult( + Lists.newArrayList(column), null, + Lists.newArrayList(IndexSchemaProcNode.COMMENT_COLUMN_TITLE)) + .getRows().get(0).get(1); + + Assert.assertTrue(displayedType.contains("comment \"owner's \\\\path\"")); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java index 272e0f7557ec7e..13bac4982449ee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverterTest.java @@ -37,10 +37,28 @@ class ConnectorColumnConverterTest { + @Test + void connectorWriteDefaultSqlSurvivesColumnRoundtrip() { + ConnectorColumn connectorColumn = new ConnectorColumn( + "payload", ConnectorType.of("VARBINARY", 16, -1), "", true, null) + .withDefaultValueSql("UNHEX('000FFF')"); + + ConnectorColumn roundTripped = ConnectorColumnConverter.toConnectorColumn( + ConnectorColumnConverter.convertColumn(connectorColumn)); + + Assertions.assertNull(roundTripped.getDefaultValue()); + Assertions.assertEquals("UNHEX('000FFF')", roundTripped.getDefaultValueSql(), + "request-scoped connector defaults must survive sink-column binding"); + } + @Test void testScalarTypeRoundtrip() { // INT → ConnectorType → Doris Type should roundtrip ConnectorType ct = ConnectorColumnConverter.toConnectorType(ScalarType.INT); + // Parameterless scalars must use the connector's canonical -1/-1 representation. Iceberg publishes + // INT this way; emitting Doris' internal 0/0 defaults makes an unchanged write look like schema drift. + Assertions.assertEquals(-1, ct.getPrecision()); + Assertions.assertEquals(-1, ct.getScale()); Type back = ConnectorColumnConverter.convertType(ct); Assertions.assertEquals(ScalarType.INT, back); } @@ -382,7 +400,7 @@ void convertColumnStampsNestedFieldIdsOntoChildTree() { // them recursively). The connector carries the top-level id on ConnectorColumn.withUniqueId and the // per-child ids on ConnectorType.withChildrenFieldIds; convertColumn must stamp the whole child tree so // SlotTypeReplacer can rewrite the nested access path to ids and BE matches the pruned leaf by id (a -1 - // leaf is skipped -> NULL). MUTATION: dropping the applyNestedFieldIds call leaves children at -1 -> red. + // leaf is skipped -> NULL). MUTATION: dropping applyNestedFieldMetadata leaves children at -1 -> red. // struct, top-level field-id 3 ConnectorType structType = ConnectorType.structOf( Arrays.asList("a", "b"), @@ -424,10 +442,30 @@ void convertColumnStampsDeeplyNestedAndArrayMapFieldIds() { Assertions.assertEquals(10, mapCol.getChildren().get(1).getUniqueId(), "map value carries field-id 10"); } + @Test + void toConnectorColumnPreservesDeeplyNestedFieldIds() { + ConnectorType inner = ConnectorType.structOf( + Arrays.asList("leaf"), Arrays.asList(ConnectorType.of("INT"))) + .withChildrenFieldIds(Arrays.asList(12)); + ConnectorType array = ConnectorType.arrayOf(inner, false).withChildrenFieldIds(Arrays.asList(11)); + Column column = ConnectorColumnConverter.convertColumn( + new ConnectorColumn("payload", array, "", true, null).withUniqueId(10)); + Assertions.assertFalse(column.getChildren().get(0).isAllowNull(), + "the Doris child column must retain the connector element requiredness"); + + ConnectorColumn rebound = ConnectorColumnConverter.toConnectorColumn(column); + + // Write conflict detection needs the full identity path, not only the top-level column id. + Assertions.assertEquals(10, rebound.getUniqueId()); + Assertions.assertEquals(11, rebound.getType().getChildFieldId(0)); + Assertions.assertEquals(12, rebound.getType().getChildren().get(0).getChildFieldId(0)); + Assertions.assertFalse(rebound.getType().isChildNullable(0)); + } + @Test void convertColumnLeavesNestedUniqueIdsUnsetWithoutFieldIds() { // Regression guard: a connector that does NOT carry nested field ids (no withChildrenFieldIds, e.g. - // paimon) must leave every child uniqueId at the default -1 — applyNestedFieldIds must be inert, so a + // paimon) must leave every child uniqueId at the default -1 — field-id propagation must be inert, so a // non-iceberg connector's nested columns are never accidentally stamped. ConnectorType structType = ConnectorType.structOf( Arrays.asList("a", "b"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTableTest.java index dca9738771d194..03188415831ed4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/mvcc/PluginDrivenMvccExternalTableTest.java @@ -1253,6 +1253,18 @@ public void testRangeViewAppliesSnapshotBeforeQueryingViewOnPinnedHandle() { .listPartitions(Mockito.any(), Mockito.any(), Mockito.any()); } + @Test + public void testResolvedEmptyUnpartitionedViewSkipsLiveListFallback() { + Fixture f = Fixture.rangeView(ConnectorMvccPartitionView.unpartitioned()); + Mockito.when(f.metadata.beginQuerySnapshot(f.session, f.handle)) + .thenReturn(Optional.of(ConnectorMvccSnapshot.builder().snapshotId(-1L).build())); + + Assertions.assertTrue(f.table.getNameToPartitionItems(Optional.empty()).isEmpty(), + "a query-begin empty pin must not mix in partitions from a later LIST read"); + Mockito.verify(f.metadata, Mockito.never()) + .listPartitions(Mockito.any(), Mockito.any(), Mockito.any()); + } + @Test public void testAbsentRangeViewKeepsLegacyListPath() throws AnalysisException { // Paimon-parity guard: a connector WITHOUT a range view (getMvccPartitionView empty) keeps the legacy diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTablePartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTablePartitionTest.java index 29c7b1f5cb5917..34f0c0033cf869 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTablePartitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTablePartitionTest.java @@ -186,7 +186,8 @@ public void testInitSchemaExtractsPartitionColumnsMappingRemoteNames() { new ConnectorColumn("REGION", ConnectorType.of("INT"), "", true, null), new ConnectorColumn("VAL", ConnectorType.of("INT"), "", true, null)), "max_compute", - Collections.singletonMap(ConnectorTableSchema.PARTITION_COLUMNS_KEY, "YEAR,REGION")); + Collections.singletonMap(ConnectorTableSchema.PARTITION_COLUMNS_KEY, "YEAR,REGION"), + Collections.emptySet(), "uuid-u0/schema-1"); Mockito.when(metadata.getTableSchema(session, handle)).thenReturn(tableSchema); // Identifier mapping lowercases the remote names (raw "YEAR" -> mapped "year"). Mockito.when(metadata.fromRemoteColumnName(Mockito.eq(session), Mockito.anyString(), @@ -206,6 +207,8 @@ public void testInitSchemaExtractsPartitionColumnsMappingRemoteNames() { "partition columns must be the MAPPED Doris columns identified via fromRemoteColumnName"); Assertions.assertEquals(Arrays.asList("YEAR", "REGION"), value.getPartitionColumnRemoteNames(), "remote names must be kept raw for addressing connector partition values"); + Assertions.assertEquals("uuid-u0/schema-1", value.getWriteMetadataIdentity(), + "schema caching must preserve the opaque generation captured with the remote columns"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java index 4ecaee7a5d42bf..58c975046b0d60 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java @@ -40,6 +40,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.PluginDrivenMvccSnapshot; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -412,6 +413,8 @@ public void resolveWriteColumnsRunsAndRestoresPluginContextClassLoader() { observed.set(Thread.currentThread().getContextClassLoader()); return Optional.of(Collections.emptyList()); }); + Mockito.when(provider.getWriteMetadataIdentity(Mockito.any(), Mockito.any())) + .thenReturn("pinned-generation"); ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); Mockito.when(metadata.getTableHandle(Mockito.any(), Mockito.any(), Mockito.any())) @@ -426,6 +429,10 @@ public void resolveWriteColumnsRunsAndRestoresPluginContextClassLoader() { PluginDrivenExternalTable table = Mockito.mock( PluginDrivenExternalTable.class, Mockito.CALLS_REAL_METHODS); Deencapsulation.setField(table, "catalog", catalog); + Mockito.doReturn(99L).when(table).getId(); + ConnectContext ctx = new ConnectContext(); + ctx.setStatementContext(new StatementContext()); + ctx.setThreadLocalInfo(); ClassLoader original = Thread.currentThread().getContextClassLoader(); ClassLoader previous = new ClassLoader(null) { }; @@ -433,9 +440,38 @@ public void resolveWriteColumnsRunsAndRestoresPluginContextClassLoader() { try { Assertions.assertTrue(table.resolveWriteColumns(Optional.empty()).isPresent()); Assertions.assertSame(provider.getClass().getClassLoader(), observed.get()); + Assertions.assertEquals("pinned-generation", ctx.getStatementContext() + .getConnectorWriteMetadataIdentity(99L).orElse(null)); Assertions.assertSame(previous, Thread.currentThread().getContextClassLoader()); } finally { Thread.currentThread().setContextClassLoader(original); + ConnectContext.remove(); + } + } + + @Test + public void writeSchemaSnapshotUsesStatementPinnedMetadataIdentity() { + PluginDrivenSchemaCacheValue cached = Mockito.mock(PluginDrivenSchemaCacheValue.class); + Mockito.when(cached.getSchema()).thenReturn(BASE_SCHEMA); + Mockito.when(cached.getPartitionColumns()).thenReturn(Collections.emptyList()); + Mockito.when(cached.getWriteMetadataIdentity()).thenReturn("cached-generation"); + PluginDrivenExternalTable table = Mockito.mock( + PluginDrivenExternalTable.class, Mockito.CALLS_REAL_METHODS); + Mockito.doNothing().when(table).makeSureInitialized(); + Mockito.doReturn(Optional.of(cached)).when(table).getSchemaCacheValue(Optional.empty()); + Mockito.doReturn(99L).when(table).getId(); + ConnectContext ctx = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + statementContext.setConnectorWriteMetadataIdentity(99L, "pinned-generation"); + ctx.setStatementContext(statementContext); + ctx.setThreadLocalInfo(); + + try { + Assertions.assertEquals("pinned-generation", + table.getWriteSchemaSnapshot().getWriteMetadataIdentity(), + "the sink fence must use the identity captured with its statement-pinned columns"); + } finally { + ConnectContext.remove(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java index 1b1b1f06d858e5..53d259bd341463 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorAdmissionGateTest.java @@ -17,25 +17,33 @@ package org.apache.doris.nereids.glue.translator; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorColumn; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorStatementScope; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.handle.WriteOperation; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.connector.spi.write.ConnectorWriteSortColumn; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalConnectorTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalExternalRowLevelDeleteSink; +import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.planner.DataSink; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PluginDrivenTableSink; +import org.apache.doris.thrift.TSortInfo; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; @@ -44,6 +52,7 @@ import org.mockito.Mockito; import java.util.EnumSet; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -59,6 +68,9 @@ public class PhysicalPlanTranslatorAdmissionGateTest { private static final Column DATA = new Column("data", PrimitiveType.INT); + private static final Column A = new Column("a", PrimitiveType.INT); + private static final Column B = new Column("b", PrimitiveType.INT); + private static final Column C = new Column("c", PrimitiveType.INT); @Test public void insertGateAllowsConnectorDeclaringInsert() { @@ -129,6 +141,68 @@ public void rowLevelDmlGateRejectsConnectorDeclaringOnlyInsertWithDistinctMessag + ex.getMessage()); } + @Test + public void rowLevelDmlPreservesBindTimeWriteMetadataIdentity() { + PlanTranslatorContext context = new PlanTranslatorContext(); + PlanFragment childFragment = Mockito.mock(PlanFragment.class); + ConnectorWritePlanProvider provider = Mockito.mock(ConnectorWritePlanProvider.class); + PluginDrivenExternalTable table = pluginTable(EnumSet.of(WriteOperation.DELETE), provider); + + @SuppressWarnings("unchecked") + PhysicalExternalRowLevelDeleteSink sink = Mockito.mock(PhysicalExternalRowLevelDeleteSink.class); + Mockito.doReturn(mockChild(childFragment)).when(sink).child(); + Mockito.doReturn(table).when(sink).getTargetTable(); + Mockito.doReturn(ImmutableList.of(DATA)).when(sink).getCols(); + Mockito.doReturn("uuid-u0/schema-1").when(sink).getBoundWriteMetadataIdentity(); + + new PhysicalPlanTranslator(context, null).visitPhysicalExternalRowLevelDeleteSink(sink, context); + + PluginDrivenTableSink pluginSink = capturePluginSink(childFragment); + Assertions.assertEquals("uuid-u0/schema-1", + Deencapsulation.getField(pluginSink, "boundWriteMetadataIdentity")); + // Re-reading at translation time would accept a replacement table as this write's conflict baseline. + Mockito.verify(provider, Mockito.never()).getWriteMetadataIdentity(Mockito.any(), Mockito.any()); + } + + @Test + public void partialInsertResolvesWriteSortAgainstFullOutput() { + assertWriteSortUsesBoundOutputColumn(ImmutableList.of(C), C); + } + + @Test + public void reorderedInsertResolvesWriteSortAgainstFullOutput() { + assertWriteSortUsesBoundOutputColumn(ImmutableList.of(C, A), C); + } + + @Test + public void staticPartitionInsertKeepsSortColumnInFullOutput() { + assertWriteSortUsesBoundOutputColumn(ImmutableList.of(A, C), B); + } + + @Test + public void insertPreservesBindTimeWriteMetadataIdentity() { + PlanTranslatorContext context = new PlanTranslatorContext(); + PlanFragment childFragment = Mockito.mock(PlanFragment.class); + ConnectorWritePlanProvider provider = Mockito.mock(ConnectorWritePlanProvider.class); + PluginDrivenExternalTable table = pluginTable(EnumSet.of(WriteOperation.INSERT), provider); + + @SuppressWarnings("unchecked") + PhysicalConnectorTableSink sink = Mockito.mock(PhysicalConnectorTableSink.class); + Mockito.doReturn(mockChild(childFragment)).when(sink).child(); + Mockito.doReturn(table).when(sink).getTargetTable(); + Mockito.doReturn(ImmutableList.of(DATA)).when(sink).getCols(); + Mockito.doReturn("uuid-u0/schema-1").when(sink).getBoundWriteMetadataIdentity(); + Mockito.doReturn(false).when(sink).isRewrite(); + + new PhysicalPlanTranslator(context, null).visitPhysicalConnectorTableSink(sink, context); + + PluginDrivenTableSink pluginSink = capturePluginSink(childFragment); + Assertions.assertEquals("uuid-u0/schema-1", + Deencapsulation.getField(pluginSink, "boundWriteMetadataIdentity")); + // A translator-time refresh must not move the fence away from the schema used by BindSink. + Mockito.verify(provider, Mockito.never()).getWriteMetadataIdentity(Mockito.any(), Mockito.any()); + } + // ==================== helpers ==================== private static Plan mockChild(PlanFragment childFragment) { @@ -137,15 +211,75 @@ private static Plan mockChild(PlanFragment childFragment) { return child; } + private static void assertWriteSortUsesBoundOutputColumn(List writeColumns, Column sortColumn) { + PlanTranslatorContext context = new PlanTranslatorContext(); + PlanFragment childFragment = Mockito.mock(PlanFragment.class); + ConnectorWritePlanProvider provider = Mockito.mock(ConnectorWritePlanProvider.class); + Mockito.when(provider.getWriteSortColumns(Mockito.any(), Mockito.any(), Mockito.anyList())) + .thenAnswer(invocation -> { + List columns = invocation.getArgument(2); + for (int i = 0; i < columns.size(); i++) { + if (columns.get(i).getName().equals(sortColumn.getName())) { + return ImmutableList.of(new ConnectorWriteSortColumn(i, true, true)); + } + } + return ImmutableList.of(); + }); + PluginDrivenExternalTable table = pluginTable(EnumSet.of(WriteOperation.INSERT), provider); + Mockito.when(table.requiresFullSchemaWriteOrder()).thenReturn(true); + + SlotReference aOutput = new SlotReference("a", IntegerType.INSTANCE); + SlotReference bOutput = new SlotReference("b", IntegerType.INSTANCE); + SlotReference cOutput = new SlotReference("c", IntegerType.INSTANCE); + TupleDescriptor tuple = context.generateTupleDesc(); + SlotRef aSlot = registerLegacySlot(context, tuple, aOutput, A); + SlotRef bSlot = registerLegacySlot(context, tuple, bOutput, B); + SlotRef cSlot = registerLegacySlot(context, tuple, cOutput, C); + + @SuppressWarnings("unchecked") + PhysicalConnectorTableSink sink = Mockito.mock(PhysicalConnectorTableSink.class); + Mockito.doReturn(mockChild(childFragment)).when(sink).child(); + Mockito.doReturn(table).when(sink).getTargetTable(); + Mockito.doReturn(writeColumns).when(sink).getCols(); + Mockito.doReturn(ImmutableList.of(A, B, C)).when(sink).getBoundTargetSchema(); + Mockito.doReturn(ImmutableList.of(aOutput, bOutput, cOutput)).when(sink).getOutput(); + Mockito.doReturn(false).when(sink).isRewrite(); + + new PhysicalPlanTranslator(context, null).visitPhysicalConnectorTableSink(sink, context); + + TSortInfo sortInfo = Deencapsulation.getField(capturePluginSink(childFragment), "writeSortInfo"); + Assertions.assertNotNull(sortInfo); + Assertions.assertEquals(1, sortInfo.getOrderingExprsSize()); + int actualSlotId = sortInfo.getOrderingExprs().get(0).getNodes().get(0).getSlotRef().getSlotId(); + SlotRef expected = sortColumn == A ? aSlot : sortColumn == B ? bSlot : cSlot; + Assertions.assertEquals(expected.getDesc().getId().asInt(), actualSlotId, + "write-sort position must use the same full-schema coordinates as the sink output"); + } + + private static SlotRef registerLegacySlot(PlanTranslatorContext context, TupleDescriptor tuple, + SlotReference output, Column column) { + SlotDescriptor descriptor = context.addSlotDesc(tuple); + descriptor.setColumn(column); + descriptor.setType(column.getType()); + descriptor.setIsNullable(true); + SlotRef slotRef = new SlotRef(descriptor); + context.addExprIdSlotRefPair(output.getExprId(), slotRef); + return slotRef; + } + /** A plugin-driven table whose connector declares exactly the given write operations. */ private static PluginDrivenExternalTable pluginTable(Set ops) { + return pluginTable(ops, Mockito.mock(ConnectorWritePlanProvider.class)); + } + + private static PluginDrivenExternalTable pluginTable(Set ops, + ConnectorWritePlanProvider provider) { ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); ConnectorSession session = Mockito.mock(ConnectorSession.class); // The write seams now resolve metadata through the per-statement funnel, which reads the session's // statement scope; offline tests use NONE (a fresh getMetadata per call, byte-identical to pre-funnel). Mockito.when(session.getStatementScope()).thenReturn(ConnectorStatementScope.NONE); - ConnectorWritePlanProvider provider = Mockito.mock(ConnectorWritePlanProvider.class); Connector connector = Mockito.mock(Connector.class); Mockito.when(connector.getWritePlanProvider()).thenReturn(provider); // Production selects the write provider per-handle; a plain mock does not run the interface default. diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java index b18d613d13cb99..ba275fd20246dc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorIcebergRowLevelDmlTest.java @@ -393,5 +393,9 @@ private static void assertConnectorColumnsFromCols(PluginDrivenTableSink pluginS "the connector columns must be derived from the sink's getCols()"); Assertions.assertEquals("data", connectorColumns.get(0).getName(), "the connector column name must carry the sink column name"); + // This is the exact translator output consumed by the provider. Parameterless Doris scalars expose + // internal 0/0 defaults, but the connector schema contract uses the canonical -1/-1 representation. + Assertions.assertEquals(-1, connectorColumns.get(0).getType().getPrecision()); + Assertions.assertEquals(-1, connectorColumns.get(0).getType().getScale()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindConnectorSinkStaticPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindConnectorSinkStaticPartitionTest.java index 48f1f32833dcd8..e51cee3fbf87de 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindConnectorSinkStaticPartitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindConnectorSinkStaticPartitionTest.java @@ -22,6 +22,7 @@ import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import com.google.common.collect.ImmutableList; @@ -34,7 +35,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -61,7 +61,7 @@ public class BindConnectorSinkStaticPartitionTest { private static PluginDrivenExternalTable partitionedTable() { PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); Mockito.when(table.getBaseSchema(true)).thenReturn(BASE_SCHEMA); - Mockito.when(table.getFullSchema(Optional.empty())).thenReturn(BASE_SCHEMA); + stubWriteSchemaSnapshot(table, BASE_SCHEMA, ImmutableList.of(DS, REGION)); // Model ExternalTable.getColumn, which resolves case-INSENSITIVELY (equalsIgnoreCase) for every // external table. Stubbing only the exact spelling would hide the very behavior under test. Mockito.when(table.getColumn(Mockito.anyString())).thenAnswer(inv -> { @@ -82,23 +82,40 @@ private static Map partitionSpec(String... colNames) { return spec; } - /** - * A table carrying an invisible column after the visible data columns, modelling an iceberg v3 table - * whose row-lineage {@code _row_id} is appended {@code .invisible()} by the connector. - */ - private static PluginDrivenExternalTable tableWithRowLineage() { - Column rowId = new Column("_row_id", PrimitiveType.BIGINT); - rowId.setIsVisible(false); - List schema = ImmutableList.of(ID, VAL, rowId); + private static PluginDrivenExternalTable tableWithRewriteColumns(boolean includeLineage) { + Column rowLocator = new Column("__DORIS_ICEBERG_ROWID_COL__", PrimitiveType.STRING); + rowLocator.setIsVisible(false); + ImmutableList.Builder schemaBuilder = ImmutableList.builder(); + schemaBuilder.add(ID, VAL); + if (includeLineage) { + Column rowId = new Column("_row_id", PrimitiveType.BIGINT); + rowId.setIsVisible(false); + rowId.setReservedPassthrough(true); + Column sequenceNumber = new Column("_last_updated_sequence_number", PrimitiveType.BIGINT); + sequenceNumber.setIsVisible(false); + sequenceNumber.setReservedPassthrough(true); + schemaBuilder.add(rowId, sequenceNumber); + } + schemaBuilder.add(rowLocator); + List schema = schemaBuilder.build(); PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); Mockito.when(table.getBaseSchema(true)).thenReturn(schema); - Mockito.when(table.getFullSchema(Optional.empty())).thenReturn(schema); + stubWriteSchemaSnapshot(table, schema, Collections.emptyList()); for (Column c : schema) { Mockito.when(table.getColumn(c.getName())).thenReturn(c); } return table; } + private static void stubWriteSchemaSnapshot(PluginDrivenExternalTable table, + List schema, List partitionColumns) { + PluginDrivenExternalTable.WriteSchemaSnapshot snapshot = + Mockito.mock(PluginDrivenExternalTable.WriteSchemaSnapshot.class); + Mockito.when(snapshot.getFullSchema()).thenReturn(schema); + Mockito.when(snapshot.getPartitionColumns()).thenReturn(partitionColumns); + Mockito.when(table.getWriteSchemaSnapshot()).thenReturn(snapshot); + } + private static List names(List columns) { return columns.stream().map(Column::getName).collect(Collectors.toList()); } @@ -163,7 +180,7 @@ public void explicitColumnListUsesLatestTargetSchemaInsteadOfAmbientSourceSnapsh // therefore sees old_name, while an explicit empty pin means the latest write-target schema. Mockito.when(table.getColumn("id")).thenReturn(ID); Mockito.when(table.getColumn("new_name")).thenReturn(null); - Mockito.when(table.getFullSchema(Optional.empty())).thenReturn(ImmutableList.of(ID, newName)); + stubWriteSchemaSnapshot(table, ImmutableList.of(ID, newName), Collections.emptyList()); Mockito.when(table.getBaseSchema(true)).thenReturn(ImmutableList.of(ID, oldName)); List bound = BindSink.selectConnectorSinkBindColumns( @@ -172,6 +189,17 @@ public void explicitColumnListUsesLatestTargetSchemaInsteadOfAmbientSourceSnapsh "a historical source pin must not replace the latest write-target schema"); } + @Test + public void explicitColumnListLoadsLatestTargetSchemaOnce() { + PluginDrivenExternalTable table = partitionedTable(); + + List bound = BindSink.selectConnectorSinkBindColumns( + table, ImmutableList.of("id", "val", "region"), Collections.emptySet(), false); + + Assertions.assertEquals(ImmutableList.of("id", "val", "region"), names(bound)); + Mockito.verify(table, Mockito.times(1)).getWriteSchemaSnapshot(); + } + /** * A column whose value comes from the PARTITION clause must not ALSO be listed in the insert column * list. Encodes WHY: the materialize block re-projects the PARTITION literal over that column, so the @@ -286,7 +314,8 @@ public void explicitColumnListUnknownColumnThrows() { @Test public void pinnedDataSchemaStillRejectsExplicitInvisibleColumn() { - PluginDrivenExternalTable table = tableWithRowLineage(); + // Reuse the rewrite schema so the invariant covers the same reserved lineage metadata as production. + PluginDrivenExternalTable table = tableWithRewriteColumns(true); AnalysisException ex = Assertions.assertThrows(AnalysisException.class, () -> BindSink.selectConnectorSinkBindColumns( table, ImmutableList.of(ID, VAL), ImmutableList.of("_row_id"), @@ -305,21 +334,57 @@ public void pinnedDataSchemaStillRejectsExplicitInvisibleColumn() { @Test public void noColumnListOrdinaryWriteExcludesInvisibleColumns() { List bound = BindSink.selectConnectorSinkBindColumns( - tableWithRowLineage(), Collections.emptyList(), Collections.emptySet(), false); + tableWithRewriteColumns(true), Collections.emptyList(), Collections.emptySet(), false); Assertions.assertEquals(ImmutableList.of("id", "val"), names(bound), "invisible row-lineage columns must be excluded from an ordinary write target"); } /** - * No column list, rewrite (distributed {@code rewrite_data_files}): invisible columns are RETAINED so - * the engine-managed row-lineage values read from the source rows are preserved through the rewrite, - * mirroring the legacy {@code bindIcebergTableSink} rewrite branch. + * A v2 rewrite under show-hidden carries the request-scoped row locator in the table's full schema, but + * the rewrite sink has no physical field for it. */ @Test - public void noColumnListRewriteRetainsInvisibleColumns() { + public void noColumnListV2RewriteExcludesRequestScopedRowLocator() { List bound = BindSink.selectConnectorSinkBindColumns( - tableWithRowLineage(), Collections.emptyList(), Collections.emptySet(), true); - Assertions.assertEquals(ImmutableList.of("id", "val", "_row_id"), names(bound), - "a rewrite must retain invisible row-lineage columns to preserve their values"); + tableWithRewriteColumns(false), Collections.emptyList(), Collections.emptySet(), true); + Assertions.assertEquals(ImmutableList.of("id", "val"), names(bound), + "a v2 rewrite must not emit the request-scoped row locator"); + } + + /** + * A v3 rewrite preserves persistent lineage fields, while excluding the unrelated request-scoped locator. + */ + @Test + public void noColumnListV3RewriteRetainsLineageButExcludesRequestScopedRowLocator() { + List bound = BindSink.selectConnectorSinkBindColumns( + tableWithRewriteColumns(true), Collections.emptyList(), Collections.emptySet(), true); + Assertions.assertEquals(ImmutableList.of("id", "val", "_row_id", "_last_updated_sequence_number"), + names(bound), "a v3 rewrite must retain only persistent lineage metadata"); + } + + @Test + public void rewriteSourceOutputExcludesRequestScopedRowLocator() { + NamedExpression id = namedExpression("id"); + NamedExpression val = namedExpression("val"); + NamedExpression rowId = namedExpression("_row_id"); + NamedExpression sequenceNumber = namedExpression("_last_updated_sequence_number"); + NamedExpression locator = namedExpression("__DORIS_ICEBERG_ROWID_COL__"); + + List v2Selected = BindSink.selectConnectorRewriteOutputs( + ImmutableList.of(ID, VAL), ImmutableList.of(id, val, locator)); + List v3WriteSchema = BindSink.selectConnectorSinkBindColumns( + tableWithRewriteColumns(true), Collections.emptyList(), Collections.emptySet(), true); + List v3Selected = BindSink.selectConnectorRewriteOutputs( + v3WriteSchema, ImmutableList.of(id, val, rowId, sequenceNumber, locator)); + + Assertions.assertEquals(ImmutableList.of(id, val), v2Selected); + Assertions.assertEquals(ImmutableList.of(id, val, rowId, sequenceNumber), v3Selected, + "v2/v3 rewrite input must use the same physical column set as its sink schema"); + } + + private static NamedExpression namedExpression(String name) { + NamedExpression expression = Mockito.mock(NamedExpression.class); + Mockito.when(expression.getName()).thenReturn(name); + return expression; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java index c367d8ade27594..1982a6ae67f742 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StructLiteralTest.java @@ -18,15 +18,23 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.apache.doris.nereids.rules.expression.check.CheckCast; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.TryCast; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; public class StructLiteralTest { @@ -65,4 +73,37 @@ public void testNamedStructInfersValueNullability() { StructType nullableType = (StructType) nullable.customSignature().returnType; Assertions.assertTrue(nullableType.getFields().get(0).isNullable()); } + + @Test + public void testStructFunctionsKeepPhysicalCastNullabilityInStrictMode() { + SlotReference requiredString = new SlotReference("metric", StringType.INSTANCE, false); + Cast cast = new Cast(requiredString, IntegerType.INSTANCE); + TryCast tryCast = new TryCast(requiredString, IntegerType.INSTANCE); + + try (MockedStatic mockedSessionVariable = Mockito.mockStatic(SessionVariable.class)) { + mockedSessionVariable.when(SessionVariable::enableStrictCast).thenReturn(true); + // Strict-mode failure semantics do not change the BE column representation: narrowing + // casts still return ColumnNullable with an all-clear map for valid rows. + Assertions.assertTrue(cast.nullable()); + + CreateStruct struct = new CreateStruct(cast); + StructType structType = (StructType) struct.getSignatures().get(0).returnType; + Assertions.assertTrue(structType.getFields().get(0).isNullable()); + + CreateNamedStruct namedStruct = new CreateNamedStruct(new StringLiteral("metric"), cast); + StructType namedStructType = (StructType) namedStruct.customSignature().returnType; + Assertions.assertTrue(namedStructType.getFields().get(0).isNullable()); + + CreateNamedStruct namedTryStruct = new CreateNamedStruct(new StringLiteral("metric"), tryCast); + StructType namedTryStructType = (StructType) namedTryStruct.customSignature().returnType; + Assertions.assertTrue(namedTryStructType.getFields().get(0).isNullable()); + } + + try (MockedStatic mockedSessionVariable = Mockito.mockStatic(SessionVariable.class)) { + mockedSessionVariable.when(SessionVariable::enableStrictCast).thenReturn(false); + CreateStruct struct = new CreateStruct(cast); + StructType structType = (StructType) struct.getSignatures().get(0).returnType; + Assertions.assertTrue(structType.getFields().get(0).isNullable()); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java index 96fc06491a02e5..84509129231ccc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelMergePlanBuilderTest.java @@ -22,7 +22,8 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.trees.expressions.Cast; @@ -44,6 +45,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.mockito.InOrder; import org.mockito.Mockito; import java.util.List; @@ -72,6 +74,8 @@ public void tearDown() { @Test public void mergeSinkRequestsTheSqlMergeCardinalityCheck() { ConnectContext ctx = new ConnectContext(); + // Writer schemas are pinned per statement in production; the focused builder test needs the same scope. + ctx.setStatementContext(new StatementContext()); ctx.setThreadLocalInfo(); LogicalPlan source = new LogicalOneRowRelation(new RelationId(0), ImmutableList.of(new UnboundAlias(new IntegerLiteral(1), "dummy"))); @@ -89,10 +93,16 @@ public void mergeSinkRequestsTheSqlMergeCardinalityCheck() { Column rowId = new Column(Column.ICEBERG_ROWID_COL, ScalarType.createStringType()); rowId.setIsVisible(false); Column data = new Column("c1", ScalarType.createType(PrimitiveType.INT)); - ExternalTable table = Mockito.mock(ExternalTable.class); + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + PluginDrivenExternalTable.WriteSchemaSnapshot writeSchema = + Mockito.mock(PluginDrivenExternalTable.WriteSchemaSnapshot.class); Mockito.when(table.getName()).thenReturn("test_table"); Mockito.doReturn(Mockito.mock(ExternalDatabase.class)).when(table).getDatabase(); - Mockito.doReturn(ImmutableList.of(data)).when(table).getBaseSchema(true); + Mockito.doReturn(ImmutableList.of(data)).when(writeSchema).getBaseSchema(); + Mockito.doReturn("uuid-u0/schema-1").when(writeSchema).getWriteMetadataIdentity(); + Mockito.doReturn(writeSchema).when(table).getWriteSchemaSnapshot(); + Mockito.doReturn(Optional.of(ImmutableList.of(data))) + .when(table).resolveWriteColumns(Optional.empty()); Mockito.doReturn(ImmutableList.of(data, rowId)).when(table).getFullSchema(); LogicalPlan plan = builder.buildMergePlan(ctx, table); @@ -105,6 +115,13 @@ public void mergeSinkRequestsTheSqlMergeCardinalityCheck() { Assertions.assertTrue(plan instanceof LogicalExternalRowLevelMergeSink); Assertions.assertTrue(((LogicalExternalRowLevelMergeSink) plan).isRequireMergeCardinalityCheck(), "SQL MERGE INTO must request the cardinality validation"); + // The branch projections, sink columns, and conflict fence must come from one generation. + Assertions.assertEquals("uuid-u0/schema-1", + ((LogicalExternalRowLevelMergeSink) plan).getBoundWriteMetadataIdentity()); + // The snapshot identity must be read only after resolving the request-scoped writer columns. + InOrder metadataOrder = Mockito.inOrder(table); + metadataOrder.verify(table).resolveWriteColumns(Optional.empty()); + metadataOrder.verify(table).getWriteSchemaSnapshot(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilderTest.java index 33be03bc8a0ebf..7ebc2e87af7c6d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExternalRowLevelUpdatePlanBuilderTest.java @@ -22,7 +22,8 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -40,11 +41,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.mockito.InOrder; import org.mockito.Mockito; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; public class ExternalRowLevelUpdatePlanBuilderTest { @@ -62,6 +65,8 @@ public void tearDown() { @Test public void testBuildMergeProjectPlanProjectsRowId() { ConnectContext ctx = new ConnectContext(); + // Writer schemas are pinned per statement in production; the focused builder test needs the same scope. + ctx.setStatementContext(new StatementContext()); ctx.setThreadLocalInfo(); LogicalPlan basePlan = new LogicalOneRowRelation(new RelationId(0), ImmutableList.of(new UnboundAlias(new IntegerLiteral(1), "dummy"))); @@ -107,23 +112,38 @@ public void updateSinkCarriesNoSqlMergeCardinalityRequirement() { // no one-source-row rule, and stamping true would both reject legal UPDATEs at the BE and force the // merge distribution even when enable_strict_consistency_dml is off. ConnectContext ctx = new ConnectContext(); + // Writer schemas are pinned per statement in production; the focused builder test needs the same scope. + ctx.setStatementContext(new StatementContext()); ctx.setThreadLocalInfo(); LogicalPlan basePlan = new LogicalOneRowRelation(new RelationId(0), ImmutableList.of(new UnboundAlias(new IntegerLiteral(1), "dummy"))); ExternalRowLevelUpdatePlanBuilder builder = new ExternalRowLevelUpdatePlanBuilder( ImmutableList.of("test_catalog", "test_db", "test_table"), null, ImmutableList.of(), basePlan); - ExternalTable table = Mockito.mock(ExternalTable.class); + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + PluginDrivenExternalTable.WriteSchemaSnapshot writeSchema = + Mockito.mock(PluginDrivenExternalTable.WriteSchemaSnapshot.class); + Column data = new Column("c1", ScalarType.createType(PrimitiveType.INT)); Mockito.when(table.getName()).thenReturn("test_table"); Mockito.doReturn(Mockito.mock(ExternalDatabase.class)).when(table).getDatabase(); - Mockito.doReturn(ImmutableList.of(new Column("c1", ScalarType.createType(PrimitiveType.INT)))) - .when(table).getBaseSchema(true); + Mockito.doReturn(ImmutableList.of(data)).when(writeSchema).getBaseSchema(); + Mockito.doReturn("uuid-u0/schema-1").when(writeSchema).getWriteMetadataIdentity(); + Mockito.doReturn(writeSchema).when(table).getWriteSchemaSnapshot(); + Mockito.doReturn(Optional.of(ImmutableList.of(data))) + .when(table).resolveWriteColumns(Optional.empty()); LogicalPlan plan = builder.buildMergePlan(ctx, basePlan, ImmutableList.of(), table); Assertions.assertTrue(plan instanceof LogicalExternalRowLevelMergeSink); Assertions.assertFalse(((LogicalExternalRowLevelMergeSink) plan).isRequireMergeCardinalityCheck(), "UPDATE must not request the SQL MERGE cardinality validation"); + // Projection, sink metadata, and the conflict fence must share one schema generation. + Assertions.assertEquals("uuid-u0/schema-1", + ((LogicalExternalRowLevelMergeSink) plan).getBoundWriteMetadataIdentity()); + // The snapshot identity must be read only after resolving the request-scoped writer columns. + InOrder metadataOrder = Mockito.inOrder(table); + metadataOrder.verify(table).resolveWriteColumns(Optional.empty()); + metadataOrder.verify(table).getWriteSchemaSnapshot(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergRowLevelDmlTransformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergRowLevelDmlTransformTest.java index 3e03a230aeba32..a079b576708d97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergRowLevelDmlTransformTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IcebergRowLevelDmlTransformTest.java @@ -212,10 +212,14 @@ public void synthesizeDeleteOnPluginTableBuildsSinkTargetingIt() { // re-parameterization; the full plan execution is flip-e2e-gated. (Reverting the cast/param back to // IcebergExternalTable would not even compile against this plugin-typed argument.) PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + PluginDrivenExternalTable.WriteSchemaSnapshot writeSchema = + Mockito.mock(PluginDrivenExternalTable.WriteSchemaSnapshot.class); ExternalDatabase database = Mockito.mock(ExternalDatabase.class); Mockito.when(table.getDatabase()).thenReturn(database); - Mockito.when(table.getBaseSchema(true)) - .thenReturn(ImmutableList.of(new Column("id", ScalarType.INT))); + // The DELETE projection and conflict fence must be bound to the same metadata generation. + Mockito.doReturn(ImmutableList.of(new Column("id", ScalarType.INT))).when(writeSchema).getBaseSchema(); + Mockito.doReturn("uuid-u0/schema-1").when(writeSchema).getWriteMetadataIdentity(); + Mockito.doReturn(writeSchema).when(table).getWriteSchemaSnapshot(); Mockito.when(table.getId()).thenReturn(TARGET_ID); LogicalPlan query = (LogicalPlan) filterOver(table, "id"); @@ -228,6 +232,9 @@ public void synthesizeDeleteOnPluginTableBuildsSinkTargetingIt() { Assertions.assertTrue(plan instanceof LogicalExternalRowLevelDeleteSink, plan.getClass().getName()); Assertions.assertSame(table, ((LogicalExternalRowLevelDeleteSink) plan).getTargetTable()); + Assertions.assertEquals("uuid-u0/schema-1", + ((LogicalExternalRowLevelDeleteSink) plan).getBoundWriteMetadataIdentity()); + Mockito.verify(table, Mockito.times(1)).getWriteSchemaSnapshot(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java index 4e5e5347dd68f6..22a7b4aca04bb3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalConnectorTableSinkTest.java @@ -101,6 +101,27 @@ public void dynamicPartitionWriteRequiresHashAndLocalSort() { "local sort must be on the partition column"); } + @Test + public void distributionUsesBindTimeSchemaAfterLiveRefresh() { + SlotReference dataSlot = new SlotReference("data", IntegerType.INSTANCE); + SlotReference partSlot = new SlotReference("part", IntegerType.INSTANCE); + PluginDrivenExternalTable table = table( + true, true, ImmutableList.of(PART), ImmutableList.of(DATA, PART)); + PhysicalConnectorTableSink sink = sink( + table, Arrays.asList(DATA, PART), ImmutableList.of(dataSlot, partSlot)); + + // Simulate a refresh after binding that reuses names at different positions. Distribution must keep + // the coherent schema generation captured by the sink instead of indexing its older output with S1. + Mockito.when(table.getPartitionColumns()).thenReturn(ImmutableList.of(DATA)); + Mockito.when(table.getFullSchema()).thenReturn(ImmutableList.of(PART, DATA)); + + PhysicalProperties props = sink.getRequirePhysicalProperties(); + DistributionSpecHiveTableSinkHashPartitioned dist = + (DistributionSpecHiveTableSinkHashPartitioned) props.getDistributionSpec(); + Assertions.assertEquals(ImmutableList.of(partSlot.getExprId()), dist.getOutputColExprIds()); + Assertions.assertEquals(partSlot, props.getOrderSpec().getOrderKeys().get(0).getExpr()); + } + /** * Pure-dynamic write with a REORDERED explicit column list ({@code INSERT INTO mc (part, data) * SELECT vpart, vdata}, schema [data, part]): the bind layer projects the child to FULL-SCHEMA @@ -324,9 +345,8 @@ private static PluginDrivenExternalTable table(boolean parallelWrite, boolean re /** * Builds a {@link PhysicalConnectorTableSink} exercising only {@code getRequirePhysicalProperties()}. - * Uses CALLS_REAL_METHODS to skip the heavyweight ctor and injects the three fields the method - * reads ({@code targetTable}, {@code cols}, and the single child via the {@code children} field, so - * the real {@code child()} resolves to it). + * Uses CALLS_REAL_METHODS to skip the heavyweight ctor and injects the bind-time schemas, target, + * columns, and single child needed by the real method. */ private static PhysicalConnectorTableSink sink(PluginDrivenExternalTable table, List cols, List childOutput) { @@ -336,6 +356,10 @@ private static PhysicalConnectorTableSink sink(PluginDrivenExternalTable t PhysicalConnectorTableSink sink = Mockito.mock(PhysicalConnectorTableSink.class, Mockito.CALLS_REAL_METHODS); Deencapsulation.setField(sink, "targetTable", table); + // Production carries both lists from one bind-time cache value; seed the real-method mock with + // that immutable view so the test also fails if the implementation re-reads the live table. + Deencapsulation.setField(sink, "boundTargetSchema", table.getFullSchema()); + Deencapsulation.setField(sink, "boundPartitionColumns", table.getPartitionColumns()); Deencapsulation.setField(sink, "cols", cols); Deencapsulation.setField(sink, "children", ImmutableList.of(child)); return sink; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSinkTest.java index c108a61084ff34..793f14a3a834c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalExternalRowLevelMergeSinkTest.java @@ -166,6 +166,20 @@ public void reconstructUnresolvedExprIdHardFailsAndClears() { Assertions.assertEquals(Integer.valueOf(7), result.partitionSpecId); } + @Test + public void reconstructRejectsSameNameWithReplacementFieldId() { + ExprId oldIdExpr = exprId("id"); + List out = new ArrayList<>(); + + InsertPartitionFieldResult result = PhysicalExternalRowLevelMergeSink.reconstructPartitionFields(out, + spec(8, field("identity", null, "id", "id", 2)), map("id", oldIdExpr), + java.util.Collections.singletonMap(1, oldIdExpr)); + + // The live field reused the name but not the bound identity; name fallback would route wrong values. + Assertions.assertFalse(result.success); + Assertions.assertTrue(out.isEmpty()); + } + @Test public void reconstructNonIdentityPrePassSeesFieldsAfterHardFail() { // PARITY-2 independence: the build loop short-circuits on field 0 (null name), but the diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java index b91a71b62ce111..658b5cf2ae2f67 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkTest.java @@ -106,6 +106,25 @@ public void bindDataSinkDelegatesToWritePlanProvider() throws AnalysisException Assert.assertNull(provider.seenHandle.getSortInfo()); } + @Test + public void bindDataSinkKeepsWriteSubsetSeparateFromBoundTargetSchema() throws AnalysisException { + RecordingWritePlanProvider provider = new RecordingWritePlanProvider( + new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK))); + ConnectorColumn id = Mockito.mock(ConnectorColumn.class); + ConnectorColumn name = Mockito.mock(ConnectorColumn.class); + List writeColumns = Collections.singletonList(id); + List boundTargetColumns = java.util.Arrays.asList(id, name); + + PluginDrivenTableSink sink = new PluginDrivenTableSink( + null, provider, null, new ConnectorTableHandle() { }, writeColumns, + boundTargetColumns, null, WriteOperation.INSERT, false); + sink.bindDataSink(Optional.empty()); + + Assert.assertSame(writeColumns, provider.seenHandle.getColumns()); + Assert.assertEquals(boundTargetColumns, provider.seenHandle.getBoundTargetColumns()); + Assert.assertNotSame(boundTargetColumns, provider.seenHandle.getBoundTargetColumns()); + } + @Test public void bindDataSinkThreadsEngineBuiltWriteSortInfoToHandle() throws AnalysisException { // WHY: the connector's planWrite cannot build a TSortInfo (the bound output exprs live only in the @@ -125,6 +144,20 @@ public void bindDataSinkThreadsEngineBuiltWriteSortInfoToHandle() throws Analysi Assert.assertSame(engineBuilt, provider.seenHandle.getSortInfo()); } + @Test + public void bindDataSinkThreadsBoundWriteMetadataIdentityToHandle() throws AnalysisException { + RecordingWritePlanProvider provider = new RecordingWritePlanProvider( + new ConnectorSinkPlan(new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK))); + String metadataIdentity = "sort-generation-3/spec-generation-7"; + + PluginDrivenTableSink sink = new PluginDrivenTableSink( + null, provider, null, new ConnectorTableHandle() { }, new ArrayList<>(), + Collections.emptyList(), null, WriteOperation.INSERT, false, metadataIdentity); + sink.bindDataSink(Optional.empty()); + + Assert.assertEquals(metadataIdentity, provider.seenHandle.getBoundWriteMetadataIdentity()); + } + @Test public void bindDataSinkThreadsBranchNameToHandle() throws AnalysisException { // WHY: INSERT INTO t@branch carries the target branch on the PluginDrivenInsertCommandContext; diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java index 1bf16ca7a642ee..3c8632e1a7be1d 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.TScalarType; @@ -458,6 +459,11 @@ public String hideVersionForVersionColumn(Boolean isToSql) { } public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedComment) { + return hideVersionForVersionColumn(isToSql, showNestedComment, false); + } + + public String hideVersionForVersionColumn( + Boolean isToSql, boolean showNestedComment, boolean noBackslashEscapes) { if (isDatetime() || isDatetimeV2()) { StringBuilder typeStr = new StringBuilder("datetime"); if (((ScalarType) this).getScalarScale() > 0) { @@ -487,13 +493,13 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom return typeStr.toString(); } else if (isArrayType()) { String nestedDesc = ((ArrayType) this).getItemType() - .hideVersionForVersionColumn(isToSql, showNestedComment); + .hideVersionForVersionColumn(isToSql, showNestedComment, noBackslashEscapes); return "array<" + nestedDesc + ">"; } else if (isMapType()) { String keyDesc = ((MapType) this).getKeyType() - .hideVersionForVersionColumn(isToSql, showNestedComment); + .hideVersionForVersionColumn(isToSql, showNestedComment, noBackslashEscapes); String valueDesc = ((MapType) this).getValueType() - .hideVersionForVersionColumn(isToSql, showNestedComment); + .hideVersionForVersionColumn(isToSql, showNestedComment, noBackslashEscapes); return "map<" + keyDesc + "," + valueDesc + ">"; } else if (isStructType()) { List fieldDesc = new ArrayList<>(); @@ -501,7 +507,8 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom for (int i = 0; i < structType.getFields().size(); i++) { StructField field = structType.getFields().get(i); StringBuilder desc = new StringBuilder(field.getName()).append(":") - .append(field.getType().hideVersionForVersionColumn(isToSql, showNestedComment)); + .append(field.getType().hideVersionForVersionColumn( + isToSql, showNestedComment, noBackslashEscapes)); // Requiredness is schema semantics and must survive independently of whether // nested documentation is requested for DESCRIBE output. if (!field.getContainsNull()) { @@ -509,7 +516,9 @@ public String hideVersionForVersionColumn(Boolean isToSql, boolean showNestedCom } // Nested docs are part of DESCRIBE output only when comments were explicitly requested. if (showNestedComment && field.isCommentSpecified()) { - desc.append(String.format(" comment '%s'", field.getComment())); + // Comments must remain parseable even when they contain quotes or backslashes. + desc.append(" comment ").append( + SqlUtils.quoteStringLiteral(field.getComment(), noBackslashEscapes)); } fieldDesc.add(desc.toString()); } From 95098e348d52bf77e618b6f589576100ad58e8fd Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Fri, 7 Aug 2026 08:16:53 +0800 Subject: [PATCH 10/12] [refactor](connector) give every connector the same property layout, and make each key have one reader (#66507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? Issue: #65185 Problem Summary: Every connector had grown its own way of dealing with catalog properties. Some kept a `XxxConnectorProperties` constant class and parsed values at each read site; some had no property class at all and inlined the key names; iceberg and paimon had *two* readers for the same keys — a typed holder used for CREATE-time validation, and a separate raw-map scan with its own copy of the alias arrays used to actually build the catalog. Nothing kept those two in agreement, so an alias priority or a blank-value rule could be validated one way and assembled another. This PR gives every connector the same four-way split, and makes each key have exactly one reader. No new SPI: `ConnectorPluginSurfaceTest` is untouched throughout. ### Where things live now **A — `CatalogProperties`: what a user writes in `CREATE CATALOG`.** One class per connector, in the connector module. Fields carry `@ConnectorProperty` with the alias list, so a key name and its aliases are declared exactly once, and `ConnectorPropertiesUtils` binds them. This is what the connector, the metadata layer and the scan/write planners read — none of them touch the raw map for a key this class declares. The two entry points are deliberately not interchangeable, and this is the part most worth reviewing: - `of(Map)` **binds and derives, and never throws.** It runs at CREATE, at ALTER validation, and on every connector build — including the lazy rebuild after an FE restart. A rule placed here that a live catalog violates does not fail at review time; it fails months later, as a catalog that stops coming back after a restart. - `checkCreateTimeOnlyRules()` carries everything that judges, and only the provider calls it, from `validateProperties` — one line in most connectors. Unknown keys are never rejected: the same map carries engine keys and storage keys, and `ALTER CATALOG` can only overwrite a key, never remove one, so a rejected unknown key could not be repaired. **B — `Conf`: deployment-level settings.** The keys of the plugin's own `.conf`, each falling back to the `fe.conf` key it used to live under. Static accessors (`driversDir(context)`, `metastoreClientTimeoutSecond(context)`, …) rather than a bound object, because these belong to the deployment and not to any one catalog. Only the connectors that actually have such settings have one. **Per-flavor `*MetaStoreProperties` (iceberg / paimon): the keys of one metastore backend.** These already existed in `fe-connector-metastore-{iceberg,paimon}` but described themselves as "validation only" while the connector re-scanned the same keys to build the catalog. They are now the single declaration: they gained the fields the assembly needed and the getters it reads, and the factories consume the bound holder instead of `firstNonBlank(props, ALIASES)`. The alias arrays that duplicated them are deleted. The metastore modules stay SDK-free — they expose neutral getters and maps; engine-SDK option assembly stays in the connector factory. Splitting by flavor also splits the "annotation count == key count" invariant: the connector-level holder declares the flavor-independent keys, each backend holder declares its own. **F/G — literals the assembly emits, and mode enums.** Option keys and values that a connector *writes* (rather than a user setting them) are private to the class that writes them — `IcebergCatalogFactory` for the iceberg SDK dialect, and so on. Cache key names sit next to the cache they configure. In several places these had drifted into two spellings of the same string in two files; folding them removed those duplicates. ### Responsibilities, end to end | Layer | Reads | Writes | |---|---|---| | `ConnectorProvider` | — | `validateProperties` → `of(props).checkCreateTimeOnlyRules()` | | `CatalogProperties` | the raw map, once | typed getters; the derived flavor | | `*MetaStoreProperties` | the raw map, once, for its own backend | typed getters; neutral conf maps | | `CatalogFactory` | the bound holders (+ raw map only for copy-all / prefix passthroughs) | the engine-SDK option map | | `Conf` | `ConnectorContext` conf + environment | — | | connector / metadata / scan / write | the bound holders | — | The raw map is still read in three legitimate shapes, each commented where it happens: copy-all passthrough into the SDK options, whole-namespace forwarding (`fs.` / `dfs.` / `hadoop.`, `paimon.`, `jdbc.`), and alias sets that span namespaces and so belong to no single flavor (the S3 region aliases, the AWS credentials-provider mode). ### Verification Unit tests only — this is a refactor with no intended behavior change, and the guard against unintended change is a set of whole-map snapshot tests added *before* each rework: paimon 8 cases and iceberg 20 cases assert the ENTIRE catalog option map, one per flavor and per emission branch. Both SDKs silently ignore an option they do not recognize, so a dropped or misspelled key does not throw — it produces a catalog that connects with different settings than the operator asked for. Those tests stay in the tree afterwards as the permanent guard that the holder and the assembly agree. Every touched module, run together at the final commit with `-Dmaven.build.cache.enabled=false` (the build cache otherwise reports a stale green): | module | tests | module | tests | |---|---|---|---| | iceberg | 1215 (5 skip) | hudi | 203 | | paimon | 532 (1 skip) | adbc | 200 | | hive | 411 | maxcompute | 147 (1 skip) | | jdbc | 222 | es | 118 | | connector-spi | 140 | trino | 55 | | hms shared lib | 107 | metastore-{iceberg,paimon,spi,api} | 77 | | foundation | 161 | cache framework | 33 | **3780 tests, 0 failures, 0 errors, 7 skips** — every skip is a pre-existing live-connectivity test gated on environment variables, and each was already skipped before this PR. checkstyle clean; `ConnectorPluginSurfaceTest` green. ### Behavior changes Small, and all on inputs that are already degenerate. Full per-connector tables are in the commit messages; the classes of change are: 1. **Values are trimmed.** The property binder trims and the old hand-written scans did not, so a `uri` written with a trailing space was already *validated* trimmed while the catalog was *built* from the untrimmed string. The two now agree. For paimon HMS this removed an existing internal inconsistency (HiveConf got the trimmed value, the paimon `Options` got the raw one). 2. **Blank now means unset**, where old code used `containsKey` / `getOrDefault`. Affects e.g. `iceberg.rest.view-enabled = ""` (was false, now its default true) and `external_catalog.name = ""` (was an empty namespace level, now absent). 3. **Values the FE interprets are now sent downstream interpreted.** This fixed three real bugs where the FE parsed a value and then forwarded the unparsed original — the ES `http_ssl_enabled` payload to BE, the hive `uri` shorthand, and the trino `connector.name`. Numeric keys were audited per connector and deliberately left as Strings wherever the value is forwarded verbatim to an engine SDK, so that a catalog created with a value the SDK tolerates keeps building; the JDBC connection-pool knobs are the one place where the strict/lenient choice is made per key, with the reasoning in that commit. ### Release note None ### Check List (For Author) - Test - [x] Unit Test - [ ] Regression test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - Behavior changed: - [x] Yes. Whitespace around property values is now trimmed; an explicitly blank value now reads as unset rather than as an empty string; and three keys the FE interprets are now forwarded in their interpreted form. Details per connector are in the individual commit messages. - Does this need documentation? - [x] No. --------- Co-authored-by: Claude Opus 5 (1M context) --- fe/fe-connector/README.md | 75 ++- fe/fe-connector/fe-connector-adbc/pom.xml | 7 + .../connector/adbc/AdbcCatalogProperties.java | 321 +++++++++ .../doris/connector/adbc/AdbcClient.java | 2 +- .../apache/doris/connector/adbc/AdbcConf.java | 77 +++ .../doris/connector/adbc/AdbcConnector.java | 66 +- .../connector/adbc/AdbcConnectorMetadata.java | 14 +- .../adbc/AdbcConnectorProperties.java | 226 ------- .../connector/adbc/AdbcConnectorProvider.java | 30 +- .../connector/adbc/AdbcDialectRegistry.java | 2 +- .../connector/adbc/AdbcDialectSelector.java | 13 +- .../adbc/AdbcDriverPathResolver.java | 28 +- .../connector/adbc/AdbcScanPlanProvider.java | 41 +- .../adbc/AdbcCatalogPropertiesTest.java | 262 ++++++++ .../adbc/AdbcConnectorCacheHookTest.java | 4 +- .../connector/adbc/AdbcConnectorConfTest.java | 59 +- .../adbc/AdbcConnectorMetadataTest.java | 2 +- .../adbc/AdbcConnectorPropertiesTest.java | 167 ----- .../doris/connector/adbc/AdbcDialectTest.java | 12 +- .../adbc/AdbcDriverPathResolverTest.java | 2 +- .../adbc/AdbcMetadataCacheNativeTest.java | 2 +- .../adbc/AdbcPartitionedReadNativeTest.java | 8 +- .../adbc/AdbcScanPlanProviderTest.java | 63 +- fe/fe-connector/fe-connector-es/pom.xml | 7 + .../connector/es/EsCatalogProperties.java | 281 ++++++++ .../doris/connector/es/EsConnector.java | 26 +- .../connector/es/EsConnectorMetadata.java | 41 +- .../connector/es/EsConnectorProperties.java | 101 --- .../connector/es/EsConnectorProvider.java | 12 +- .../connector/es/EsScanPlanProvider.java | 68 +- .../connector/es/EsCatalogPropertiesTest.java | 295 +++++++++ .../connector/es/EsScanPlanProviderTest.java | 10 +- fe/fe-connector/fe-connector-hive/pom.xml | 7 + .../connector/hive/HiveCatalogProperties.java | 266 ++++++++ .../doris/connector/hive/HiveConnector.java | 65 +- .../connector/hive/HiveConnectorMetadata.java | 103 +-- .../hive/HiveConnectorProperties.java | 150 ----- .../connector/hive/HiveConnectorProvider.java | 27 +- .../connector/hive/HiveFileListingCache.java | 21 +- .../connector/hive/HiveScanPlanProvider.java | 18 +- .../hive/HiveTableFormatDetector.java | 10 +- .../doris/connector/hive/HiveTableHandle.java | 4 +- .../connector/hive/HiveTextProperties.java | 7 + .../connector/hive/HiveWritePlanProvider.java | 17 +- .../apache/doris/connector/hive/HmsConf.java | 67 ++ .../hive/IcebergSiblingProperties.java | 4 +- .../hive/HiveCatalogPropertiesTest.java | 245 +++++++ .../hive/HiveConnectorCapabilitiesTest.java | 3 +- .../hive/HiveConnectorClientCacheTest.java | 2 +- .../hive/HiveConnectorContractTest.java | 5 +- .../HiveConnectorMetadataColumnStatsTest.java | 2 +- .../hive/HiveConnectorMetadataDdlTest.java | 30 +- ...iveConnectorMetadataFileListStatsTest.java | 6 +- .../HiveConnectorMetadataFreshnessTest.java | 2 +- ...iveConnectorMetadataPartitionListTest.java | 2 +- ...ConnectorMetadataPartitionPruningTest.java | 8 +- ...nnectorMetadataPartitionViewCacheTest.java | 4 +- .../hive/HiveConnectorMetadataSchemaTest.java | 4 +- ...onnectorMetadataSiblingDelegationTest.java | 16 +- ...eConnectorMetadataStaticPartitionTest.java | 2 +- .../HiveConnectorMetadataStatisticsTest.java | 3 +- .../HiveConnectorMetadataSysTableTest.java | 2 +- ...HiveConnectorMetadataTableCommentTest.java | 4 +- ...onnectorMetadataTableHandleDivertTest.java | 8 +- .../hive/HiveConnectorMetadataViewTest.java | 4 +- .../hive/HiveConnectorSiblingTest.java | 28 +- .../hive/HiveCreateTableValidationTest.java | 2 +- .../hive/HiveFileListingCacheTest.java | 42 +- .../hive/HiveReadTransactionTest.java | 4 +- .../connector/hive/HiveScanBatchModeTest.java | 12 +- .../hive/HiveTableFormatDetectionTest.java | 2 +- .../connector/hive/HiveTestProperties.java | 62 ++ .../hive/HiveWritePlanProviderTest.java | 2 +- .../doris/connector/hive/HmsConfTest.java | 93 +++ fe/fe-connector/fe-connector-hudi/pom.xml | 7 + .../connector/hudi/HudiCatalogProperties.java | 135 ++++ .../doris/connector/hudi/HudiConnector.java | 25 +- .../connector/hudi/HudiConnectorMetadata.java | 17 +- .../hudi/HudiConnectorProperties.java | 56 -- .../hudi/HudiBackendDescriptorTest.java | 2 +- .../hudi/HudiCatalogPropertiesTest.java | 129 ++++ .../hudi/HudiConnectorHmsCacheTest.java | 4 +- .../hudi/HudiConnectorOwnsHandleTest.java | 3 +- .../HudiConnectorPartitionListingTest.java | 6 +- .../connector/hudi/HudiIncrementalTest.java | 3 +- .../hudi/HudiPartitionPruningTest.java | 10 +- .../hudi/HudiReadOnlyWriteRejectTest.java | 4 +- .../hudi/HudiSchemaAtInstantTest.java | 2 +- .../connector/hudi/HudiStatementMemoTest.java | 2 +- .../connector/hudi/HudiTableTypeTest.java | 2 +- .../connector/hudi/HudiTestProperties.java | 52 ++ .../connector/hudi/HudiTimeTravelTest.java | 2 +- .../iceberg/AwsCredentialsProviderModes.java | 20 +- .../iceberg/IcebergCatalogFactory.java | 271 ++++---- .../iceberg/IcebergCatalogProperties.java | 202 ++++++ .../doris/connector/iceberg/IcebergConf.java | 78 +++ .../connector/iceberg/IcebergConnector.java | 154 ++--- .../iceberg/IcebergConnectorMetadata.java | 46 +- .../iceberg/IcebergConnectorProperties.java | 189 ------ .../iceberg/IcebergConnectorProvider.java | 48 +- .../iceberg/IcebergScanPlanProvider.java | 52 +- .../iceberg/IcebergWritePlanProvider.java | 22 +- .../IcebergBuildTableDescriptorTest.java | 8 +- .../iceberg/IcebergCatalogFactoryTest.java | 160 +++-- .../IcebergCatalogOptionsSnapshotTest.java | 607 ++++++++++++++++++ .../iceberg/IcebergCatalogPropertiesTest.java | 160 +++++ .../iceberg/IcebergConnectorCacheTest.java | 54 +- .../iceberg/IcebergConnectorConfTest.java | 66 +- ...gConnectorDeriveStoragePropertiesTest.java | 4 +- ...gConnectorMetadataColumnEvolutionTest.java | 4 +- .../IcebergConnectorMetadataDdlTest.java | 72 +-- .../IcebergConnectorMetadataMvccTest.java | 12 +- ...nnectorMetadataPartitionViewCacheTest.java | 4 +- ...cebergConnectorMetadataStatisticsTest.java | 4 +- .../IcebergConnectorMetadataSysTableTest.java | 8 +- .../iceberg/IcebergConnectorMetadataTest.java | 6 +- .../IcebergConnectorTestConnectionTest.java | 16 +- .../IcebergConnectorTransactionTest.java | 4 +- .../IcebergCreateTableValidationTest.java | 2 +- .../iceberg/IcebergLiveConnectivityTest.java | 2 +- .../IcebergProviderSessionRoutingTest.java | 6 +- ...ergScanPlanProviderClassifyColumnTest.java | 2 +- ...ergScanPlanProviderKerberosScanIoTest.java | 10 +- ...ergScanPlanProviderPartitionCountTest.java | 4 +- .../iceberg/IcebergScanPlanProviderTest.java | 198 +++--- .../iceberg/IcebergWritePlanProviderTest.java | 24 +- fe/fe-connector/fe-connector-jdbc/pom.xml | 7 + .../connector/jdbc/JdbcCatalogProperties.java | 418 ++++++++++++ .../apache/doris/connector/jdbc/JdbcConf.java | 67 ++ .../connector/jdbc/JdbcConnectorMetadata.java | 55 +- .../jdbc/JdbcConnectorProperties.java | 104 --- .../connector/jdbc/JdbcConnectorProvider.java | 150 +---- .../connector/jdbc/JdbcDorisConnector.java | 133 ++-- .../connector/jdbc/JdbcScanPlanProvider.java | 50 +- .../connector/jdbc/JdbcWritePlanProvider.java | 43 +- .../jdbc/JdbcCatalogPropertiesTest.java | 271 ++++++++ .../jdbc/JdbcConnectorMetadataTest.java | 17 +- .../JdbcConnectorProviderValidateTest.java | 5 +- .../jdbc/JdbcDorisConnectorTest.java | 14 +- .../jdbc/JdbcScanRangeAndPropertiesTest.java | 57 -- .../connector/jdbc/JdbcUrlNormalizerTest.java | 37 +- .../jdbc/JdbcWritePlanProviderTest.java | 27 +- .../fe-connector-maxcompute/pom.xml | 7 + .../maxcompute/MCCatalogProperties.java | 496 ++++++++++++++ .../maxcompute/MCConnectorClientFactory.java | 110 +--- .../maxcompute/MCConnectorEndpoint.java | 33 +- .../maxcompute/MCConnectorProperties.java | 101 --- .../MaxComputeConnectorProvider.java | 110 +--- .../maxcompute/MaxComputeDorisConnector.java | 62 +- .../MaxComputeScanPlanProvider.java | 45 +- .../MaxComputeWritePlanProvider.java | 18 +- .../maxcompute/MCCatalogPropertiesTest.java | 257 ++++++++ .../maxcompute/MCTestProperties.java | 55 ++ .../MaxComputeConnectorContractTest.java | 8 +- .../MaxComputeConnectorProviderTest.java | 92 +-- .../maxcompute/MaxComputeScanRangeTest.java | 8 +- .../MaxComputeWritePlanProviderTest.java | 4 +- .../OdpsClassloaderIsolationTest.java | 16 +- .../maxcompute/OdpsLiveConnectivityTest.java | 8 +- .../glue/IcebergGlueMetaStoreProperties.java | 46 ++ .../jdbc/IcebergJdbcMetaStoreProperties.java | 73 +++ .../rest/IcebergRestMetaStoreProperties.java | 157 ++++- .../hms/PaimonHmsMetaStoreProperties.java | 40 ++ .../jdbc/PaimonJdbcMetaStoreProperties.java | 22 + .../rest/PaimonRestMetaStoreProperties.java | 19 +- .../spi/AbstractHmsMetaStoreProperties.java | 14 + .../spi/AbstractMetaStoreProperties.java | 11 + .../metastore/spi/MetaStoreParseUtils.java | 14 - .../spi/MetaStoreParseUtilsTest.java | 7 - fe/fe-connector/fe-connector-paimon/pom.xml | 9 + .../paimon/PaimonCatalogFactory.java | 115 ++-- .../paimon/PaimonCatalogProperties.java | 214 ++++++ .../doris/connector/paimon/PaimonConf.java | 84 +++ .../connector/paimon/PaimonConnector.java | 66 +- .../paimon/PaimonConnectorMetadata.java | 27 +- .../paimon/PaimonConnectorProperties.java | 135 ---- .../paimon/PaimonConnectorProvider.java | 73 +-- .../paimon/PaimonScanPlanProvider.java | 39 +- .../paimon/PaimonBackendBoundTableTest.java | 4 +- .../PaimonBuildTableDescriptorTest.java | 2 +- .../paimon/PaimonCatalogFactoryTest.java | 30 +- .../PaimonCatalogOptionsSnapshotTest.java | 273 ++++++++ .../paimon/PaimonCatalogPropertiesTest.java | 164 +++++ .../paimon/PaimonConnectorConfTest.java | 43 +- .../PaimonConnectorMetadataDbDdlTest.java | 6 +- .../PaimonConnectorMetadataDdlTest.java | 2 +- .../PaimonConnectorMetadataMvccTest.java | 4 +- ...monConnectorMetadataOptionsTagPinTest.java | 2 +- .../PaimonConnectorMetadataPartitionTest.java | 4 +- ...nnectorMetadataPartitionViewCacheTest.java | 2 +- .../PaimonConnectorMetadataReadAuthTest.java | 2 +- ...PaimonConnectorMetadataStatisticsTest.java | 4 +- .../PaimonConnectorMetadataSysTableTest.java | 2 +- .../paimon/PaimonConnectorMetadataTest.java | 10 +- .../PaimonCreateTableValidationTest.java | 2 +- .../paimon/PaimonLiveConnectivityTest.java | 4 +- .../paimon/PaimonScanExplainTest.java | 2 +- .../PaimonScanPlanProviderCapabilityTest.java | 8 +- .../paimon/PaimonScanPlanProviderTest.java | 150 +++-- .../paimon/PaimonScanRangeBucketTest.java | 2 +- .../PaimonTableHandleScanOptionsTest.java | 2 +- .../paimon/PaimonTableOptionsTest.java | 2 +- .../doris/connector/spi/package-info.java | 2 +- fe/fe-connector/fe-connector-trino/pom.xml | 7 + .../doris/connector/trino/TrinoBootstrap.java | 12 +- .../trino/TrinoCatalogProperties.java | 153 +++++ .../doris/connector/trino/TrinoConf.java | 53 ++ .../trino/TrinoConnectorProvider.java | 21 +- .../connector/trino/TrinoDorisConnector.java | 56 +- .../connector/trino/TrinoBootstrapTest.java | 30 +- .../trino/TrinoCatalogPropertiesTest.java | 176 +++++ .../connector/DefaultConnectorContext.java | 2 +- .../ConnectorPropertiesUtilsTest.java | 70 ++ .../auth_call/test_hive_base_case_auth.groovy | 6 +- .../tvf/test_catalogs_tvf.groovy | 6 + 215 files changed, 8362 insertions(+), 3677 deletions(-) create mode 100644 fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcCatalogProperties.java create mode 100644 fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConf.java delete mode 100644 fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcCatalogPropertiesTest.java delete mode 100644 fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcConnectorPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsCatalogProperties.java delete mode 100644 fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HmsConf.java create mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveTestProperties.java create mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HmsConfTest.java create mode 100644 fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiCatalogProperties.java delete mode 100644 fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTestProperties.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConf.java delete mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogOptionsSnapshotTest.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcCatalogProperties.java create mode 100644 fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConf.java delete mode 100644 fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MCCatalogProperties.java delete mode 100644 fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MCConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MCCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-maxcompute/src/test/java/org/apache/doris/connector/maxcompute/MCTestProperties.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConf.java delete mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProperties.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogOptionsSnapshotTest.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogPropertiesTest.java create mode 100644 fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoCatalogProperties.java create mode 100644 fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConf.java create mode 100644 fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoCatalogPropertiesTest.java diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index 198dbe5a884b61..b455e1c0bfcda9 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -186,7 +186,8 @@ metastore/shade/cache). For a write path, the richest example is module plus its services line. 6. **Property ownership.** Metadata-connection properties are parsed in your connector (or the metastore layer). Storage properties belong to - `fe-filesystem`. Do not add parsing to fe-core — rule 2 above. + `fe-filesystem`. Do not add parsing to fe-core — rule 2 above. How to + organize the keys themselves: see "Property Organization" below. 7. **Caching.** Reuse `fe-connector-cache` (example: `PaimonLatestSnapshotCache`). Bundle the caching library into your plugin zip and keep shared third-party versions aligned with the other consumers @@ -234,12 +235,82 @@ metastore/shade/cache). For a write path, the richest example is 15. **Packaging.** Add `src/main/assembly/plugin-zip.xml` (copy from es or paimon). Verify your module through `package`/`install`, not just `test-compile` — shades and the plugin zip only materialize then. -15. **Gates and e2e.** Your module must pass the forbidden-import gate (runs +16. **Gates and e2e.** Your module must pass the forbidden-import gate (runs automatically; see `AGENTS.md`). Add a docker environment under `docker/thirdparties/docker-compose/` and regression suites under `regression-test/suites/external_table_p0` / `regression-test/suites/external_table_p2`. +## Property Organization + +Two named classes per connector, nothing else: + +- **`CatalogProperties`** — everything a user writes in `CREATE CATALOG`. + A typed holder: `@ConnectorProperty` fields bound by + `ConnectorPropertiesUtils`, plus derived read-only fields (enums, prefix maps, + bounded numbers), plus the raw map, plus the key-name constants. `of(map)` + binds, derives and validates in one step and throws `IllegalArgumentException` + on a bad value, so an instance that exists has valid properties — which is + what lets every reader downstream take a getter instead of parsing the map + again. `ConnectorProvider.validateProperties` is then exactly one line, + `XxxCatalogProperties.of(properties)`, and that same line guards ALTER through + the SPI default `validatePropertiesForUpdate`. +- **`Conf`** — the keys of the plugin's own `.conf` (item 14 above), + their defaults, their legacy fe.conf fallback keys, and small static readers + wrapping `ConnectorConf.get`. Only create it if the connector really has such + settings. + +Rules for `of(map)`: + +- **No I/O and no heavyweight types.** It runs at CREATE, at ALTER validation, + and on every connector rebuild — including on an FE replaying the edit log, + where reaching the filesystem would let a missing file stop FE from starting. + Remote and filesystem checks belong in `Connector#preCreateValidation`, which + runs on interactive CREATE only; note that ALTER does not run it, so those + checks are deferred to first access after an ALTER. +- **Reject bad values, never unknown keys.** The catalog property map also + carries engine keys (`type`, `meta.cache.*`, ...) and storage keys (`s3.*`, + ...), and `ALTER CATALOG` merges properties — it can overwrite a key but never + remove one, so a rejected unknown key would leave a catalog that no statement + can repair. +- **Required-ness is expressed by `ParamRules.require` alone.** The annotation's + `required` attribute is not read by the connector binder; set it truthfully as + documentation and mark optional fields `required = false` explicitly. Mind the + polarity of `ParamRules.check(condition, message)`: it throws when the + condition *holds*, so the lambda states the failure case — the opposite of + Guava's `checkArgument`. Reversing it compiles cleanly and inverts the check. +- **Mark secrets `sensitive = true`** and implement `toString()` as + `ConnectorPropertiesUtils.toMaskedString(this)`. That covers logs only; the + `SHOW CREATE CATALOG` masking is a separate, hand-maintained list in fe-core + (`DatasourcePrintableMap.SENSITIVE_KEY`). +- **Never pass the raw map into anything that can reach user-visible output.** + It holds the credentials. `ConnectorTableSchema`'s table properties are the + trap: `SHOW CREATE TABLE` renders them unmasked, and the only thing keeping + that rendering away from a given connector is that the connector does not + declare the `SUPPORTS_SHOW_CREATE_DDL` capability. + +Everything else — session variable names, DDL statement keys, remote +table-parameter keys, outbound SDK/BE payload keys, value literals — lives next +to its single reader, not in a properties class. + +Binding and annotation reflection must happen inside the plugin. +`org.apache.doris.foundation.` is child-first for the connector family (it is +parent-first for the filesystem family), so each plugin carries its own +`@ConnectorProperty` class, and fe-core reflecting on a plugin object would find +no annotations at all — silently, since `getAnnotation` just returns null. +Referencing another plugin's key therefore means copying the literal with a +comment naming the owner; `IcebergSiblingProperties` in the hive connector is +the precedent. + +When migrating a connector that is already released, audit every numeric key +first. The binder throws on a malformed number, while the hand-written `getInt` +helpers it replaces silently fell back to the default — so a stored dirty value +becomes "catalog unusable until `ALTER CATALOG` overwrites it". List each such +key in the PR description and decide strict (the default) or lenient (bind as +String, convert inside `of()`, log a warning) explicitly. + +`fe-connector-adbc` is the reference implementation of both classes. + ## Testing and Verification - **Unit tests** live in each module; build/test recipes are in `AGENTS.md`. diff --git a/fe/fe-connector/fe-connector-adbc/pom.xml b/fe/fe-connector/fe-connector-adbc/pom.xml index e604ef8fa99720..4a8af27d1c4722 100644 --- a/fe/fe-connector/fe-connector-adbc/pom.xml +++ b/fe/fe-connector/fe-connector-adbc/pom.xml @@ -55,6 +55,13 @@ under the License. provided + + + ${project.groupId} + fe-foundation + ${project.version} + + + + ${project.groupId} + fe-foundation + ${project.version} + + ${project.groupId} diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsCatalogProperties.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsCatalogProperties.java new file mode 100644 index 00000000000000..576c8a280a0c92 --- /dev/null +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsCatalogProperties.java @@ -0,0 +1,281 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.es; + +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; +import org.apache.doris.foundation.property.ParamRules; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Everything a user writes in {@code CREATE CATALOG} for an Elasticsearch catalog, bound and checked. + * + *

fe-core parses no connector properties, so this class is where they are interpreted. It carries + * the longest compatibility history of any connector here: keys reached Doris first as "Doris On ES" + * table properties, and both an {@code elasticsearch.} prefix and four older names are still accepted. + * The prefix is stripped on the way in ({@link #stripEsPrefix}) because it applies to every key alike; + * the four renames are expressed as {@code names} on the field that owns them, so each alias sits next + * to the property it is an alias for. + * + *

{@link #of(Map)} strips, binds, derives and validates in one step, so an instance that exists has + * valid properties and every reader downstream uses a getter. It performs no I/O and is idempotent: it + * runs at {@code CREATE CATALOG}, again on the merged candidate when {@code ALTER CATALOG} validates, + * and once more every time the connector is rebuilt -- including on an FE replaying the edit log. + * Checks that do need the cluster stay in {@code EsConnector#testConnection}. + * + *

Unknown keys are accepted, always. The same map carries engine keys ({@code type}, + * {@code meta.cache.*}, ...) and storage keys, and {@code ALTER CATALOG} merges properties: it can + * overwrite a key but never remove one, so a key refused here would leave a catalog that no statement + * could repair. Bad values are refused; unrecognized names are not. + */ +public final class EsCatalogProperties { + + /** + * The prefix every key could carry when these were table properties of a "Doris On ES" table. It is + * stripped from every key, so it is not an alias of any one property and cannot be expressed as one. + */ + public static final String ES_PROPERTIES_PREFIX = "elasticsearch."; + + /** Comma-separated ES HTTP endpoints. A missing scheme is filled in from {@link #HTTP_SSL_ENABLED}. */ + public static final String HOSTS = "hosts"; + + public static final String USER = "user"; + public static final String PASSWORD = "password"; + + /** + * Read values from doc_values instead of _source where the mapping allows it. Also gated per query + * by {@link #MAX_DOCVALUE_FIELDS} and by whether every selected field actually has a doc_values + * mapping. + */ + public static final String DOC_VALUE_SCAN = "doc_values_mode"; + + /** Resolve a {@code text} field's {@code keyword} sub-field so equality can be pushed down. */ + public static final String KEYWORD_SNIFF = "keyword_sniff"; + + /** Ask the cluster for its node topology and scan the data nodes directly. */ + public static final String NODES_DISCOVERY = "nodes_discovery"; + + /** Talk https rather than http. Also decides the scheme filled into a bare {@link #HOSTS} entry. */ + public static final String HTTP_SSL_ENABLED = "http_ssl_enabled"; + + /** Expose the document {@code _id} as a column. */ + public static final String MAPPING_ES_ID = "mapping_es_id"; + + /** Push {@code LIKE} down as an ES wildcard query. */ + public static final String LIKE_PUSH_DOWN = "like_push_down"; + + /** List indices whose name starts with a dot, which ES hides by default. */ + public static final String INCLUDE_HIDDEN_INDEX = "include_hidden_index"; + + /** + * The mapping type to read, for the ES versions that still had one. Absent is meaningful -- + * it selects the type-less (ES 7+) mapping shape and leaves the type out of the BE payload -- so + * this one property defaults to {@code null} rather than to the empty string. + */ + public static final String MAPPING_TYPE = "mapping_type"; + + /** The most selected fields a query may have before doc_values reading is given up on. */ + public static final String MAX_DOCVALUE_FIELDS = "max_docvalue_fields"; + + // Older names, each expressed as an alias on the field it renames. Two of them win over the current + // name and two lose to it -- see the fields for why the orders differ. + private static final String LEGACY_HTTP_SSL_ENABLED = "ssl"; + private static final String LEGACY_USER = "username"; + private static final String LEGACY_DOC_VALUE_SCAN = "enable_docvalue_scan"; + private static final String LEGACY_KEYWORD_SNIFF = "enable_keyword_sniff"; + + @ConnectorProperty(names = {HOSTS}, description = "comma-separated ES HTTP endpoints") + private String hosts; + + // "username" first: before this class the rename overwrote whatever "user" held, so a catalog + // carrying both has always been served by the older name. Reversing it now would silently switch + // which credential such a catalog logs in with. + @ConnectorProperty(names = {LEGACY_USER, USER}, required = false, description = "ES user name") + private String user = ""; + + @ConnectorProperty(names = {PASSWORD}, required = false, sensitive = true, description = "ES password") + private String password = ""; + + // "ssl" first, for the same reason as "username" above. + @ConnectorProperty(names = {LEGACY_HTTP_SSL_ENABLED, HTTP_SSL_ENABLED}, required = false, + description = "talk https rather than http") + private boolean httpSslEnabled; + + // Current name first: this rename only ever applied when the current name was absent, so the + // current name has always won. + @ConnectorProperty(names = {DOC_VALUE_SCAN, LEGACY_DOC_VALUE_SCAN}, required = false, + description = "read values from doc_values where the mapping allows it") + private boolean docValuesMode = true; + + // Current name first, as for doc_values_mode above. + @ConnectorProperty(names = {KEYWORD_SNIFF, LEGACY_KEYWORD_SNIFF}, required = false, + description = "resolve a text field's keyword sub-field for pushdown") + private boolean keywordSniff = true; + + @ConnectorProperty(names = {NODES_DISCOVERY}, required = false, + description = "scan the data nodes directly, using the cluster's node topology") + private boolean nodesDiscovery = true; + + @ConnectorProperty(names = {MAPPING_ES_ID}, required = false, + description = "expose the document _id as a column") + private boolean mappingEsId; + + @ConnectorProperty(names = {LIKE_PUSH_DOWN}, required = false, + description = "push LIKE down as an ES wildcard query") + private boolean likePushDown = true; + + @ConnectorProperty(names = {INCLUDE_HIDDEN_INDEX}, required = false, + description = "list indices whose name starts with a dot") + private boolean includeHiddenIndex; + + // No initializer on purpose: null means "type-less mapping", which is a different shape, not a + // default value. See MAPPING_TYPE above. + @ConnectorProperty(names = {MAPPING_TYPE}, required = false, + description = "the mapping type to read, for ES versions that had one") + private String mappingType; + + @ConnectorProperty(names = {MAX_DOCVALUE_FIELDS}, required = false, + description = "the most selected fields a query may have before doc_values reading is given up") + private int maxDocValueFields = 20; + + private String[] hostUrls; + private String[] seeds; + + private EsCatalogProperties() { + } + + public static EsCatalogProperties of(Map properties) { + EsCatalogProperties p = new EsCatalogProperties(); + ConnectorPropertiesUtils.bindConnectorProperties(p, stripEsPrefix(properties)); + new ParamRules() + .require(p.hosts, "Required property '" + HOSTS + "' is missing") + .validate(); + p.seeds = p.hosts.split(","); + p.hostUrls = fillUrlsWithSchema(p.hosts.trim().split(","), p.httpSslEnabled); + return p; + } + + /** + * Returns the properties with {@link #ES_PROPERTIES_PREFIX} removed from every key that carries it. + * + *

Where a catalog somehow holds both spellings of one key, the unprefixed one wins. Before this + * class the winner was whichever spelling the hash map's iteration happened to reach last, i.e. not + * a decision anyone made; picking the current spelling makes it one. + */ + private static Map stripEsPrefix(Map properties) { + Map stripped = new LinkedHashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + if (!entry.getKey().startsWith(ES_PROPERTIES_PREFIX)) { + stripped.put(entry.getKey(), entry.getValue()); + } + } + for (Map.Entry entry : properties.entrySet()) { + if (entry.getKey().startsWith(ES_PROPERTIES_PREFIX)) { + stripped.putIfAbsent(entry.getKey().substring(ES_PROPERTIES_PREFIX.length()), + entry.getValue()); + } + } + return stripped; + } + + /** Fills in http:// or https:// on any host written without a scheme. */ + private static String[] fillUrlsWithSchema(String[] urls, boolean ssl) { + String schema = ssl ? "https://" : "http://"; + for (int i = 0; i < urls.length; i++) { + String trimmed = urls[i].trim(); + if (!trimmed.startsWith("http://") && !trimmed.startsWith("https://")) { + urls[i] = schema + trimmed; + } else { + urls[i] = trimmed; + } + } + return urls; + } + + public String getHosts() { + return hosts; + } + + /** + * The hosts as URLs the REST client can call: trimmed and given a scheme. A fresh array per call, + * since the caller owns it. + */ + public String[] getHostUrls() { + return hostUrls.clone(); + } + + /** + * The hosts as node-discovery seeds -- split only, deliberately not the same derivation as + * {@link #getHostUrls()}, which is what the discovery path has always been given. + */ + public String[] getSeeds() { + return seeds.clone(); + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public boolean isHttpSslEnabled() { + return httpSslEnabled; + } + + public boolean isDocValuesMode() { + return docValuesMode; + } + + public boolean isKeywordSniff() { + return keywordSniff; + } + + public boolean isNodesDiscovery() { + return nodesDiscovery; + } + + public boolean isMappingEsId() { + return mappingEsId; + } + + public boolean isLikePushDown() { + return likePushDown; + } + + public boolean isIncludeHiddenIndex() { + return includeHiddenIndex; + } + + /** Null when the catalog names no mapping type, which is its own shape -- see {@link #MAPPING_TYPE}. */ + public String getMappingType() { + return mappingType; + } + + public int getMaxDocValueFields() { + return maxDocValueFields; + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnector.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnector.java index 12d4948521f474..0a6a6118667118 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnector.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnector.java @@ -39,23 +39,23 @@ public class EsConnector implements Connector, ConnectorRestPassthrough { private static final Logger LOG = LogManager.getLogger(EsConnector.class); - private final Map properties; + private final EsCatalogProperties props; private final ConnectorContext context; private volatile EsConnectorRestClient restClient; public EsConnector(Map properties, ConnectorContext context) { - this.properties = EsConnectorProperties.processCompatible(properties); + this.props = EsCatalogProperties.of(properties); this.context = context; } @Override public ConnectorMetadata getMetadata(ConnectorSession session) { - return new EsConnectorMetadata(getOrCreateRestClient(), properties); + return new EsConnectorMetadata(getOrCreateRestClient(), props); } @Override public ConnectorScanPlanProvider getScanPlanProvider() { - return new EsScanPlanProvider(getOrCreateRestClient(), properties); + return new EsScanPlanProvider(getOrCreateRestClient(), props); } @Override @@ -93,23 +93,9 @@ private EsConnectorRestClient getOrCreateRestClient() { if (restClient == null) { synchronized (this) { if (restClient == null) { - String hosts = properties.getOrDefault( - EsConnectorProperties.HOSTS, ""); - String sslEnabled = properties.getOrDefault( - EsConnectorProperties.HTTP_SSL_ENABLED, - EsConnectorProperties.HTTP_SSL_ENABLED_DEFAULT); - String[] hostUrls = hosts.trim().split(","); - EsConnectorProperties.fillUrlsWithSchema( - hostUrls, Boolean.parseBoolean(sslEnabled)); - - String user = properties.getOrDefault( - EsConnectorProperties.USER, ""); - String password = properties.getOrDefault( - EsConnectorProperties.PASSWORD, ""); - boolean ssl = Boolean.parseBoolean(sslEnabled); restClient = new EsConnectorRestClient( - hostUrls, user, password, ssl, - context.getHttpSecurityHook()); + props.getHostUrls(), props.getUser(), props.getPassword(), + props.isHttpSslEnabled(), context.getHttpSecurityHook()); } } } diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java index 69add4e1e41377..2e41db387acf31 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorMetadata.java @@ -21,7 +21,6 @@ import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorTableSchema; -import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.connector.spi.handle.ConnectorColumnHandle; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.handle.NamedColumnHandle; @@ -43,7 +42,7 @@ public class EsConnectorMetadata implements ConnectorMetadata { public static final String DEFAULT_DB = "default_db"; private final EsConnectorRestClient restClient; - private final Map properties; + private final EsCatalogProperties props; // ES-F3 per-statement schema memo. This metadata instance is created fresh per statement // (funnel-memoized one-per-statement), so an index's mapping is resolved into columns once and @@ -54,9 +53,9 @@ public class EsConnectorMetadata implements ConnectorMetadata { private final Map schemaMemo = new ConcurrentHashMap<>(); public EsConnectorMetadata(EsConnectorRestClient restClient, - Map properties) { + EsCatalogProperties props) { this.restClient = restClient; - this.properties = properties; + this.props = props; } @Override @@ -71,10 +70,7 @@ public boolean databaseExists(ConnectorSession session, String dbName) { @Override public List listTableNames(ConnectorSession session, String dbName) { - boolean includeHidden = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.INCLUDE_HIDDEN_INDEX, - EsConnectorProperties.INCLUDE_HIDDEN_INDEX_DEFAULT)); - return restClient.listTable(includeHidden); + return restClient.listTable(props.isIncludeHiddenIndex()); } @Override @@ -97,12 +93,8 @@ public ConnectorTableSchema getTableSchema( // collapses repeat getTableSchema calls within this metadata instance. String mapping = EsStatementScope.sharedIndexMapping( session, idx, () -> restClient.getMapping(idx)); - boolean mappingEsId = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.MAPPING_ES_ID, - EsConnectorProperties.MAPPING_ES_ID_DEFAULT)); - List columns = EsTypeMapping.parseMapping( - idx, mapping, mappingEsId); + idx, mapping, props.isMappingEsId()); return new ConnectorTableSchema(idx, columns, "ELASTICSEARCH", Collections.emptyMap()); }); @@ -135,18 +127,6 @@ public boolean supportsCastPredicatePushdown(ConnectorSession session) { return true; } - /** - * Validates that required properties are present. - * Called during connector creation. - */ - public static void validateProperties(Map properties) { - if (!properties.containsKey(EsConnectorProperties.HOSTS) - || properties.get(EsConnectorProperties.HOSTS).trim().isEmpty()) { - throw new DorisConnectorException( - "Required property '" + EsConnectorProperties.HOSTS + "' is missing"); - } - } - @Override public org.apache.doris.thrift.TTableDescriptor buildTableDescriptor( ConnectorSession session, @@ -170,16 +150,9 @@ public org.apache.doris.thrift.TTableDescriptor buildTableDescriptor( */ public EsMetadataState fetchMetadataState(ConnectorSession session, String indexName, List columnNames) { - String mappingType = properties.getOrDefault( - EsConnectorProperties.MAPPING_TYPE, null); - boolean nodesDiscovery = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.NODES_DISCOVERY, - EsConnectorProperties.NODES_DISCOVERY_DEFAULT)); - String hostsStr = properties.getOrDefault(EsConnectorProperties.HOSTS, ""); - String[] seeds = hostsStr.split(","); - EsMetadataState state = new EsMetadataState( - indexName, mappingType, columnNames, nodesDiscovery, seeds); + indexName, props.getMappingType(), columnNames, props.isNodesDiscovery(), + props.getSeeds()); EsMetadataFetcher fetcher = new EsMetadataFetcher(restClient, state, session); return fetcher.fetch(); } diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProperties.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProperties.java deleted file mode 100644 index c5257eb7a209f6..00000000000000 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProperties.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.es; - -import java.util.HashMap; -import java.util.Map; - -/** - * ES catalog property constants and utility methods. - * Adapted from fe-core's EsProperties — zero fe-core dependency. - */ -public final class EsConnectorProperties { - - public static final String ES_PROPERTIES_PREFIX = "elasticsearch."; - public static final String HOSTS = "hosts"; - public static final String USER = "user"; - public static final String PASSWORD = "password"; - public static final String DOC_VALUE_SCAN = "doc_values_mode"; - public static final String DOC_VALUE_SCAN_DEFAULT = "true"; - public static final String KEYWORD_SNIFF = "keyword_sniff"; - public static final String KEYWORD_SNIFF_DEFAULT = "true"; - public static final String NODES_DISCOVERY = "nodes_discovery"; - public static final String NODES_DISCOVERY_DEFAULT = "true"; - public static final String HTTP_SSL_ENABLED = "http_ssl_enabled"; - public static final String HTTP_SSL_ENABLED_DEFAULT = "false"; - public static final String MAPPING_ES_ID = "mapping_es_id"; - public static final String MAPPING_ES_ID_DEFAULT = "false"; - public static final String LIKE_PUSH_DOWN = "like_push_down"; - public static final String LIKE_PUSH_DOWN_DEFAULT = "true"; - public static final String INCLUDE_HIDDEN_INDEX = "include_hidden_index"; - public static final String INCLUDE_HIDDEN_INDEX_DEFAULT = "false"; - public static final String MAPPING_TYPE = "mapping_type"; - public static final String MAX_DOCVALUE_FIELDS = "max_docvalue_fields"; - public static final int MAX_DOCVALUE_FIELDS_DEFAULT = 20; - - // Legacy key names (from old EsProperties) that must still be recognized - private static final String LEGACY_DOC_VALUE_SCAN = "enable_docvalue_scan"; - private static final String LEGACY_KEYWORD_SNIFF = "enable_keyword_sniff"; - - private EsConnectorProperties() { - } - - /** - * Processes compatibility transformations from legacy "Doris On ES" properties. - * Strips the "elasticsearch." prefix and maps legacy key names. - */ - public static Map processCompatible(Map props) { - Map result = new HashMap<>(); - for (Map.Entry kv : props.entrySet()) { - String key = kv.getKey(); - if (key.startsWith(ES_PROPERTIES_PREFIX)) { - key = key.substring(ES_PROPERTIES_PREFIX.length()); - } - result.put(key, kv.getValue()); - } - if (result.containsKey("ssl")) { - result.put(HTTP_SSL_ENABLED, result.remove("ssl")); - } - if (result.containsKey("username")) { - result.put(USER, result.remove("username")); - } - // Map legacy property keys to new canonical names - if (result.containsKey(LEGACY_DOC_VALUE_SCAN) && !result.containsKey(DOC_VALUE_SCAN)) { - result.put(DOC_VALUE_SCAN, result.remove(LEGACY_DOC_VALUE_SCAN)); - } - if (result.containsKey(LEGACY_KEYWORD_SNIFF) && !result.containsKey(KEYWORD_SNIFF)) { - result.put(KEYWORD_SNIFF, result.remove(LEGACY_KEYWORD_SNIFF)); - } - return result; - } - - /** - * Fills URL array with http:// or https:// schema if missing. - */ - public static void fillUrlsWithSchema(String[] urls, boolean ssl) { - String schema = ssl ? "https://" : "http://"; - for (int i = 0; i < urls.length; i++) { - String trimmed = urls[i].trim(); - if (!trimmed.startsWith("http://") && !trimmed.startsWith("https://")) { - urls[i] = schema + trimmed; - } else { - urls[i] = trimmed; - } - } - } -} diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProvider.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProvider.java index d9c8de935f3f8d..cc9b8a3a63707d 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProvider.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorProvider.java @@ -42,14 +42,14 @@ public Optional defaultDatabaseOnUse() { return Optional.of(EsConnectorMetadata.DEFAULT_DB); } + /** + * All of it is {@link EsCatalogProperties}: building one strips the legacy prefix, resolves the + * legacy key names, binds and validates, so this door and the connector cannot disagree about what + * a valid catalog is. + */ @Override public void validateProperties(Map properties) { - Map processed = EsConnectorProperties.processCompatible(properties); - String hosts = processed.get(EsConnectorProperties.HOSTS); - if (hosts == null || hosts.trim().isEmpty()) { - throw new IllegalArgumentException( - "Required property '" + EsConnectorProperties.HOSTS + "' is missing"); - } + EsCatalogProperties.of(properties); } @Override diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java index dc94f8c6e47c66..71a343ffc02380 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java @@ -92,7 +92,7 @@ public class EsScanPlanProvider implements ConnectorScanPlanProvider { private static final String SESSION_BATCH_SIZE = "batch_size"; private final EsConnectorRestClient restClient; - private final Map properties; + private final EsCatalogProperties props; // ES-F1 per-scan hoist. planScan and buildScanNodeProperties of one scan node run on the same // per-scan-node provider instance on the synchronous FE planning thread, and each used to fetch @@ -104,9 +104,9 @@ public class EsScanPlanProvider implements ConnectorScanPlanProvider { private EsMetadataState memoizedState; public EsScanPlanProvider(EsConnectorRestClient restClient, - Map properties) { + EsCatalogProperties props) { this.restClient = restClient; - this.properties = properties; + this.props = props; } @Override @@ -122,8 +122,7 @@ public List planScan(ConnectorSession session, ConnectorScan return Collections.emptyList(); } - String mappingType = properties.getOrDefault( - EsConnectorProperties.MAPPING_TYPE, null); + String mappingType = props.getMappingType(); boolean enableParallelScroll = Boolean.parseBoolean( session.getSessionProperties() @@ -196,17 +195,16 @@ private ScanNodePropertiesResult buildScanNodeProperties( nodeProps.put(PROP_ES_INDEX, esHandle.getIndexName()); // Auth properties - String user = properties.getOrDefault(EsConnectorProperties.USER, null); - if (user != null && !user.isEmpty()) { - nodeProps.put(PROP_USER, user); + if (!props.getUser().isEmpty()) { + nodeProps.put(PROP_USER, props.getUser()); } - String password = properties.getOrDefault(EsConnectorProperties.PASSWORD, null); - if (password != null && !password.isEmpty()) { - nodeProps.put(PROP_PASSWORD, password); + if (!props.getPassword().isEmpty()) { + nodeProps.put(PROP_PASSWORD, props.getPassword()); } - nodeProps.put(PROP_HTTP_SSL_ENABLED, properties.getOrDefault( - EsConnectorProperties.HTTP_SSL_ENABLED, - EsConnectorProperties.HTTP_SSL_ENABLED_DEFAULT)); + // Rendered from the parsed flag, not copied from what the user typed: BE parses this with + // std::boolalpha, which only accepts lower-case "true"/"false", while Boolean.parseBoolean here + // accepts any casing. Passing the raw string through let "TRUE" mean https to FE and http to BE. + nodeProps.put(PROP_HTTP_SSL_ENABLED, String.valueOf(props.isHttpSslEnabled())); // Query DSL (with not-pushed conjunct tracking) EsQueryDslResult dslResult = buildQueryDsl(filter, state); @@ -232,21 +230,15 @@ private void serializeFieldContexts(EsMetadataState state, Map n if (state.getFieldContext() == null) { return; } - boolean enableDocValueScan = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.DOC_VALUE_SCAN, - EsConnectorProperties.DOC_VALUE_SCAN_DEFAULT)); - boolean enableKeywordSniff = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.KEYWORD_SNIFF, - EsConnectorProperties.KEYWORD_SNIFF_DEFAULT)); try { - if (enableDocValueScan) { + if (props.isDocValuesMode()) { Map docCtx = state.getFieldContext().getDocValueFieldsContext(); if (docCtx != null && !docCtx.isEmpty()) { nodeProps.put(PROP_DOCVALUE_CONTEXT_JSON, JSON_MAPPER.writeValueAsString(docCtx)); } } - if (enableKeywordSniff) { + if (props.isKeywordSniff()) { Map fieldsCtx = state.getFieldContext().getFetchFieldsContext(); if (fieldsCtx != null && !fieldsCtx.isEmpty()) { nodeProps.put(PROP_FIELDS_CONTEXT_JSON, @@ -269,23 +261,16 @@ private EsQueryDslResult buildQueryDsl(Optional filter, Map column2typeMap = Collections.emptyMap(); if (state.getFieldContext() != null) { - boolean enableKeywordSniff = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.KEYWORD_SNIFF, - EsConnectorProperties.KEYWORD_SNIFF_DEFAULT)); - if (enableKeywordSniff) { + if (props.isKeywordSniff()) { fieldsContext = state.getFieldContext().getFetchFieldsContext(); } needCompatDateFields = state.getFieldContext().getNeedCompatDateFields(); column2typeMap = state.getFieldContext().getColumn2typeMap(); } - boolean likePushDown = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.LIKE_PUSH_DOWN, - EsConnectorProperties.LIKE_PUSH_DOWN_DEFAULT)); - return EsQueryDslBuilder.buildQueryDslWithResult( filter.get(), fieldsContext, column2typeMap, - likePushDown, needCompatDateFields); + props.isLikePushDown(), needCompatDateFields); } private EsMetadataState fetchMetadataState(ConnectorSession session, EsTableHandle handle, @@ -303,16 +288,9 @@ private EsMetadataState fetchMetadataState(ConnectorSession session, EsTableHand return cached; } - String mappingType = properties.getOrDefault( - EsConnectorProperties.MAPPING_TYPE, null); - boolean nodesDiscovery = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.NODES_DISCOVERY, - EsConnectorProperties.NODES_DISCOVERY_DEFAULT)); - String hostsStr = properties.getOrDefault(EsConnectorProperties.HOSTS, ""); - String[] seeds = hostsStr.split(","); - EsMetadataState state = new EsMetadataState( - indexName, mappingType, columnNames, nodesDiscovery, seeds); + indexName, props.getMappingType(), columnNames, props.isNodesDiscovery(), + props.getSeeds()); EsMetadataFetcher fetcher = new EsMetadataFetcher(restClient, state, session); state = fetcher.fetch(); memoizedState = state; @@ -333,10 +311,7 @@ private EsMetadataState fetchMetadataState(ConnectorSession session, EsTableHand */ private int useDocValueScan(List columns, EsMetadataState state) { - boolean enableDocValueScan = Boolean.parseBoolean(properties.getOrDefault( - EsConnectorProperties.DOC_VALUE_SCAN, - EsConnectorProperties.DOC_VALUE_SCAN_DEFAULT)); - if (!enableDocValueScan || state.getFieldContext() == null) { + if (!props.isDocValuesMode() || state.getFieldContext() == null) { return 0; } @@ -349,10 +324,7 @@ private int useDocValueScan(List columns, } // Gate 1: field count limit - int maxDocValueFields = Integer.parseInt(properties.getOrDefault( - EsConnectorProperties.MAX_DOCVALUE_FIELDS, - String.valueOf(EsConnectorProperties.MAX_DOCVALUE_FIELDS_DEFAULT))); - if (selectedFields.size() > maxDocValueFields) { + if (selectedFields.size() > props.getMaxDocValueFields()) { return 0; } diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsCatalogPropertiesTest.java new file mode 100644 index 00000000000000..1ce22b2828d68c --- /dev/null +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsCatalogPropertiesTest.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.es; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Unit tests for {@link EsCatalogProperties}. + * + *

The bulk of this file is the compatibility matrix. This connector's keys reached Doris first as + * "Doris On ES" table properties, so an existing catalog may spell any of them with an + * {@code elasticsearch.} prefix and four of them by an older name — and for two of those four the + * older name wins over the current one while for the other two it loses. That asymmetry is not a + * design, it is what the code has always done, and a catalog created years ago depends on it. Each + * cell below pins one spelling, so a future tidy-up of the alias order fails here rather than silently + * changing which value a stored catalog resolves to. + */ +class EsCatalogPropertiesTest { + + private static Map minimal() { + Map m = new LinkedHashMap<>(); + m.put(EsCatalogProperties.HOSTS, "http://es-1:9200,http://es-2:9200"); + return m; + } + + private static EsCatalogProperties with(String key, String value) { + Map m = minimal(); + m.put(key, value); + return EsCatalogProperties.of(m); + } + + @Test + void bindsEveryKeyAndDefaults() { + EsCatalogProperties p = EsCatalogProperties.of(minimal()); + Assertions.assertEquals("", p.getUser()); + Assertions.assertEquals("", p.getPassword()); + Assertions.assertFalse(p.isHttpSslEnabled()); + Assertions.assertTrue(p.isDocValuesMode()); + Assertions.assertTrue(p.isKeywordSniff()); + Assertions.assertTrue(p.isNodesDiscovery()); + Assertions.assertFalse(p.isMappingEsId()); + Assertions.assertTrue(p.isLikePushDown()); + Assertions.assertFalse(p.isIncludeHiddenIndex()); + Assertions.assertEquals(20, p.getMaxDocValueFields()); + + Map m = minimal(); + m.put(EsCatalogProperties.USER, "u"); + m.put(EsCatalogProperties.PASSWORD, "secret-p"); + m.put(EsCatalogProperties.DOC_VALUE_SCAN, "false"); + m.put(EsCatalogProperties.KEYWORD_SNIFF, "false"); + m.put(EsCatalogProperties.NODES_DISCOVERY, "false"); + m.put(EsCatalogProperties.MAPPING_ES_ID, "true"); + m.put(EsCatalogProperties.LIKE_PUSH_DOWN, "false"); + m.put(EsCatalogProperties.INCLUDE_HIDDEN_INDEX, "true"); + m.put(EsCatalogProperties.MAPPING_TYPE, "_doc"); + m.put(EsCatalogProperties.MAX_DOCVALUE_FIELDS, "5"); + EsCatalogProperties set = EsCatalogProperties.of(m); + Assertions.assertEquals("u", set.getUser()); + Assertions.assertEquals("secret-p", set.getPassword()); + Assertions.assertFalse(set.isDocValuesMode()); + Assertions.assertFalse(set.isKeywordSniff()); + Assertions.assertFalse(set.isNodesDiscovery()); + Assertions.assertTrue(set.isMappingEsId()); + Assertions.assertFalse(set.isLikePushDown()); + Assertions.assertTrue(set.isIncludeHiddenIndex()); + Assertions.assertEquals("_doc", set.getMappingType()); + Assertions.assertEquals(5, set.getMaxDocValueFields()); + } + + // mapping_type absent is not "the default value", it selects the type-less (ES 7+) mapping shape: + // EsMappingUtils.getRootSchema branches on null, and EsScanRange leaves the type out of the BE + // payload entirely. An empty-string default would silently take a catalog down the typed branch. + @Test + void mappingTypeIsNullWhenAbsent() { + Assertions.assertNull(EsCatalogProperties.of(minimal()).getMappingType()); + } + + @Test + void missingHostsFailsNamingTheKey() { + Map m = new LinkedHashMap<>(); + Assertions.assertEquals("Required property '" + EsCatalogProperties.HOSTS + "' is missing", + Assertions.assertThrows(IllegalArgumentException.class, + () -> EsCatalogProperties.of(m)).getMessage()); + + Map blank = minimal(); + blank.put(EsCatalogProperties.HOSTS, " "); + Assertions.assertThrows(IllegalArgumentException.class, () -> EsCatalogProperties.of(blank)); + } + + // --- compatibility matrix: the elasticsearch. prefix --- + + @Test + void everyKeyIsAlsoAcceptedWithTheLegacyPrefix() { + Map m = new LinkedHashMap<>(); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.HOSTS, "http://es:9200"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.USER, "u"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.PASSWORD, "p"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.HTTP_SSL_ENABLED, "true"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.DOC_VALUE_SCAN, "false"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.KEYWORD_SNIFF, "false"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.NODES_DISCOVERY, "false"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.MAPPING_ES_ID, "true"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.LIKE_PUSH_DOWN, "false"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.INCLUDE_HIDDEN_INDEX, "true"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.MAPPING_TYPE, "_doc"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.MAX_DOCVALUE_FIELDS, "7"); + + EsCatalogProperties p = EsCatalogProperties.of(m); + Assertions.assertEquals("http://es:9200", p.getHosts()); + Assertions.assertEquals("u", p.getUser()); + Assertions.assertEquals("p", p.getPassword()); + Assertions.assertTrue(p.isHttpSslEnabled()); + Assertions.assertFalse(p.isDocValuesMode()); + Assertions.assertFalse(p.isKeywordSniff()); + Assertions.assertFalse(p.isNodesDiscovery()); + Assertions.assertTrue(p.isMappingEsId()); + Assertions.assertFalse(p.isLikePushDown()); + Assertions.assertTrue(p.isIncludeHiddenIndex()); + Assertions.assertEquals("_doc", p.getMappingType()); + Assertions.assertEquals(7, p.getMaxDocValueFields()); + } + + @Test + void theLegacyPrefixStacksWithALegacyName() { + Map m = minimal(); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + "ssl", "true"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + "username", "old-user"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + "enable_docvalue_scan", "false"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + "enable_keyword_sniff", "false"); + EsCatalogProperties p = EsCatalogProperties.of(m); + Assertions.assertTrue(p.isHttpSslEnabled()); + Assertions.assertEquals("old-user", p.getUser()); + Assertions.assertFalse(p.isDocValuesMode()); + Assertions.assertFalse(p.isKeywordSniff()); + } + + @Test + void theUnprefixedSpellingWinsWhenACatalogCarriesBoth() { + Map m = new LinkedHashMap<>(); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + EsCatalogProperties.HOSTS, "http://prefixed:9200"); + m.put(EsCatalogProperties.HOSTS, "http://plain:9200"); + Assertions.assertEquals("http://plain:9200", EsCatalogProperties.of(m).getHosts()); + } + + // --- compatibility matrix: the four renamed keys --- + // + // Two of them win over the current name and two lose to it. Before this class that came from how + // the rename was written -- "ssl" and "username" overwrote their target unconditionally, while + // "enable_docvalue_scan" and "enable_keyword_sniff" only filled in an absent one -- and it is now + // the order of the aliases on each field. A catalog that carries both spellings must keep + // resolving to the same value it resolves to today. + + @Test + void sslIsAnAliasOfHttpSslEnabledAndWinsOverIt() { + Assertions.assertTrue(with("ssl", "true").isHttpSslEnabled(), "legacy name alone"); + Assertions.assertTrue(with(EsCatalogProperties.HTTP_SSL_ENABLED, "true").isHttpSslEnabled(), + "current name alone"); + + Map both = minimal(); + both.put("ssl", "true"); + both.put(EsCatalogProperties.HTTP_SSL_ENABLED, "false"); + Assertions.assertTrue(EsCatalogProperties.of(both).isHttpSslEnabled(), + "the legacy name has always overwritten the current one; it must go on winning"); + } + + @Test + void usernameIsAnAliasOfUserAndWinsOverIt() { + Assertions.assertEquals("old", with("username", "old").getUser(), "legacy name alone"); + Assertions.assertEquals("new", with(EsCatalogProperties.USER, "new").getUser(), + "current name alone"); + + Map both = minimal(); + both.put("username", "old"); + both.put(EsCatalogProperties.USER, "new"); + Assertions.assertEquals("old", EsCatalogProperties.of(both).getUser(), + "the legacy name has always overwritten the current one; it must go on winning"); + } + + @Test + void enableDocvalueScanIsAnAliasOfDocValuesModeAndLosesToIt() { + Assertions.assertFalse(with("enable_docvalue_scan", "false").isDocValuesMode(), + "legacy name alone"); + Assertions.assertFalse(with(EsCatalogProperties.DOC_VALUE_SCAN, "false").isDocValuesMode(), + "current name alone"); + + Map both = minimal(); + both.put("enable_docvalue_scan", "false"); + both.put(EsCatalogProperties.DOC_VALUE_SCAN, "true"); + Assertions.assertTrue(EsCatalogProperties.of(both).isDocValuesMode(), + "the rename only ever filled in an absent current name; the current name must go on winning"); + } + + @Test + void enableKeywordSniffIsAnAliasOfKeywordSniffAndLosesToIt() { + Assertions.assertFalse(with("enable_keyword_sniff", "false").isKeywordSniff(), + "legacy name alone"); + Assertions.assertFalse(with(EsCatalogProperties.KEYWORD_SNIFF, "false").isKeywordSniff(), + "current name alone"); + + Map both = minimal(); + both.put("enable_keyword_sniff", "false"); + both.put(EsCatalogProperties.KEYWORD_SNIFF, "true"); + Assertions.assertTrue(EsCatalogProperties.of(both).isKeywordSniff(), + "the rename only ever filled in an absent current name; the current name must go on winning"); + } + + // --- derived host lists --- + + @Test + void hostsGetTheSchemeOfTheSslFlagUnlessTheyCarryOne() { + Map m = new LinkedHashMap<>(); + m.put(EsCatalogProperties.HOSTS, " es-1:9200 , https://es-2:9200 "); + Assertions.assertArrayEquals(new String[] {"http://es-1:9200", "https://es-2:9200"}, + EsCatalogProperties.of(m).getHostUrls()); + + m.put(EsCatalogProperties.HTTP_SSL_ENABLED, "true"); + Assertions.assertArrayEquals(new String[] {"https://es-1:9200", "https://es-2:9200"}, + EsCatalogProperties.of(m).getHostUrls()); + } + + // The discovery seeds are the plain split, deliberately NOT the scheme-filled list: that is what + // the node-discovery path has always been handed, and it parses host:port itself. + @Test + void seedsAreThePlainSplitAndBothListsAreCopies() { + EsCatalogProperties p = EsCatalogProperties.of(minimal()); + Assertions.assertEquals(Arrays.asList("http://es-1:9200", "http://es-2:9200"), + Arrays.asList(p.getSeeds())); + + p.getHostUrls()[0] = "mutated"; + p.getSeeds()[0] = "mutated"; + Assertions.assertEquals("http://es-1:9200", p.getHostUrls()[0]); + Assertions.assertEquals("http://es-1:9200", p.getSeeds()[0]); + } + + // --- the three rules an of() has to obey --- + + // Guards DESIGN D3(2): the map also carries engine keys and storage keys, and ALTER CATALOG merges + // properties -- it can overwrite a key but never remove one, so refusing an unrecognized name would + // leave a catalog that no statement could repair. + @Test + void unknownKeysAreTolerated() { + Map m = minimal(); + m.put("some_future_key", "x"); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + "some_future_key", "y"); + m.put("type", "es"); + Assertions.assertDoesNotThrow(() -> EsCatalogProperties.of(m)); + } + + // Guards DESIGN D3(1): of() runs at CREATE, again on the merged candidate at ALTER, and once more + // every time the connector is rebuilt -- including on an FE replaying the edit log -- so it must be + // a pure function of its input. The prefix strip makes the "does not mutate its input" half of that + // worth checking: it is the one connector whose of() rewrites keys. + @Test + void ofIsPureAndRepeatable() { + Map m = minimal(); + m.put(EsCatalogProperties.ES_PROPERTIES_PREFIX + "ssl", "true"); + Map before = new LinkedHashMap<>(m); + + EsCatalogProperties first = EsCatalogProperties.of(m); + EsCatalogProperties second = EsCatalogProperties.of(m); + + Assertions.assertEquals(before, m, "of() must not mutate the caller's map"); + Assertions.assertEquals(first.getHosts(), second.getHosts()); + Assertions.assertEquals(first.isHttpSslEnabled(), second.isHttpSslEnabled()); + Assertions.assertArrayEquals(first.getHostUrls(), second.getHostUrls()); + } + + // Guards DESIGN D5: toString() is what a log line renders. + @Test + void toStringMasksThePassword() { + String rendered = with(EsCatalogProperties.PASSWORD, "secret-p").toString(); + Assertions.assertFalse(rendered.contains("secret-p"), "got: " + rendered); + Assertions.assertTrue(rendered.contains("password=***"), "got: " + rendered); + } +} diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java index 9385009705bd4c..a316f63ce05a03 100644 --- a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsScanPlanProviderTest.java @@ -153,12 +153,16 @@ public T computeIfAbsent(String key, Supplier loader) { private static final ConnectorSession EMPTY_SESSION = new TestSession(ConnectorStatementScope.NONE); - private static Map minimalProps() { + private static Map minimalPropertyMap() { Map props = new HashMap<>(); - props.put(EsConnectorProperties.HOSTS, "localhost:9200"); + props.put(EsCatalogProperties.HOSTS, "localhost:9200"); return props; } + private static EsCatalogProperties minimalProps() { + return EsCatalogProperties.of(minimalPropertyMap()); + } + @Test void testPlanScanAndScanNodePropertiesShareOneFetch() { CountingRestClient client = new CountingRestClient(); @@ -322,7 +326,7 @@ void testDifferentColumnsRefetch() { @Test void testEsConnectorDoesNotSupportWrite() { - EsConnector connector = new EsConnector(minimalProps(), new ConnectorContext() { + EsConnector connector = new EsConnector(minimalPropertyMap(), new ConnectorContext() { @Override public String getCatalogName() { return "test"; diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index cdbb9fea73244b..2d21c24abc181b 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -47,6 +47,13 @@ under the License. ${project.version} + + + ${project.groupId} + fe-foundation + ${project.version} + + + + ${project.groupId} + fe-foundation + ${project.version} + + ${project.groupId} diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiCatalogProperties.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiCatalogProperties.java new file mode 100644 index 00000000000000..a818029d207c74 --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiCatalogProperties.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; +import org.apache.doris.foundation.property.ParamRules; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The catalog properties this connector interprets, bound and checked. + * + *

None of these keys belong to hudi. There is no {@code type=hudi} catalog (see + * {@link HudiConnectorProvider}): this connector is always an embedded sibling of an HMS gateway and + * receives that gateway catalog's whole property map verbatim, so what it reads are the hive + * catalog's keys. Cross-plugin keys can only be referenced by copying the literal with a comment + * pointing at the owner -- the hive plugin's classes are not visible from here -- which is what the + * constants below do. + * + *

{@link #of(Map)} binds, derives and validates in one step, so an instance that exists has valid + * properties and every reader downstream uses a getter instead of re-parsing the map. It performs no + * I/O and is idempotent: the connector is rebuilt lazily on every catalog refresh and again on an FE + * replaying the edit log. + * + *

Unknown keys are accepted, always. The map is the gateway catalog's, so it carries hive's + * own keys, the engine's ({@code type}, {@code meta.cache.*}) and storage's ({@code s3.*}, + * {@code dfs.*}, ...); and {@code ALTER CATALOG} merges properties -- it can overwrite a key but never + * remove one, so a key refused here would leave a catalog no statement could repair. Bad values + * are refused; unrecognized names are not. + */ +public final class HudiCatalogProperties { + + /** + * The Hive Metastore thrift URI. Owned by the hive connector + * ({@code HiveCatalogProperties.HIVE_METASTORE_URIS}); the literal is copied here because the + * gateway's plugin classes are not visible from this one. + */ + public static final String HIVE_METASTORE_URIS = "hive.metastore.uris"; + + /** The short form of {@link #HIVE_METASTORE_URIS}, accepted as a fallback exactly as hive accepts it. */ + public static final String URI = "uri"; + + /** + * Size of the metastore client pool. Owned by the hive connector + * ({@code HiveCatalogProperties.HMS_CLIENT_POOL_SIZE}) -- see {@link #HIVE_METASTORE_URIS} for why + * the literal is copied. + */ + public static final String HMS_CLIENT_POOL_SIZE = "hive.metastore.client.pool.size"; + + /** + * Take partition names from the metastore (hive-sync) rather than from the table's own filesystem + * layout. Owned by the hive catalog (legacy {@code HMSExternalTable.USE_HIVE_SYNC_PARTITION}). + */ + public static final String USE_HIVE_SYNC_PARTITION = "use_hive_sync_partition"; + + private static final int DEFAULT_HMS_CLIENT_POOL_SIZE = 8; + + @ConnectorProperty(names = {HIVE_METASTORE_URIS, URI}, + description = "Hive Metastore thrift URI; 'uri' is accepted as the short form") + private String metastoreUri; + + @ConnectorProperty(names = {HMS_CLIENT_POOL_SIZE}, required = false, + description = "size of the metastore client pool") + private int hmsClientPoolSize = DEFAULT_HMS_CLIENT_POOL_SIZE; + + @ConnectorProperty(names = {USE_HIVE_SYNC_PARTITION}, required = false, + description = "read partition names from the metastore instead of the table's file layout") + private boolean useHiveSyncPartition; + + private final Map raw; + + private HudiCatalogProperties(Map properties) { + this.raw = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + public static HudiCatalogProperties of(Map properties) { + HudiCatalogProperties p = new HudiCatalogProperties(properties); + ConnectorPropertiesUtils.bindConnectorProperties(p, properties); + new ParamRules() + .require(p.metastoreUri, + "HMS URI ('" + HIVE_METASTORE_URIS + "') is required for Hudi connector") + .validate(); + return p; + } + + public String getMetastoreUri() { + return metastoreUri; + } + + /** + * A malformed value fails here rather than falling back to the default, per the migration rule for + * a typed holder. Note the asymmetry this creates while the hive connector still parses the same key + * leniently: on a catalog whose pool size is misspelled, hive tables keep working and hudi tables do + * not, until an {@code ALTER CATALOG} overwrites the value. + */ + public int getHmsClientPoolSize() { + return hmsClientPoolSize; + } + + public boolean isUseHiveSyncPartition() { + return useHiveSyncPartition; + } + + /** + * The gateway catalog's properties as written, unmodifiable. It is the storage/kerberos passthrough + * the Hadoop {@code Configuration} and the metastore client are built from, so it goes on being + * handed around whole; the keys above are the ones this connector itself interprets. + */ + public Map getRaw() { + return raw; + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index ca6359fd04e6ed..b3ed6548867a52 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -62,8 +62,12 @@ public class HudiConnector implements Connector { private static final Logger LOG = LogManager.getLogger(HudiConnector.class); // Catalog property key gating the plugin-side Kerberos authenticator (value matches AuthType.KERBEROS). + // Deliberately NOT a HudiCatalogProperties field: it is a raw hadoop storage key that buildHadoopConf + // below hands to the Configuration wholesale along with every other passthrough key, and what happens + // here is a peek at it, not this connector interpreting a property of its own. private static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + private final HudiCatalogProperties props; private final Map properties; private final ConnectorContext context; private volatile HmsClient hmsClient; @@ -79,13 +83,14 @@ public class HudiConnector implements Connector { private volatile boolean pluginAuthComputed; public HudiConnector(Map properties, ConnectorContext context) { - this.properties = Collections.unmodifiableMap(properties); + this.props = HudiCatalogProperties.of(properties); + this.properties = props.getRaw(); this.context = context; } @Override public ConnectorMetadata getMetadata(ConnectorSession session) { - return new HudiConnectorMetadata(getOrCreateClient(), properties, metaClientExecutor(), + return new HudiConnectorMetadata(getOrCreateClient(), props, metaClientExecutor(), HudiScanPlanProvider.storageHadoopConfig(context)); } @@ -204,18 +209,10 @@ private HmsClient getOrCreateClient() { } private HmsClient createClient() { - String metastoreUri = properties.get(HudiConnectorProperties.HIVE_METASTORE_URIS); - if (metastoreUri == null || metastoreUri.isEmpty()) { - metastoreUri = properties.get("uri"); - } - if (metastoreUri == null || metastoreUri.isEmpty()) { - throw new DorisConnectorException( - "HMS URI ('" + HudiConnectorProperties.HIVE_METASTORE_URIS + "') is required for Hudi connector"); - } - - int poolSize = HudiConnectorProperties.getInt( - properties, HudiConnectorProperties.HMS_CLIENT_POOL_SIZE, - HudiConnectorProperties.DEFAULT_HMS_CLIENT_POOL_SIZE); + // The URI (either spelling) and the pool size were checked when this connector was constructed -- + // HudiCatalogProperties.of throws for a catalog without a metastore URI, so there is nothing left + // to re-check here. + int poolSize = props.getHmsClientPoolSize(); HmsClientConfig config = new HmsClientConfig(properties, poolSize); LOG.info("Creating Hudi connector HMS client for catalog='{}', uri={}, poolSize={}", diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java index e4711deaed974c..fccc3f579dc9f4 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorMetadata.java @@ -87,9 +87,6 @@ public class HudiConnectorMetadata implements ConnectorMetadata { private static final Logger LOG = LogManager.getLogger(HudiConnectorMetadata.class); - // Catalog property gating the partition-name source (mirrors legacy HMSExternalTable.USE_HIVE_SYNC_PARTITION). - private static final String USE_HIVE_SYNC_PARTITION = "use_hive_sync_partition"; - // fe-core SPI schema property marking which emitted columns are partition columns (CSV of RAW partition-key // names, in declaration order). Mirrors HiveConnectorMetadata.PARTITION_COLUMNS_PROPERTY; fe-core derives the // partition-column set SOLELY from this key (PluginDrivenExternalTable.toSchemaCacheValue). Without it every @@ -152,7 +149,7 @@ public class HudiConnectorMetadata implements ConnectorMetadata { private static final String HUDI_COMMIT_TIME_COLUMN = "_hoodie_commit_time"; private final HmsClient hmsClient; - private final Map properties; + private final HudiCatalogProperties props; // Runs the metaClient-touching partition/snapshot work under the plugin UGI doAs + TCCL pin (see R4). private final HudiMetaClientExecutor metaClientExecutor; // Canonical fs.s3a.*/hadoop.* storage config translated from the catalog's fe-filesystem StorageProperties, @@ -161,15 +158,15 @@ public class HudiConnectorMetadata implements ConnectorMetadata { // getSchemaFromMetaClient or never touch S3). See HudiScanPlanProvider#storageHadoopConfig. private final Map storageHadoopConfig; - public HudiConnectorMetadata(HmsClient hmsClient, Map properties, + public HudiConnectorMetadata(HmsClient hmsClient, HudiCatalogProperties props, HudiMetaClientExecutor metaClientExecutor) { - this(hmsClient, properties, metaClientExecutor, Collections.emptyMap()); + this(hmsClient, props, metaClientExecutor, Collections.emptyMap()); } - public HudiConnectorMetadata(HmsClient hmsClient, Map properties, + public HudiConnectorMetadata(HmsClient hmsClient, HudiCatalogProperties props, HudiMetaClientExecutor metaClientExecutor, Map storageHadoopConfig) { this.hmsClient = hmsClient; - this.properties = properties; + this.props = props; this.metaClientExecutor = metaClientExecutor; this.storageHadoopConfig = storageHadoopConfig; } @@ -787,7 +784,7 @@ long latestInstantMillis(HudiTableHandle handle) { } private boolean useHiveSyncPartition() { - return Boolean.parseBoolean(properties.getOrDefault(USE_HIVE_SYNC_PARTITION, "false")); + return props.isUseHiveSyncPartition(); } /** @@ -1027,7 +1024,7 @@ private Configuration buildHadoopConf() { // Storage credentials (fs.s3a.* …) first so an inline user fs./dfs./hadoop. key still wins below; see // storageHadoopConfig field. Without it the metaClient's S3AFileSystem cannot read hoodie.properties. storageHadoopConfig.forEach(conf::set); - for (Map.Entry entry : properties.entrySet()) { + for (Map.Entry entry : props.getRaw().entrySet()) { String key = entry.getKey(); if (key.startsWith("hadoop.") || key.startsWith("fs.") || key.startsWith("dfs.") || key.startsWith("hive.")) { diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorProperties.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorProperties.java deleted file mode 100644 index b49eba0adcc3a4..00000000000000 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnectorProperties.java +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hudi; - -import java.util.Map; - -/** - * Property constants for Hudi connector configuration. - * Mirrors keys from fe-core's Hudi/HMS property classes without taking - * a compile-time dependency on fe-core. - */ -public final class HudiConnectorProperties { - - private HudiConnectorProperties() { - } - - // -- HMS connection -- - public static final String HIVE_METASTORE_URIS = "hive.metastore.uris"; - - // -- HMS client pool -- - public static final String HMS_CLIENT_POOL_SIZE = "hive.metastore.client.pool.size"; - public static final int DEFAULT_HMS_CLIENT_POOL_SIZE = 8; - - // -- Hudi specific -- - public static final String HUDI_TABLE_TYPE = "hoodie.datasource.write.table.type"; - - /** - * Parse an integer property with a default value. - */ - public static int getInt(Map props, String key, int defaultVal) { - String value = props.get(key); - if (value == null || value.isEmpty()) { - return defaultVal; - } - try { - return Integer.parseInt(value.trim()); - } catch (NumberFormatException e) { - return defaultVal; - } - } -} diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiBackendDescriptorTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiBackendDescriptorTest.java index 83466c78fd6aab..002f7cfbfe8749 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiBackendDescriptorTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiBackendDescriptorTest.java @@ -49,7 +49,7 @@ public class HudiBackendDescriptorTest { @Test public void buildTableDescriptorIsHiveTable() { - HudiConnectorMetadata md = new HudiConnectorMetadata(null, Collections.emptyMap(), + HudiConnectorMetadata md = new HudiConnectorMetadata(null, HudiTestProperties.minimal(), new DirectHudiMetaClientExecutor()); TTableDescriptor desc = md.buildTableDescriptor(null, 7L, "t", "db", "t", 3, 100L); diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiCatalogPropertiesTest.java new file mode 100644 index 00000000000000..8de6fe1f2b652d --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiCatalogPropertiesTest.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Unit tests for {@link HudiCatalogProperties} — the connector's single reader of the gateway catalog's + * property map. + */ +class HudiCatalogPropertiesTest { + + @Test + void bindsEveryKeyAndDefaults() { + Map m = HudiTestProperties.minimalMap(); + HudiCatalogProperties p = HudiCatalogProperties.of(m); + Assertions.assertEquals(HudiTestProperties.METASTORE_URI, p.getMetastoreUri()); + Assertions.assertEquals(8, p.getHmsClientPoolSize()); + Assertions.assertFalse(p.isUseHiveSyncPartition()); + + m.put(HudiCatalogProperties.HMS_CLIENT_POOL_SIZE, "16"); + m.put(HudiCatalogProperties.USE_HIVE_SYNC_PARTITION, "true"); + HudiCatalogProperties set = HudiCatalogProperties.of(m); + Assertions.assertEquals(16, set.getHmsClientPoolSize()); + Assertions.assertTrue(set.isUseHiveSyncPartition()); + } + + // The gateway may spell the metastore URI either way; hive accepts both and the sibling receives + // hive's map verbatim, so a catalog written with the short form must not lose its metastore here. + @Test + void uriShortFormIsTheFallbackAndTheLongFormWins() { + Map shortOnly = new LinkedHashMap<>(); + shortOnly.put(HudiCatalogProperties.URI, "thrift://short:9083"); + Assertions.assertEquals("thrift://short:9083", + HudiCatalogProperties.of(shortOnly).getMetastoreUri()); + + Map both = new LinkedHashMap<>(); + both.put(HudiCatalogProperties.URI, "thrift://short:9083"); + both.put(HudiCatalogProperties.HIVE_METASTORE_URIS, "thrift://long:9083"); + Assertions.assertEquals("thrift://long:9083", + HudiCatalogProperties.of(both).getMetastoreUri()); + } + + @Test + void missingMetastoreUriFailsNamingTheKey() { + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HudiCatalogProperties.of(new LinkedHashMap<>())); + Assertions.assertEquals("HMS URI ('" + HudiCatalogProperties.HIVE_METASTORE_URIS + + "') is required for Hudi connector", e.getMessage()); + } + + @Test + void blankMetastoreUriCountsAsMissing() { + Map m = new LinkedHashMap<>(); + m.put(HudiCatalogProperties.HIVE_METASTORE_URIS, " "); + Assertions.assertThrows(IllegalArgumentException.class, () -> HudiCatalogProperties.of(m)); + } + + // Guards the migration decision recorded on getHmsClientPoolSize(): a malformed pool size is + // refused rather than silently falling back to 8 as the pre-migration helper did. It is the one + // behaviour change of this migration, so it gets a test rather than living only in a PR description. + @Test + void malformedPoolSizeIsRejectedInsteadOfFallingBackToTheDefault() { + Map m = HudiTestProperties.minimalMap(); + m.put(HudiCatalogProperties.HMS_CLIENT_POOL_SIZE, "8x"); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HudiCatalogProperties.of(m)); + Assertions.assertTrue(e.getMessage().contains(HudiCatalogProperties.HMS_CLIENT_POOL_SIZE), + "the error must name the offending key, got: " + e.getMessage()); + } + + // Guards DESIGN D3(2): the map this connector receives is the gateway catalog's whole map — hive's + // keys, the engine's and storage's all live in it — and ALTER CATALOG merges properties, so it can + // overwrite a key but never remove one. Refusing an unrecognized name would create a catalog that no + // statement could repair. + @Test + void unknownKeysAreTolerated() { + Map m = HudiTestProperties.minimalMap(); + m.put("some_future_key", "x"); + m.put("s3.endpoint", "http://minio:9000"); + m.put("type", "hms"); + Assertions.assertDoesNotThrow(() -> HudiCatalogProperties.of(m)); + } + + // Guards DESIGN D3(1): the connector is rebuilt lazily on every refresh and again on an FE replaying + // the edit log, so of() must be a pure function of its input — no I/O, no mutation of the caller's map. + @Test + void ofIsPureAndRepeatable() { + Map m = HudiTestProperties.minimalMap(); + m.put(HudiCatalogProperties.HMS_CLIENT_POOL_SIZE, "16"); + Map before = new LinkedHashMap<>(m); + + HudiCatalogProperties first = HudiCatalogProperties.of(m); + HudiCatalogProperties second = HudiCatalogProperties.of(m); + + Assertions.assertEquals(before, m, "of() must not mutate the caller's map"); + Assertions.assertEquals(first.getMetastoreUri(), second.getMetastoreUri()); + Assertions.assertEquals(first.getHmsClientPoolSize(), second.getHmsClientPoolSize()); + Assertions.assertEquals(first.getRaw(), second.getRaw()); + } + + @Test + void rawIsAnImmutableCopyOfTheInput() { + Map m = HudiTestProperties.minimalMap(); + HudiCatalogProperties p = HudiCatalogProperties.of(m); + m.put("added_afterwards", "v"); + Assertions.assertFalse(p.getRaw().containsKey("added_afterwards")); + Assertions.assertThrows(UnsupportedOperationException.class, () -> p.getRaw().put("k", "v")); + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java index c508d470d0e01b..85d8e68571ba6b 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java @@ -151,7 +151,7 @@ public void invalidateOnUnbuiltClientIsNoOp() { // ── helpers ──────────────────────────────────────────────────────────────────────────────────────────── private static HudiConnector connector() { - return new HudiConnector(Collections.emptyMap(), new ConnectorContext() { + return new HudiConnector(HudiTestProperties.minimalMap(), new ConnectorContext() { @Override public String getCatalogName() { return "test_catalog"; @@ -173,7 +173,7 @@ private static HudiConnectorMetadata hiveSyncMetadata(HmsClient hms) { // hive-sync so collectPartitions lists partition names from HMS (where the fresh/cached split lives); // the stub executor returns the canned instant latestInstant would read off the timeline. return new HudiConnectorMetadata(hms, - Collections.singletonMap("use_hive_sync_partition", "true"), stub(7L)); + HudiTestProperties.with(HudiCatalogProperties.USE_HIVE_SYNC_PARTITION, "true"), stub(7L)); } /** Executor that ignores the action and returns a canned value (stubs out the live metaClient). */ diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java index f1ff13c9904c7e..a7b12b90175891 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorOwnsHandleTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Collections; /** * Pins {@link HudiConnector#ownsHandle} for the hms 3-way sibling routing: a flipped hms gateway embeds this @@ -37,7 +36,7 @@ public class HudiConnectorOwnsHandleTest { private static HudiConnector connector() { - return new HudiConnector(Collections.emptyMap(), new ConnectorContext() { + return new HudiConnector(HudiTestProperties.minimalMap(), new ConnectorContext() { @Override public String getCatalogName() { return "test_catalog"; diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorPartitionListingTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorPartitionListingTest.java index 77b575a51b6748..472f2baccf3273 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorPartitionListingTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorPartitionListingTest.java @@ -346,9 +346,9 @@ private static HudiTableHandle partitioned() { private static HudiConnectorMetadata metadata(boolean useHiveSync, HmsClient hms, HudiMetaClientExecutor executor) { - Map props = useHiveSync - ? Collections.singletonMap("use_hive_sync_partition", "true") - : Collections.emptyMap(); + HudiCatalogProperties props = useHiveSync + ? HudiTestProperties.with(HudiCatalogProperties.USE_HIVE_SYNC_PARTITION, "true") + : HudiTestProperties.minimal(); return new HudiConnectorMetadata(hms, props, executor); } diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiIncrementalTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiIncrementalTest.java index 6d25cc9cd2845b..5442155c378eaf 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiIncrementalTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiIncrementalTest.java @@ -30,7 +30,6 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -345,7 +344,7 @@ public void toBuilderRoundTripsWindowFields() { // ── helpers ──────────────────────────────────────────────────────────────────────────────────────── private static HudiConnectorMetadata metadata(HudiMetaClientExecutor executor) { - return new HudiConnectorMetadata(null, Collections.emptyMap(), executor); + return new HudiConnectorMetadata(null, HudiTestProperties.minimal(), executor); } private static HudiTableHandle partitioned() { diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiPartitionPruningTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiPartitionPruningTest.java index 8802f11a60e15a..f59cec26b7f074 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiPartitionPruningTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiPartitionPruningTest.java @@ -169,7 +169,7 @@ public void testNonHiveStylePositionalPathsPruneToRelativePaths() { // which finds nothing on a non-hive-style table -> 0 splits for any filtered query. HudiConnectorMetadata metadata = new HudiConnectorMetadata( new FakeHmsClient(PARTITIONS), // HMS hive-style names -- must NOT be used here - Collections.emptyMap(), // use_hive_sync_partition=false -> non-hive-sync + HudiTestProperties.minimal(), // use_hive_sync_partition=false -> non-hive-sync new StubMetaClientExecutor(Arrays.asList("2024/01", "2024/02", "2023/12"))); Optional> result = metadata.applyFilter(null, partitionedHandle(), new ConnectorFilterConstraint(eq("year", "2024"))); @@ -183,7 +183,7 @@ public void testHiveSyncBranchPrunesHmsNames() { // storage layout, so applyFilter prunes the HMS names directly (no Hudi metadata listing / stub unused). HudiConnectorMetadata metadata = new HudiConnectorMetadata( new FakeHmsClient(PARTITIONS), - Collections.singletonMap("use_hive_sync_partition", "true"), + HudiTestProperties.with(HudiCatalogProperties.USE_HIVE_SYNC_PARTITION, "true"), new StubMetaClientExecutor(Collections.emptyList())); Optional> result = metadata.applyFilter(null, partitionedHandle(), new ConnectorFilterConstraint(eq("year", "2024"))); @@ -209,7 +209,7 @@ public void testDatePartitionPredicatePrunesUnchanged() { // H2 non-regression: a DATE predicate literal is a LocalDate (not LocalDateTime), so it is NOT diverted to // hiveDateTimeString -- String.valueOf(LocalDate) = "2024-01-01" already matches the stored DATE value. HudiConnectorMetadata metadata = new HudiConnectorMetadata( - new FakeHmsClient(PARTITIONS), Collections.emptyMap(), + new FakeHmsClient(PARTITIONS), HudiTestProperties.minimal(), new StubMetaClientExecutor(Arrays.asList("2024-01-01", "2024-01-02"))); HudiTableHandle handle = new HudiTableHandle.Builder("db", "t", "s3://b/t", "COPY_ON_WRITE") .partitionKeyNames(Collections.singletonList("dt")) @@ -235,7 +235,7 @@ public void testDecimalPartitionPredicatePrunesTrailingZeros() { // literal renders "1.0000", ["1.0000"].contains("1") == false -> partition dropped -> prunedPaths // empty -> red. HudiConnectorMetadata metadata = new HudiConnectorMetadata( - new FakeHmsClient(PARTITIONS), Collections.emptyMap(), + new FakeHmsClient(PARTITIONS), HudiTestProperties.minimal(), new StubMetaClientExecutor(Arrays.asList("1", "2"))); HudiTableHandle handle = new HudiTableHandle.Builder("db", "t", "s3://b/t", "COPY_ON_WRITE") .partitionKeyNames(Collections.singletonList("d")) @@ -257,7 +257,7 @@ private Optional> applyFilter( // metadata listing. Feed the canned partition list via the stub executor (no live metaClient). The // hive-style names here parse the same via parsePartitionValues, so the pruning assertions are unchanged. HudiConnectorMetadata metadata = new HudiConnectorMetadata( - new FakeHmsClient(PARTITIONS), Collections.emptyMap(), + new FakeHmsClient(PARTITIONS), HudiTestProperties.minimal(), new StubMetaClientExecutor(PARTITIONS)); return metadata.applyFilter(null, handle, new ConnectorFilterConstraint(expr)); } diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java index 19d6a49c4dca5c..849181c6e3d342 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiReadOnlyWriteRejectTest.java @@ -44,7 +44,7 @@ public class HudiReadOnlyWriteRejectTest { new HudiTableHandle("db", "t", "s3://b/t", "COPY_ON_WRITE"); private static HudiConnector connector() { - return new HudiConnector(Collections.emptyMap(), new ConnectorContext() { + return new HudiConnector(HudiTestProperties.minimalMap(), new ConnectorContext() { @Override public String getCatalogName() { return "test_catalog"; @@ -59,7 +59,7 @@ public long getCatalogId() { /** The write/DDL methods throw before touching the client/executor, so null collaborators are sufficient. */ private static HudiConnectorMetadata metadata() { - return new HudiConnectorMetadata(null, Collections.emptyMap(), null); + return new HudiConnectorMetadata(null, HudiTestProperties.minimal(), null); } @Test diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiSchemaAtInstantTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiSchemaAtInstantTest.java index ceb59b57f42b0f..5500234d508812 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiSchemaAtInstantTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiSchemaAtInstantTest.java @@ -60,7 +60,7 @@ private static final class RecordingMetadata extends HudiConnectorMetadata { int calls; RecordingMetadata() { - super(null, Collections.emptyMap(), null); + super(null, HudiTestProperties.minimal(), null); } @Override diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java index af41d6653704ee..5342a7bfd0207a 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiStatementMemoTest.java @@ -62,7 +62,7 @@ private static final class RecordingMetadata extends HudiConnectorMetadata { int instantLoads; RecordingMetadata() { - super(null, Collections.emptyMap(), null); + super(null, HudiTestProperties.minimal(), null); } @Override diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTableTypeTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTableTypeTest.java index 8856119789b677..718717b7c9f2e9 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTableTypeTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTableTypeTest.java @@ -51,7 +51,7 @@ private String detect(String inputFormat, Map parameters) { .parameters(parameters) .build(); HudiConnectorMetadata metadata = - new HudiConnectorMetadata(new FakeHmsClient(info), Collections.emptyMap(), + new HudiConnectorMetadata(new FakeHmsClient(info), HudiTestProperties.minimal(), new DirectHudiMetaClientExecutor()); Optional handle = metadata.getTableHandle(null, "db", "t"); Assertions.assertTrue(handle.isPresent()); diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTestProperties.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTestProperties.java new file mode 100644 index 00000000000000..776254abe59879 --- /dev/null +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTestProperties.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hudi; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Minimal valid {@link HudiCatalogProperties} for tests that need a connector or a metadata object but + * are not about properties. The metastore URI is the one key {@code of()} requires, so every such test + * would otherwise repeat it. + */ +final class HudiTestProperties { + + static final String METASTORE_URI = "thrift://localhost:9083"; + + private HudiTestProperties() { + } + + /** A mutable map carrying only the required metastore URI. */ + static Map minimalMap() { + Map m = new LinkedHashMap<>(); + m.put(HudiCatalogProperties.HIVE_METASTORE_URIS, METASTORE_URI); + return m; + } + + static HudiCatalogProperties minimal() { + return HudiCatalogProperties.of(minimalMap()); + } + + /** The minimal properties plus one extra key, for a test that turns a single knob. */ + static HudiCatalogProperties with(String key, String value) { + Map m = minimalMap(); + m.put(key, value); + return HudiCatalogProperties.of(m); + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTimeTravelTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTimeTravelTest.java index 923ad2c03a77c9..4f6099c002ee0a 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTimeTravelTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiTimeTravelTest.java @@ -183,7 +183,7 @@ public void toBuilderRoundTripsQueryInstantAndPreservesEveryField() { /** resolveTimeTravel/applySnapshot never touch hmsClient or the executor → null collaborators are safe. */ private static HudiConnectorMetadata metadata() { - return new HudiConnectorMetadata(null, Collections.emptyMap(), null); + return new HudiConnectorMetadata(null, HudiTestProperties.minimal(), null); } private static HudiTableHandle partitioned() { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/AwsCredentialsProviderModes.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/AwsCredentialsProviderModes.java index 2ce155f02f7c29..b78314c2527760 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/AwsCredentialsProviderModes.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/AwsCredentialsProviderModes.java @@ -69,6 +69,16 @@ static String classNameFor(Map props, String... modeKeys) { return clazz == null ? null : clazz.getName(); } + /** + * Same, for a mode value that has ALREADY been resolved from the properties — the rest flavor binds + * {@code iceberg.rest.credentials_provider_type} into its metastore holder, so re-scanning the raw map + * for it would be a second reader of the same key. Blank/unknown still means the SDK default chain. + */ + static String classNameForMode(String mode) { + Class clazz = classFor(normalizeMode(mode)); + return clazz == null ? null : clazz.getName(); + } + /** * The AWS SDK v2 provider instance for the first non-blank mode key in {@code props}; * {@link DefaultCredentialsProvider} for DEFAULT / blank / unknown. @@ -120,9 +130,17 @@ private static String resolveMode(Map props, String... modeKeys) for (String key : modeKeys) { String value = props.get(key); if (value != null && !value.trim().isEmpty()) { - return value.trim().toUpperCase(Locale.ROOT).replace('-', '_'); + return normalizeMode(value); } } return "DEFAULT"; } + + /** Legacy AwsCredentialsProviderMode.fromString normalization; blank is DEFAULT. */ + private static String normalizeMode(String mode) { + if (mode == null || mode.trim().isEmpty()) { + return "DEFAULT"; + } + return mode.trim().toUpperCase(Locale.ROOT).replace('-', '_'); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java index 62d251db53465e..232623da99ba9d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java @@ -18,6 +18,9 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metastore.iceberg.glue.IcebergGlueMetaStoreProperties; +import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; +import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.filesystem.properties.StorageProperties; @@ -87,6 +90,50 @@ public final class IcebergCatalogFactory { "s3.region", "AWS_REGION", "region", "REGION", "aws.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region", "rest.signing-region", "client.region"}; + // --------------------------------------------------------------------- + // Catalog-option keys and values this factory EMITS that are not iceberg-SDK constants. Keys that ARE + // SDK constants (CatalogProperties / S3FileIOProperties / AwsProperties / AwsClientProperties / + // OAuth2Properties) are referenced through the SDK rather than copied. These are private because this + // class is the only writer of them -- an INPUT key a user writes is declared by the metastore + // properties for its flavor instead. + // --------------------------------------------------------------------- + + // REST. + private static final String REST_PREFIX_KEY = "prefix"; + private static final String REST_VENDED_CREDENTIALS_HEADER = "header.X-Iceberg-Access-Delegation"; + private static final String REST_VENDED_CREDENTIALS_VALUE = "vended-credentials"; + private static final String REST_CONNECTION_TIMEOUT_MS_KEY = "rest.client.connection-timeout-ms"; + private static final String REST_SOCKET_TIMEOUT_MS_KEY = "rest.client.socket-timeout-ms"; + private static final String REST_SIGNING_NAME_KEY = "rest.signing-name"; + private static final String REST_SIGV4_ENABLED_KEY = "rest.sigv4-enabled"; + private static final String REST_SIGNING_REGION_KEY = "rest.signing-region"; + private static final String SECURITY_TYPE_OAUTH2 = "oauth2"; + private static final String SIGNING_NAME_GLUE = "glue"; + private static final String SIGNING_NAME_S3TABLES = "s3tables"; + + // GLUE. + private static final String GLUE_CREDENTIALS_PROVIDER_KEY = "client.credentials-provider"; + private static final String GLUE_CREDENTIALS_PROVIDER_2X = + "org.apache.doris.connector.iceberg.glue.ConfigurationAWSCredentialsProvider2x"; + private static final String GLUE_CREDENTIALS_PROVIDER_ACCESS_KEY = + "client.credentials-provider.glue.access_key"; + private static final String GLUE_CREDENTIALS_PROVIDER_SECRET_KEY = + "client.credentials-provider.glue.secret_key"; + private static final String GLUE_CREDENTIALS_PROVIDER_SESSION_TOKEN = + "client.credentials-provider.glue.session_token"; + private static final String AWS_REGION_KEY = "aws.region"; + private static final String GLUE_CHECKED_WAREHOUSE = "s3://doris"; + private static final String GLUE_DEFAULT_REGION = "us-east-1"; + + // JDBC. iceberg.jdbc.catalog_name is the one INPUT key named here: the assembly does not read it, it + // REMOVES it from the options map (it is passed positionally as the catalog name instead). + private static final String JDBC_CATALOG_NAME = "iceberg.jdbc.catalog_name"; + private static final String JDBC_USER_KEY = "jdbc.user"; + private static final String JDBC_PASSWORD_KEY = "jdbc.password"; + private static final String JDBC_INIT_CATALOG_TABLES_KEY = "jdbc.init-catalog-tables"; + private static final String JDBC_SCHEMA_VERSION_KEY = "jdbc.schema-version"; + private static final String JDBC_STRICT_MODE_KEY = "jdbc.strict-mode"; + private IcebergCatalogFactory() { } @@ -124,9 +171,9 @@ private static void appendManifestCacheProperties(Map props, Map copyIfPresent(props, opts, CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH); if (!hasExplicitEnabled) { CacheSpec spec = CacheSpec.fromProperties(props, - IcebergConnectorProperties.MANIFEST_CACHE_ENABLE, DEFAULT_MANIFEST_CACHE_ENABLE, - IcebergConnectorProperties.MANIFEST_CACHE_TTL, DEFAULT_MANIFEST_CACHE_TTL_SECOND, - IcebergConnectorProperties.MANIFEST_CACHE_CAPACITY, DEFAULT_MANIFEST_CACHE_CAPACITY); + IcebergConnector.MANIFEST_CACHE_ENABLE, DEFAULT_MANIFEST_CACHE_ENABLE, + IcebergConnector.MANIFEST_CACHE_TTL_SECOND, DEFAULT_MANIFEST_CACHE_TTL_SECOND, + IcebergConnector.MANIFEST_CACHE_CAPACITY, DEFAULT_MANIFEST_CACHE_CAPACITY); if (CacheSpec.isCacheEnabled(spec.isEnable(), spec.getTtlSecond(), spec.getCapacity())) { opts.put(CatalogProperties.IO_MANIFEST_CACHE_ENABLED, "true"); } @@ -272,15 +319,6 @@ private static void putIfNotBlank(Map opts, String key, String v } } - /** Resolves the lower-cased flavor from {@code iceberg.catalog.type}; null/blank stays null. */ - public static String resolveFlavor(Map props) { - String catalogType = props.get(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE); - if (catalogType == null || catalogType.isEmpty()) { - return null; - } - return catalogType.toLowerCase(Locale.ROOT); - } - /** * Resolve the Iceberg catalog implementation class name from the catalog type string. PURE: * depends only on {@code catalogType}. Lifted verbatim from the former @@ -289,24 +327,24 @@ public static String resolveFlavor(Map props) { public static String resolveCatalogImpl(String catalogType) { if (catalogType == null) { throw new DorisConnectorException( - "Missing '" + IcebergConnectorProperties.ICEBERG_CATALOG_TYPE + "' property"); + "Missing '" + IcebergCatalogProperties.ICEBERG_CATALOG_TYPE + "' property"); } switch (catalogType.toLowerCase(Locale.ROOT)) { - case IcebergConnectorProperties.TYPE_REST: + case IcebergCatalogProperties.TYPE_REST: return "org.apache.iceberg.rest.RESTCatalog"; - case IcebergConnectorProperties.TYPE_HMS: + case IcebergCatalogProperties.TYPE_HMS: return "org.apache.iceberg.hive.HiveCatalog"; - case IcebergConnectorProperties.TYPE_GLUE: + case IcebergCatalogProperties.TYPE_GLUE: return "org.apache.iceberg.aws.glue.GlueCatalog"; - case IcebergConnectorProperties.TYPE_HADOOP: + case IcebergCatalogProperties.TYPE_HADOOP: return "org.apache.iceberg.hadoop.HadoopCatalog"; - case IcebergConnectorProperties.TYPE_JDBC: + case IcebergCatalogProperties.TYPE_JDBC: return "org.apache.iceberg.jdbc.JdbcCatalog"; - case IcebergConnectorProperties.TYPE_S3_TABLES: + case IcebergCatalogProperties.TYPE_S3_TABLES: return "software.amazon.s3tables.iceberg.S3TablesCatalog"; default: throw new DorisConnectorException( - "Unknown " + IcebergConnectorProperties.ICEBERG_CATALOG_TYPE + ": " + catalogType + "Unknown " + IcebergCatalogProperties.ICEBERG_CATALOG_TYPE + ": " + catalogType + ". Supported types: rest, hms, glue, hadoop, jdbc, s3tables"); } } @@ -325,32 +363,36 @@ public static String resolveCatalogImpl(String catalogType) { * of this options map. {@code s3tables} is bespoke and falls through to the base + * impl only here (the existing skeleton behavior), so this method covers exactly the five SDK-built flavors. */ - public static Map buildCatalogProperties(Map props, String flavor, + public static Map buildCatalogProperties(IcebergCatalogProperties catalogProps, Optional chosenS3) { + Map props = catalogProps.getRaw(); + String flavor = catalogProps.getFlavor(); Map opts = buildBaseCatalogProperties(props); opts.put(CatalogProperties.CATALOG_IMPL, resolveCatalogImpl(flavor)); switch (flavor) { - case IcebergConnectorProperties.TYPE_REST: - appendRestProperties(opts, props, chosenS3); + case IcebergCatalogProperties.TYPE_REST: + // Bound directly rather than through MetaStoreProviders: the flavor is already decided here, and + // this class stays a PURE offline-testable function (a ServiceLoader lookup is neither). + appendRestProperties(opts, IcebergRestMetaStoreProperties.of(props), props, chosenS3); appendS3FileIO(opts, props, chosenS3); break; - case IcebergConnectorProperties.TYPE_GLUE: + case IcebergCatalogProperties.TYPE_GLUE: // glue emits its OWN s3.* (unconditional) + glue-client creds; it does NOT use the base // S3FileIO path (legacy IcebergGlueMetaStoreProperties ignores storagePropertiesList). - appendGlueProperties(opts, props, chosenS3); + appendGlueProperties(opts, IcebergGlueMetaStoreProperties.of(props), chosenS3); break; - case IcebergConnectorProperties.TYPE_JDBC: - appendJdbcProperties(opts, props); + case IcebergCatalogProperties.TYPE_JDBC: + appendJdbcProperties(opts, IcebergJdbcMetaStoreProperties.of(props)); appendS3FileIO(opts, props, chosenS3); // iceberg.jdbc.catalog_name is the positional catalog NAME (see resolveCatalogName); legacy // removes it from the options map before building. - opts.remove(IcebergConnectorProperties.JDBC_CATALOG_NAME); + opts.remove(JDBC_CATALOG_NAME); break; - case IcebergConnectorProperties.TYPE_HMS: + case IcebergCatalogProperties.TYPE_HMS: // No S3FileIO options: legacy iceberg HMS does not call toFileIOProperties; object-store access // rides the HiveConf (fs.s3a.* from storage), built by the connector via assembleHiveConf. break; - case IcebergConnectorProperties.TYPE_HADOOP: + case IcebergCatalogProperties.TYPE_HADOOP: appendS3FileIO(opts, props, chosenS3); break; default: @@ -379,8 +421,9 @@ public static Map buildCatalogProperties(Map pro * The control-plane {@code S3TablesClient} (region + credentials + endpoint + http) is built LIVE by the * connector ({@code IcebergConnector.buildS3TablesClient}); it is not part of this pure options map. */ - public static Map buildS3TablesCatalogProperties(Map props, + public static Map buildS3TablesCatalogProperties(IcebergCatalogProperties catalogProps, Optional chosenS3) { + Map props = catalogProps.getRaw(); Map opts = buildBaseCatalogProperties(props); if (chosenS3.isPresent()) { appendS3TablesFileIOProperties(opts, chosenS3.get(), props); @@ -399,12 +442,12 @@ public static Map buildS3TablesCatalogProperties(Map props, String flavor, String defaultName) { - if (IcebergConnectorProperties.TYPE_JDBC.equals(flavor)) { - String name = firstNonBlank(props, IcebergConnectorProperties.JDBC_CATALOG_NAME); + public static String resolveCatalogName(IcebergCatalogProperties catalogProps, String defaultName) { + if (IcebergCatalogProperties.TYPE_JDBC.equals(catalogProps.getFlavor())) { + String name = IcebergJdbcMetaStoreProperties.of(catalogProps.getRaw()).getJdbcCatalogName(); if (StringUtils.isBlank(name)) { throw new DorisConnectorException( - IcebergConnectorProperties.JDBC_CATALOG_NAME + " is required for an iceberg jdbc catalog"); + JDBC_CATALOG_NAME + " is required for an iceberg jdbc catalog"); } return name; } @@ -420,69 +463,60 @@ public static String resolveCatalogName(Map props, String flavor * ({@code prefix} / vended-credentials header / the two effectively-always timeouts), oauth2, and the glue * sigv4 signing block (with credentials sourced from the chosen S3 store for glue/s3tables, else from the * {@code iceberg.rest.*} aliases). PURE. + * + *

Every {@code iceberg.rest.*} value is read off the BOUND {@code rest} holder, which declares the alias + * set once. {@code props} is still needed for the credential-provider mode, whose alias set spans the + * s3/glue/rest namespaces and so belongs to no single flavor. */ - public static void appendRestProperties(Map opts, Map props, - Optional chosenS3) { + public static void appendRestProperties(Map opts, IcebergRestMetaStoreProperties rest, + Map props, Optional chosenS3) { // Core: uri is put UNCONDITIONALLY (legacy field default ""), alias priority iceberg.rest.uri > uri. - opts.put(CatalogProperties.URI, - firstNonBlankOrEmpty(props, IcebergConnectorProperties.REST_URI, IcebergConnectorProperties.URI)); + opts.put(CatalogProperties.URI, rest.getUri()); // Optional. - putIfNotBlank(opts, IcebergConnectorProperties.REST_PREFIX_KEY, - firstNonBlank(props, IcebergConnectorProperties.REST_PREFIX)); - String vendedEnabled = - firstNonBlankOrEmpty(props, IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED); - if (Boolean.parseBoolean(vendedEnabled)) { - opts.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_HEADER, - IcebergConnectorProperties.REST_VENDED_CREDENTIALS_VALUE); + putIfNotBlank(opts, REST_PREFIX_KEY, rest.getPrefix()); + if (rest.isVendedCredentialsEnabled()) { + opts.put(REST_VENDED_CREDENTIALS_HEADER, + REST_VENDED_CREDENTIALS_VALUE); } // Timeouts: legacy fields default non-blank, so they are effectively always emitted. - opts.put(IcebergConnectorProperties.REST_CONNECTION_TIMEOUT_MS_KEY, - firstNonBlankOr(props, IcebergConnectorProperties.DEFAULT_REST_CONNECTION_TIMEOUT_MS, - IcebergConnectorProperties.REST_CONNECTION_TIMEOUT_MS)); - opts.put(IcebergConnectorProperties.REST_SOCKET_TIMEOUT_MS_KEY, - firstNonBlankOr(props, IcebergConnectorProperties.DEFAULT_REST_SOCKET_TIMEOUT_MS, - IcebergConnectorProperties.REST_SOCKET_TIMEOUT_MS)); - appendRestOAuth2Properties(opts, props); - appendRestSigningProperties(opts, props, chosenS3); + opts.put(REST_CONNECTION_TIMEOUT_MS_KEY, rest.getConnectionTimeoutMs()); + opts.put(REST_SOCKET_TIMEOUT_MS_KEY, rest.getSocketTimeoutMs()); + appendRestOAuth2Properties(opts, rest); + appendRestSigningProperties(opts, rest, props, chosenS3); } - private static void appendRestOAuth2Properties(Map opts, Map props) { - String securityType = firstNonBlank(props, IcebergConnectorProperties.REST_SECURITY_TYPE); - if (!IcebergConnectorProperties.SECURITY_TYPE_OAUTH2.equalsIgnoreCase(securityType)) { + private static void appendRestOAuth2Properties(Map opts, IcebergRestMetaStoreProperties rest) { + if (!SECURITY_TYPE_OAUTH2.equalsIgnoreCase(rest.getSecurityType())) { return; } - String credential = firstNonBlank(props, IcebergConnectorProperties.REST_OAUTH2_CREDENTIAL); + String credential = rest.getOauth2Credential(); if (StringUtils.isNotBlank(credential)) { // Client Credentials Flow. opts.put(OAuth2Properties.CREDENTIAL, credential); - putIfNotBlank(opts, OAuth2Properties.OAUTH2_SERVER_URI, - firstNonBlank(props, IcebergConnectorProperties.REST_OAUTH2_SERVER_URI)); - putIfNotBlank(opts, OAuth2Properties.SCOPE, - firstNonBlank(props, IcebergConnectorProperties.REST_OAUTH2_SCOPE)); - opts.put(OAuth2Properties.TOKEN_REFRESH_ENABLED, - firstNonBlankOr(props, String.valueOf(OAuth2Properties.TOKEN_REFRESH_ENABLED_DEFAULT), - IcebergConnectorProperties.REST_OAUTH2_TOKEN_REFRESH_ENABLED)); + putIfNotBlank(opts, OAuth2Properties.OAUTH2_SERVER_URI, rest.getOauth2ServerUri()); + putIfNotBlank(opts, OAuth2Properties.SCOPE, rest.getOauth2Scope()); + // The holder cannot name the SDK's own default (it stays SDK-free), so it reports blank instead. + String tokenRefreshEnabled = rest.getOauth2TokenRefreshEnabled(); + opts.put(OAuth2Properties.TOKEN_REFRESH_ENABLED, StringUtils.isNotBlank(tokenRefreshEnabled) + ? tokenRefreshEnabled : String.valueOf(OAuth2Properties.TOKEN_REFRESH_ENABLED_DEFAULT)); } else { // Pre-configured Token Flow (validation guarantees a token here when credential is absent). - opts.put(OAuth2Properties.TOKEN, - firstNonBlankOrEmpty(props, IcebergConnectorProperties.REST_OAUTH2_TOKEN)); + opts.put(OAuth2Properties.TOKEN, nullToEmpty(rest.getOauth2Token())); } } - private static void appendRestSigningProperties(Map opts, Map props, - Optional chosenS3) { - String signingName = firstNonBlank(props, IcebergConnectorProperties.REST_SIGNING_NAME); + private static void appendRestSigningProperties(Map opts, IcebergRestMetaStoreProperties rest, + Map props, Optional chosenS3) { + String signingName = rest.getSigningName(); if (StringUtils.isBlank(signingName)) { return; } // signing-name is case-sensitive; do not lower-case it. - opts.put(IcebergConnectorProperties.REST_SIGNING_NAME_KEY, signingName); - opts.put(IcebergConnectorProperties.REST_SIGV4_ENABLED_KEY, - firstNonBlankOrEmpty(props, IcebergConnectorProperties.REST_SIGV4_ENABLED)); - opts.put(IcebergConnectorProperties.REST_SIGNING_REGION_KEY, - firstNonBlankOrEmpty(props, IcebergConnectorProperties.REST_SIGNING_REGION)); - if (IcebergConnectorProperties.SIGNING_NAME_GLUE.equals(signingName) - || IcebergConnectorProperties.SIGNING_NAME_S3TABLES.equals(signingName)) { + opts.put(REST_SIGNING_NAME_KEY, signingName); + opts.put(REST_SIGV4_ENABLED_KEY, rest.getSigV4Enabled()); + opts.put(REST_SIGNING_REGION_KEY, rest.getSigningRegion()); + if (SIGNING_NAME_GLUE.equals(signingName) + || SIGNING_NAME_S3TABLES.equals(signingName)) { // glue/s3tables: credentials come from the chosen S3 store, switching on its credential type // (legacy getCredentialType precedence: EXPLICIT before ASSUME_ROLE before PROVIDER_CHAIN). if (chosenS3.isPresent()) { @@ -501,15 +535,13 @@ private static void appendRestSigningProperties(Map opts, Map opts, String * aliases (which fe-filesystem does not read into the S3 store) the s3.* FileIO creds may be absent — a * UT-invisible edge handled at the P6.6 docker gate (the glue-client creds below still come from glue.*). */ - public static void appendGlueProperties(Map opts, Map props, + public static void appendGlueProperties(Map opts, IcebergGlueMetaStoreProperties glue, Optional chosenS3) { chosenS3.ifPresent(s3 -> { opts.put(S3FileIOProperties.ACCESS_KEY_ID, nullToEmpty(s3.getAccessKey())); @@ -548,39 +580,37 @@ public static void appendGlueProperties(Map opts, Map props, String glueEndpoint) { - String region = firstNonBlank(props, IcebergConnectorProperties.GLUE_REGION); + private static String resolveGlueRegion(String region, String glueEndpoint) { if (StringUtils.isNotBlank(region)) { return region; } @@ -590,7 +620,7 @@ private static String resolveGlueRegion(Map props, String glueEn return matcher.group(1); } } - return IcebergConnectorProperties.GLUE_DEFAULT_REGION; + return GLUE_DEFAULT_REGION; } // --------------------------------------------------------------------- @@ -605,18 +635,13 @@ private static String resolveGlueRegion(Map props, String glueEn * not repeated here. The {@code catalog_name} positional removal + driver registration are handled by the * connector. PURE. */ - public static void appendJdbcProperties(Map opts, Map props) { - opts.put(CatalogProperties.URI, firstNonBlankOrEmpty(props, IcebergConnectorProperties.JDBC_URI)); - putIfNotBlank(opts, IcebergConnectorProperties.JDBC_USER_KEY, - firstNonBlank(props, IcebergConnectorProperties.JDBC_USER)); - putIfNotBlank(opts, IcebergConnectorProperties.JDBC_PASSWORD_KEY, - firstNonBlank(props, IcebergConnectorProperties.JDBC_PASSWORD)); - putIfNotBlank(opts, IcebergConnectorProperties.JDBC_INIT_CATALOG_TABLES_KEY, - firstNonBlank(props, IcebergConnectorProperties.JDBC_INIT_CATALOG_TABLES)); - putIfNotBlank(opts, IcebergConnectorProperties.JDBC_SCHEMA_VERSION_KEY, - firstNonBlank(props, IcebergConnectorProperties.JDBC_SCHEMA_VERSION)); - putIfNotBlank(opts, IcebergConnectorProperties.JDBC_STRICT_MODE_KEY, - firstNonBlank(props, IcebergConnectorProperties.JDBC_STRICT_MODE)); + public static void appendJdbcProperties(Map opts, IcebergJdbcMetaStoreProperties jdbc) { + opts.put(CatalogProperties.URI, jdbc.getUri()); + putIfNotBlank(opts, JDBC_USER_KEY, jdbc.getUser()); + putIfNotBlank(opts, JDBC_PASSWORD_KEY, jdbc.getPassword()); + putIfNotBlank(opts, JDBC_INIT_CATALOG_TABLES_KEY, jdbc.getInitCatalogTables()); + putIfNotBlank(opts, JDBC_SCHEMA_VERSION_KEY, jdbc.getSchemaVersion()); + putIfNotBlank(opts, JDBC_STRICT_MODE_KEY, jdbc.getStrictMode()); } // --------------------------------------------------------------------- @@ -707,8 +732,12 @@ static void addConfResources(HiveConf hiveConf, String confResources) { // Pure helpers // --------------------------------------------------------------------- - /** Returns the first non-blank value among the given keys (alias priority), or {@code null} if none set. */ - public static String firstNonBlank(Map props, String... keys) { + /** + * Returns the first non-blank value among the given keys (alias priority), or {@code null} if none set. + * The metastore keys are read off their bound holders instead; the sole remaining caller is the S3 region + * scan, whose alias set belongs to the storage keys rather than to any one metastore flavor. + */ + private static String firstNonBlank(Map props, String... keys) { for (String key : keys) { String value = props.get(key); if (StringUtils.isNotBlank(value)) { @@ -718,16 +747,6 @@ public static String firstNonBlank(Map props, String... keys) { return null; } - private static String firstNonBlankOrEmpty(Map props, String... keys) { - String value = firstNonBlank(props, keys); - return value == null ? "" : value; - } - - private static String firstNonBlankOr(Map props, String defaultValue, String... keys) { - String value = firstNonBlank(props, keys); - return value == null ? defaultValue : value; - } - private static String nullToEmpty(String value) { return value == null ? "" : value; } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java new file mode 100644 index 00000000000000..8cf4db88386e8e --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metastore.spi.MetaStoreProviders; +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * Everything a user writes in {@code CREATE CATALOG} for an iceberg catalog that is not specific to one + * metastore backend (the "A class" of the connector property convention; deployment-level settings live + * in {@link IcebergConf}). The per-flavor keys — uri, credentials, warehouse — belong to the bound + * {@code *MetaStoreProperties} instead, which is why only four keys are declared here. + * + *

The two entry points are not interchangeable. {@link #of(Map)} runs at CREATE, at ALTER + * validation, AND on every connector build — including the lazy rebuild after an FE restart, which + * happens at least twice for an interactively created catalog. So it binds and derives, and it does not + * enforce anything the connector can run without: a rule added after a catalog was created must not stop + * that catalog from coming back. {@link #checkCreateTimeOnlyRules()} carries the rules that only ever ran + * against a statement, and only the provider calls it. + * + *

The flavor is bound as a plain String, not an enum. Backends are discovered through + * {@code ServiceLoader}, so the set is open by design and a central enum would have to be edited every + * time one is added. + */ +public final class IcebergCatalogProperties { + + /** Iceberg catalog backend type: rest, hms, glue, jdbc, hadoop, s3tables. */ + public static final String ICEBERG_CATALOG_TYPE = "iceberg.catalog.type"; + + // ---- Flavor literals (the accepted iceberg.catalog.type values) ---- + public static final String TYPE_REST = "rest"; + public static final String TYPE_HMS = "hms"; + public static final String TYPE_GLUE = "glue"; + public static final String TYPE_JDBC = "jdbc"; + public static final String TYPE_HADOOP = "hadoop"; + public static final String TYPE_S3_TABLES = "s3tables"; + + /** + * Whether to map iceberg BINARY/FIXED to Doris VARBINARY instead of STRING. + * + *

Canonical (dotted) CREATE-CATALOG key, mirroring fe-core + * {@code CatalogProperty.ENABLE_MAPPING_VARBINARY}. The connector receives the raw catalog property + * map, which only ever carries this dotted key, so the read MUST use the dotted spelling — an + * underscore variant is never present and would silently read false. + */ + public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; + + /** Whether to map iceberg TIMESTAMPTZ to Doris TIMESTAMPTZ. Dotted key, same rationale as above. */ + public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; + + /** + * The catalog level of a 3-level REST namespace ({@code ..

}), mirroring legacy + * {@code IcebergExternalCatalog.EXTERNAL_CATALOG_NAME}: when present it is appended to every namespace + * and roots database listing. Despite reading like a REST-only knob it is honored for EVERY flavor + * ({@code IcebergCatalogOps} applies it without a flavor gate), which is why it lives here rather than + * on the REST backend properties. + */ + public static final String EXTERNAL_CATALOG_NAME = "external_catalog.name"; + + @ConnectorProperty(names = {ICEBERG_CATALOG_TYPE}, required = false, + description = "The metastore backend: rest, hms, glue, jdbc, hadoop or s3tables.") + private String catalogType = ""; + + @ConnectorProperty(names = {ENABLE_MAPPING_VARBINARY}, required = false, + description = "Map iceberg BINARY/FIXED to Doris VARBINARY instead of STRING.") + private boolean enableMappingVarbinary; + + @ConnectorProperty(names = {ENABLE_MAPPING_TIMESTAMP_TZ}, required = false, + description = "Map iceberg TIMESTAMPTZ to Doris TIMESTAMPTZ.") + private boolean enableMappingTimestampTz; + + // Null, NOT "": the consumers ask for an Optional, and a blank default would turn every catalog that + // never set the key into one carrying an empty namespace level. + @ConnectorProperty(names = {EXTERNAL_CATALOG_NAME}, required = false, + description = "The catalog level of a 3-level REST namespace.") + private String externalCatalogName; + + private final Map raw; + private String flavor; + + private IcebergCatalogProperties(Map properties) { + this.raw = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + /** + * Binds and derives. Pure, idempotent, and free of I/O — it runs on every connector build. + * + *

Unknown keys are never rejected: the same map carries engine keys ({@code type}, + * {@code meta.cache.*}, ...) and storage keys ({@code s3.*}, ...), and ALTER CATALOG can only + * overwrite a key, never remove one, so a rejected unknown key could not be repaired. + */ + public static IcebergCatalogProperties of(Map properties) { + IcebergCatalogProperties p = new IcebergCatalogProperties(properties); + ConnectorPropertiesUtils.bindConnectorProperties(p, properties); + // Null rather than a default flavor: "the catalog names no backend" is a distinct state that the + // build and the CREATE-time dispatch both report on, and inventing a default here would silently + // route such a catalog somewhere. + p.flavor = p.catalogType.isEmpty() ? null : p.catalogType.toLowerCase(Locale.ROOT); + return p; + } + + /** + * The checks that only ever ran against a CREATE/ALTER CATALOG statement, kept out of {@link #of(Map)} + * so they cannot break the rebuild of a catalog that predates them. + * + *

Order is load-bearing and matches the provider method this replaces: the meta-cache values are + * rejected first (restoring the legacy {@code IcebergExternalCatalog.checkProperties} fail-fast that + * the SPI cutover dropped, so a bad {@code meta.cache.iceberg.*} is rejected rather than silently + * coerced to a cache-disabling default), then the backend's own rules. + * + * @return this, so the provider can call it in one expression + */ + public IcebergCatalogProperties checkCreateTimeOnlyRules() { + checkMetaCacheProperties(raw); + // Selects the backend by flavor and enforces its fail-fast rules -- REST (security/creds enums, + // OAuth2, signing, AK/SK), Glue (AK/SK-together, endpoint https, at-least-one-credential), JDBC + // (uri/catalog_name/warehouse), the shared HMS connection checks; hadoop/s3tables are no-op (their + // storage is validated upstream at fe-filesystem bind). A blank or unknown flavor makes bindForType + // throw (no provider supports it). Storage plays no part in validation, so an empty storage map is + // passed, and the bound instance is deliberately not kept -- it was built without storage and must + // not be mistaken for the one the HMS catalog build needs. + MetaStoreProviders.bindForType(flavor, raw, Collections.emptyMap()).validate(); + return this; + } + + /** + * Byte-for-byte parity with the legacy {@code IcebergExternalCatalog.checkProperties}: table/manifest + * {@code enable} must be boolean, {@code ttl-second} must be a long ≥ -1 (the "no expiration" + * sentinel), {@code capacity} must be a long ≥ 0. Absent keys are skipped. + */ + private static void checkMetaCacheProperties(Map properties) { + CacheSpec.checkBooleanProperty(properties.get(IcebergConnector.TABLE_CACHE_ENABLE), + IcebergConnector.TABLE_CACHE_ENABLE); + CacheSpec.checkLongProperty(properties.get(IcebergConnector.TABLE_CACHE_TTL_SECOND), + -1L, IcebergConnector.TABLE_CACHE_TTL_SECOND); + CacheSpec.checkLongProperty(properties.get(IcebergConnector.TABLE_CACHE_CAPACITY), + 0L, IcebergConnector.TABLE_CACHE_CAPACITY); + + CacheSpec.checkBooleanProperty(properties.get(IcebergConnector.MANIFEST_CACHE_ENABLE), + IcebergConnector.MANIFEST_CACHE_ENABLE); + CacheSpec.checkLongProperty(properties.get(IcebergConnector.MANIFEST_CACHE_TTL_SECOND), + -1L, IcebergConnector.MANIFEST_CACHE_TTL_SECOND); + CacheSpec.checkLongProperty(properties.get(IcebergConnector.MANIFEST_CACHE_CAPACITY), + 0L, IcebergConnector.MANIFEST_CACHE_CAPACITY); + } + + /** The metastore backend, lower-cased; {@code null} when the catalog does not name one. */ + public String getFlavor() { + return flavor; + } + + public boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + + /** Empty when the catalog names no extra namespace level. */ + public Optional getExternalCatalogName() { + return Optional.ofNullable(externalCatalogName); + } + + /** + * The raw catalog properties, unmodifiable. Kept because several consumers forward whole key + * NAMESPACES the connector does not model — the copy-all seed of the catalog options, + * {@code fs./dfs./hadoop.} into the Hadoop config, the S3 region aliases. Those are passthroughs, not + * properties this class owns. + */ + public Map getRaw() { + return raw; + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConf.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConf.java new file mode 100644 index 00000000000000..f8d1f377d56384 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConf.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.spi.ConnectorConf; +import org.apache.doris.connector.spi.ConnectorContext; + +/** + * This connector's deployment-level settings — the "B class" of the connector property convention, as + * opposed to the per-catalog properties in {@link IcebergCatalogProperties}. These are read from the + * plugin's own {@code iceberg.conf} (named after {@code ConnectorProvider.name()}), each falling back to + * the {@code ENV_} name below it, which is the fe.conf key it used to live under and still works. + * + *

Both conf keys are shared with other connectors at the fe.conf end — one {@code jdbc_drivers_dir} + * and one {@code hive_metastore_client_timeout_second} serve jdbc, iceberg and paimon. A plugin conf + * cannot express that, so a deployment that moves to these files sets the value in each plugin's own + * conf. That is the accepted cost of a per-plugin file; the fe.conf keys stay as the shared fallback. + */ +public final class IcebergConf { + + public static final String CONF_DRIVERS_DIR = "drivers_dir"; + public static final String CONF_METASTORE_CLIENT_TIMEOUT_SECOND = "metastore_client_timeout_second"; + + /** The fe.conf name of {@link #CONF_DRIVERS_DIR}, forwarded through the engine environment. */ + public static final String ENV_JDBC_DRIVERS_DIR = "jdbc_drivers_dir"; + /** The fe.conf name of {@link #CONF_METASTORE_CLIENT_TIMEOUT_SECOND}. */ + public static final String ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND = + "hive_metastore_client_timeout_second"; + /** Engine-wide, not this connector's: the FE install root. Stays in the engine environment. */ + public static final String ENV_DORIS_HOME = "doris_home"; + /** Legacy default when neither channel names a metastore client timeout. */ + public static final String DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND = "10"; + + private IcebergConf() { + } + + /** + * The directory a bare driver jar name resolves under, from this plugin's own {@code iceberg.conf} + * or fe.conf's {@code jdbc_drivers_dir}. Null when neither names one — {@code JdbcDriverSupport} + * then falls back to {@code /plugins/jdbc_drivers}, as before. + * + *

A null context is a direct-construction unit test, which has neither channel. + */ + public static String driversDir(ConnectorContext context) { + return context == null ? null + : ConnectorConf.get(context, CONF_DRIVERS_DIR, ENV_JDBC_DRIVERS_DIR, null); + } + + /** The FE install root. Engine-wide rather than this connector's, so it stays in the environment. */ + public static String dorisHome(ConnectorContext context) { + return context == null ? null : context.getEnvironment().get(ENV_DORIS_HOME); + } + + /** + * The hive metastore client socket timeout, in seconds, handed to the shared HMS parser so it can + * apply it when the catalog does not set {@code hive.metastore.client.socket.timeout} itself. Falls + * back to the legacy literal so a deployment with neither file behaves as it did. + */ + public static String metastoreClientTimeoutSecond(ConnectorContext context) { + return ConnectorConf.get(context, CONF_METASTORE_CLIENT_TIMEOUT_SECOND, + ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 260f33c7c98856..47df2d3f9c6f06 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -19,11 +19,13 @@ import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; +import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; +import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.AbstractHmsMetaStoreProperties; import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorCapability; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorPartitionInfo; @@ -132,7 +134,16 @@ public class IcebergConnector implements Connector { // T08 latest-snapshot cache knobs (mirror PaimonConnector). The TTL pins a STABLE snapshot across queries; // <= 0 means "no-cache catalog" (always live). Defaults mirror the legacy iceberg table cache // (Config.external_cache_expire_time_seconds_after_access = 24h, Config.max_external_table_cache_num). + // The six meta-cache key names live here, next to the caches they configure: this class builds every + // one of them, and the other readers (the CREATE-time validation on IcebergCatalogProperties, the + // manifest-cache derivation in IcebergCatalogFactory, the manifest gate in IcebergScanPlanProvider) + // are all in this package. They used to be spelled out in two places each. + static final String TABLE_CACHE_ENABLE = "meta.cache.iceberg.table.enable"; static final String TABLE_CACHE_TTL_SECOND = "meta.cache.iceberg.table.ttl-second"; + static final String TABLE_CACHE_CAPACITY = "meta.cache.iceberg.table.capacity"; + static final String MANIFEST_CACHE_ENABLE = "meta.cache.iceberg.manifest.enable"; + static final String MANIFEST_CACHE_TTL_SECOND = "meta.cache.iceberg.manifest.ttl-second"; + static final String MANIFEST_CACHE_CAPACITY = "meta.cache.iceberg.manifest.capacity"; static final long DEFAULT_TABLE_CACHE_TTL_SECOND = 86400L; static final int DEFAULT_TABLE_CACHE_CAPACITY = 1000; @@ -151,6 +162,14 @@ public class IcebergConnector implements Connector { private static final String BE_TEST_LOCATION = "test_location"; private final Map properties; + // The bound connector-level facts (flavor, type-mapping switches, namespace level), the single + // reader of those keys. Also handed to the metadata / scan / write providers, so all four agree. + private final IcebergCatalogProperties catalogProps; + // The bound iceberg.rest.* facts, the single reader of those keys on this side. Bound for EVERY flavor: + // of() only binds (it never validates, that is the provider's CREATE-time job), and the two listing flags + // it carries are read unconditionally below -- IcebergCatalogOps gates them on restFlavor instead, so a + // non-REST catalog reads the same defaults it read from the raw map before. + private final IcebergRestMetaStoreProperties restProperties; private final ConnectorContext context; private volatile Catalog icebergCatalog; // Session-aware REST catalog, built for every REST catalog (plain or iceberg.rest.session=user). A SINGLE @@ -215,6 +234,9 @@ public class IcebergConnector implements Connector { public IcebergConnector(Map properties, ConnectorContext context) { this.properties = Collections.unmodifiableMap(properties); + // Both before isUserSessionEnabled(), which the cache fields below gate on. + this.catalogProps = IcebergCatalogProperties.of(this.properties); + this.restProperties = IcebergRestMetaStoreProperties.of(this.properties); // Pin the thread-context classloader to the plugin loader for the duration of every // executeAuthenticated call (see TcclPinningConnectorContext). The injected context is fanned out to // the metadata / transaction / procedure ops below; wrapping it once here is what extends the @@ -310,7 +332,7 @@ static long resolveTableCacheTtlSecond(Map properties) { @Override public ConnectorMetadata getMetadata(ConnectorSession session) { - return new IcebergConnectorMetadata(newCatalogBackedOps(session), properties, context, + return new IcebergConnectorMetadata(newCatalogBackedOps(session), catalogProps, context, latestSnapshotCache, tableCache, partitionCache, commentCache, mvccPartitionViewCache, listPartitionsViewCache); } @@ -339,7 +361,7 @@ public boolean ownsHandle(ConnectorTableHandle handle) { @Override public ConnectorTestResult testConnection(ConnectorSession session) { String catalogType = properties.getOrDefault( - IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, ""); + IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, ""); // -- Metastore probe (REST, HMS, Glue, S3Tables) -- // Listing databases forces a real round-trip that validates the URI, auth and warehouse config. @@ -464,11 +486,11 @@ ConnectorTestResult probeStorageFromBackend(String location) { * (Iceberg {@code warehouse} or Polaris {@code default-base-location}). */ private String resolveS3TestLocation(String catalogType) { - String location = toS3Location(properties.get(IcebergConnectorProperties.WAREHOUSE)); + String location = toS3Location(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); if (location != null) { return location; } - if (IcebergConnectorProperties.TYPE_REST.equalsIgnoreCase(catalogType)) { + if (IcebergCatalogProperties.TYPE_REST.equalsIgnoreCase(catalogType)) { Catalog catalog = getOrCreateCatalog(); // The server-merged config (Iceberg warehouse / Polaris default-base-location) lives on the REST // session catalog we now hold (the wrapped RESTSessionCatalog; its properties() delegates to the raw @@ -502,10 +524,10 @@ private String resolveS3TestLocation(String catalogType) { * their "metastore" is the warehouse itself, which the storage probe below covers. */ static boolean probesMetastore(String catalogType) { - return IcebergConnectorProperties.TYPE_REST.equalsIgnoreCase(catalogType) - || IcebergConnectorProperties.TYPE_HMS.equalsIgnoreCase(catalogType) - || IcebergConnectorProperties.TYPE_GLUE.equalsIgnoreCase(catalogType) - || IcebergConnectorProperties.TYPE_S3_TABLES.equalsIgnoreCase(catalogType); + return IcebergCatalogProperties.TYPE_REST.equalsIgnoreCase(catalogType) + || IcebergCatalogProperties.TYPE_HMS.equalsIgnoreCase(catalogType) + || IcebergCatalogProperties.TYPE_GLUE.equalsIgnoreCase(catalogType) + || IcebergCatalogProperties.TYPE_S3_TABLES.equalsIgnoreCase(catalogType); } static String metaFailureMessage(String catalogType, Throwable cause) { @@ -563,14 +585,12 @@ private static boolean isBlank(String s) { * catalogs resolve to the wrong namespace. */ private IcebergCatalogOps newCatalogBackedOps() { - String flavor = IcebergCatalogFactory.resolveFlavor(properties); - boolean restFlavor = IcebergConnectorProperties.TYPE_REST.equals(flavor); - boolean nestedNamespaceEnabled = Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.REST_NESTED_NAMESPACE_ENABLED, "false")); - boolean viewEnabled = Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.REST_VIEW_ENABLED, "true")); + String flavor = catalogProps.getFlavor(); + boolean restFlavor = IcebergCatalogProperties.TYPE_REST.equals(flavor); + boolean nestedNamespaceEnabled = restProperties.isNestedNamespaceEnabled(); + boolean viewEnabled = restProperties.isViewEnabled(); Optional externalCatalogName = - Optional.ofNullable(properties.get(IcebergConnectorProperties.EXTERNAL_CATALOG_NAME)); + catalogProps.getExternalCatalogName(); Catalog sharedCatalog = getOrCreateCatalog(); // Plain REST now holds the bare (wrapped) RESTSessionCatalog: its asCatalog(empty) is a Catalog + // SupportsNamespaces but NOT a ViewCatalog, so inject the view facet asViewCatalog(empty) explicitly (what @@ -602,12 +622,10 @@ private IcebergCatalogOps newCatalogBackedOps(ConnectorSession session) { IcebergSessionCatalogAdapter adapter = sessionCatalogAdapter; Catalog perUserCatalog = adapter.delegatedCatalog(session); ViewCatalog perUserViewCatalog = adapter.delegatedViewCatalog(session); - boolean nestedNamespaceEnabled = Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.REST_NESTED_NAMESPACE_ENABLED, "false")); - boolean viewEnabled = Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.REST_VIEW_ENABLED, "true")); + boolean nestedNamespaceEnabled = restProperties.isNestedNamespaceEnabled(); + boolean viewEnabled = restProperties.isViewEnabled(); Optional externalCatalogName = - Optional.ofNullable(properties.get(IcebergConnectorProperties.EXTERNAL_CATALOG_NAME)); + catalogProps.getExternalCatalogName(); // restFlavor is unconditionally true here (isUserSessionEnabled() ⇒ a REST catalog). return new IcebergCatalogOps.CatalogBackedIcebergCatalogOps(perUserCatalog, perUserViewCatalog, true, nestedNamespaceEnabled, viewEnabled, externalCatalogName); @@ -782,7 +800,7 @@ public ConnectorScanPlanProvider getScanPlanProvider() { // external_catalog.name (REST 3-level catalogs), so it must share getMetadata's fully-threaded ops // (newCatalogBackedOps) — the listing-only flags (nested-namespace / view) are inert on this path but // threaded for parity with the legacy single per-catalog IcebergMetadataOps. - return new IcebergScanPlanProvider(properties, + return new IcebergScanPlanProvider(catalogProps, this::newCatalogBackedOps, context, manifestCache, tableCache, formatCache); } @@ -793,7 +811,7 @@ public ConnectorWritePlanProvider getWritePlanProvider() { // provider builds the TIcebergTableSink and binds the write to the executor-opened // IcebergConnectorTransaction. It resolves the target via catalogOps.loadTable, so it shares the // fully-threaded ops (newCatalogBackedOps) — external_catalog.name must apply to INSERT/DELETE/MERGE. - return new IcebergWritePlanProvider(properties, + return new IcebergWritePlanProvider(catalogProps, this::newCatalogBackedOps, context); } @@ -874,9 +892,8 @@ public Set getCapabilities() { * that {@code session=user} implies {@code security.type=oauth2}. */ boolean isUserSessionEnabled() { - return IcebergConnectorProperties.SESSION_USER.equalsIgnoreCase( - properties.get(IcebergConnectorProperties.REST_SESSION)) - && IcebergConnectorProperties.TYPE_REST.equals(IcebergCatalogFactory.resolveFlavor(properties)); + return restProperties.isUserSession() + && IcebergCatalogProperties.TYPE_REST.equals(catalogProps.getFlavor()); } private Catalog getOrCreateCatalog() { @@ -891,49 +908,46 @@ private Catalog getOrCreateCatalog() { } private Catalog createCatalog() { - String flavor = IcebergCatalogFactory.resolveFlavor(properties); + String flavor = catalogProps.getFlavor(); if (flavor == null) { throw new DorisConnectorException( - "Missing '" + IcebergConnectorProperties.ICEBERG_CATALOG_TYPE + "' property"); + "Missing '" + IcebergCatalogProperties.ICEBERG_CATALOG_TYPE + "' property"); } Optional chosenS3 = IcebergCatalogFactory.chooseS3Compatible(storage().getStorageProperties()); - String catalogName = IcebergCatalogFactory.resolveCatalogName(properties, flavor, context.getCatalogName()); + String catalogName = IcebergCatalogFactory.resolveCatalogName(catalogProps, context.getCatalogName()); // s3tables is bespoke: it is NOT built via CatalogUtil.buildIcebergCatalog. Legacy // IcebergS3TablesMetaStoreProperties hand-builds an S3TablesClient and calls the 3-arg // S3TablesCatalog.initialize(name, opts, client). Routed before the CatalogUtil flavor switch. - if (IcebergConnectorProperties.TYPE_S3_TABLES.equals(flavor)) { + if (IcebergCatalogProperties.TYPE_S3_TABLES.equals(flavor)) { return createS3TablesCatalog(catalogName, chosenS3); } - Map catalogProps = - IcebergCatalogFactory.buildCatalogProperties(properties, flavor, chosenS3); + Map catalogOptions = + IcebergCatalogFactory.buildCatalogProperties(catalogProps, chosenS3); Map storageHadoopConfig = buildStorageHadoopConfig(); Configuration conf; switch (flavor) { - case IcebergConnectorProperties.TYPE_HMS: { + case IcebergCatalogProperties.TYPE_HMS: { // Reuse the shared metastore-spi parser (Q2=B bindForType): iceberg passes its own flavor token // so the metastore-spi never learns iceberg.catalog.type. Only toHiveConfOverrides is used // (iceberg HMS does NOT call paimon's validate(); it does not require a warehouse). The external // hive.conf.resources hive-site.xml is resolved by the connector itself (addConfResources). - HmsMetaStoreProperties hms = (HmsMetaStoreProperties) MetaStoreProviders.bindForType( - IcebergConnectorProperties.TYPE_HMS, properties, storageHadoopConfig); + AbstractHmsMetaStoreProperties hms = (AbstractHmsMetaStoreProperties) MetaStoreProviders + .bindForType(IcebergCatalogProperties.TYPE_HMS, properties, storageHadoopConfig); conf = IcebergCatalogFactory.assembleHiveConf( - IcebergCatalogFactory.firstNonBlank(properties, "hive.conf.resources"), - hms.toHiveConfOverrides(ConnectorConf.get(context, - IcebergConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND))); + hms.getConfResources(), + hms.toHiveConfOverrides(IcebergConf.metastoreClientTimeoutSecond(context))); break; } - case IcebergConnectorProperties.TYPE_GLUE: + case IcebergCatalogProperties.TYPE_GLUE: // Legacy IcebergGlueMetaStoreProperties builds the catalog with conf=null. conf = null; break; - case IcebergConnectorProperties.TYPE_JDBC: + case IcebergCatalogProperties.TYPE_JDBC: maybeRegisterJdbcDriver(); conf = IcebergCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig); break; @@ -949,14 +963,14 @@ private Catalog createCatalog() { // is asCatalog(empty), identical to what a RESTCatalog exposes; wrapping it recovers a catalog-identity // token expiry (#64966), and for session=user the shared session catalog + adapter also drive per-request // asCatalog(ctx)/asViewCatalog(ctx) delegated-credential routing (#63068). - if (IcebergConnectorProperties.TYPE_REST.equals(flavor)) { - return buildRestSessionCatalogDefault(catalogName, catalogProps, conf); + if (IcebergCatalogProperties.TYPE_REST.equals(flavor)) { + return buildRestSessionCatalogDefault(catalogName, catalogOptions, conf); } LOG.info("Creating Iceberg catalog '{}' flavor='{}' impl='{}'", - catalogName, flavor, catalogProps.get(CatalogProperties.CATALOG_IMPL)); + catalogName, flavor, catalogOptions.get(CatalogProperties.CATALOG_IMPL)); return buildCatalogAuthenticated(flavor, - () -> CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf)); + () -> CatalogUtil.buildIcebergCatalog(catalogName, catalogOptions, conf)); } /** @@ -970,27 +984,26 @@ private Catalog createCatalog() { * Memoizes {@link #restSessionCatalog} (always) and {@link #sessionCatalogAdapter} (session=user only) as a * side effect. The optional {@code iceberg.rest.session-timeout} maps to the iceberg AuthSession timeout. */ - private Catalog buildRestSessionCatalogDefault(String catalogName, Map catalogProps, + private Catalog buildRestSessionCatalogDefault(String catalogName, Map catalogOptions, Configuration conf) { - Map sessionProps = new HashMap<>(catalogProps); + Map sessionProps = new HashMap<>(catalogOptions); // Built directly via new RESTSessionCatalog(), so the CatalogUtil catalog-impl key is neither needed nor // valid here; drop it. sessionProps.remove(CatalogProperties.CATALOG_IMPL); - String sessionTimeout = properties.get(IcebergConnectorProperties.REST_SESSION_TIMEOUT); + String sessionTimeout = restProperties.getSessionTimeout(); if (StringUtils.isNotBlank(sessionTimeout)) { sessionProps.put(CatalogProperties.AUTH_SESSION_TIMEOUT_MS, sessionTimeout); } boolean userSession = isUserSessionEnabled(); IcebergSessionCatalogAdapter.DelegatedTokenMode tokenMode = - IcebergSessionCatalogAdapter.DelegatedTokenMode.fromString(properties.getOrDefault( - IcebergConnectorProperties.REST_DELEGATED_TOKEN_MODE, - IcebergConnectorProperties.DELEGATED_TOKEN_MODE_ACCESS_TOKEN)); + IcebergSessionCatalogAdapter.DelegatedTokenMode.fromString( + restProperties.getDelegatedTokenMode()); // Frozen catalog-identity properties for the 401-recovery rebuild: an unmodifiable copy so the rebuild // always re-resolves from the catalog's OWN credential and can never capture a per-user delegated token. Map frozenProps = Collections.unmodifiableMap(new HashMap<>(sessionProps)); LOG.info("Creating Iceberg REST catalog '{}' (userSession={}, delegated-token-mode={})", catalogName, userSession, tokenMode); - return buildCatalogAuthenticated(IcebergConnectorProperties.TYPE_REST, () -> { + return buildCatalogAuthenticated(IcebergCatalogProperties.TYPE_REST, () -> { RESTSessionCatalog rawSessionCatalog = newRestSessionCatalog(catalogName, sessionProps, conf); // Wrap so a 401 on the catalog's own identity rebuilds the client and retries once. The wrapper is a // thin BaseViewSessionCatalog: asCatalog(empty)/asViewCatalog(empty) and the per-user asCatalog(ctx) @@ -1049,14 +1062,14 @@ private RESTSessionCatalog rebuildRestSessionCatalog(String catalogName, Map chosenS3) { String region = resolveS3TablesRegion(chosenS3, properties); - Map catalogProps = - IcebergCatalogFactory.buildS3TablesCatalogProperties(properties, chosenS3); + Map catalogOptions = + IcebergCatalogFactory.buildS3TablesCatalogProperties(catalogProps, chosenS3); LOG.info("Creating Iceberg s3tables catalog '{}' region='{}' boundStorage={}", catalogName, region, chosenS3.isPresent()); - return buildCatalogAuthenticated(IcebergConnectorProperties.TYPE_S3_TABLES, () -> { + return buildCatalogAuthenticated(IcebergCatalogProperties.TYPE_S3_TABLES, () -> { S3TablesClient client = buildS3TablesClient(chosenS3, region); S3TablesCatalog catalog = new S3TablesCatalog(); - catalog.initialize(catalogName, catalogProps, client); + catalog.initialize(catalogName, catalogOptions, client); return catalog; }); } @@ -1172,11 +1185,11 @@ public Map deriveStorageProperties(Map rawCatalo * (rest/hms/glue/jdbc/s3tables) contributes nothing. */ static Map deriveStorageDefaults(Map rawCatalogProps) { - if (!IcebergConnectorProperties.TYPE_HADOOP.equalsIgnoreCase( - rawCatalogProps.get(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE))) { + if (!IcebergCatalogProperties.TYPE_HADOOP.equalsIgnoreCase( + rawCatalogProps.get(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE))) { return Collections.emptyMap(); } - return deriveHdfsDefaultFsFromWarehouse(rawCatalogProps.get(IcebergConnectorProperties.WAREHOUSE)); + return deriveHdfsDefaultFsFromWarehouse(rawCatalogProps.get(CatalogProperties.WAREHOUSE_LOCATION)); } /** @@ -1257,9 +1270,9 @@ static HadoopAuthenticator buildPluginAuthenticator(Map properti return HadoopAuthenticator.getHadoopAuthenticator( IcebergCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig)); } - if (IcebergConnectorProperties.TYPE_HMS.equals(IcebergCatalogFactory.resolveFlavor(properties))) { + if (IcebergCatalogProperties.TYPE_HMS.equals(IcebergCatalogProperties.of(properties).getFlavor())) { HmsMetaStoreProperties hms = (HmsMetaStoreProperties) MetaStoreProviders.bindForType( - IcebergConnectorProperties.TYPE_HMS, properties, storageHadoopConfig); + IcebergCatalogProperties.TYPE_HMS, properties, storageHadoopConfig); Optional spec = hms.kerberos(); if (spec.isPresent() && spec.get().hasCredentials()) { Configuration conf = @@ -1382,10 +1395,10 @@ static boolean pinPoolThreadsToClassLoader(ExecutorService pool, int poolSize, C */ @Override public void preCreateValidation(ConnectorValidationContext validationContext) throws Exception { - if (!IcebergConnectorProperties.TYPE_JDBC.equals(IcebergCatalogFactory.resolveFlavor(properties))) { + if (!IcebergCatalogProperties.TYPE_JDBC.equals(catalogProps.getFlavor())) { return; } - String driverUrl = IcebergCatalogFactory.firstNonBlank(properties, IcebergConnectorProperties.JDBC_DRIVER_URL); + String driverUrl = IcebergJdbcMetaStoreProperties.of(properties).getDriverUrl(); if (StringUtils.isNotBlank(driverUrl)) { validationContext.validateAndResolveDriverPath(driverUrl); } @@ -1400,13 +1413,12 @@ public void preCreateValidation(ConnectorValidationContext validationContext) th * {@link JdbcDriverSupport#resolveDriverUrl} against {@code ConnectorContext.getEnvironment()}. */ private void maybeRegisterJdbcDriver() { - String driverUrl = IcebergCatalogFactory.firstNonBlank(properties, IcebergConnectorProperties.JDBC_DRIVER_URL); + IcebergJdbcMetaStoreProperties jdbc = IcebergJdbcMetaStoreProperties.of(properties); + String driverUrl = jdbc.getDriverUrl(); if (StringUtils.isBlank(driverUrl)) { return; } - String driverClass = - IcebergCatalogFactory.firstNonBlank(properties, IcebergConnectorProperties.JDBC_DRIVER_CLASS); - registerJdbcDriver(driverUrl, driverClass); + registerJdbcDriver(driverUrl, jdbc.getDriverClass()); LOG.info("Using dynamic JDBC driver for Iceberg JDBC catalog from: {}", driverUrl); } @@ -1418,15 +1430,13 @@ private void maybeRegisterJdbcDriver() { *

A null context is a direct-construction unit test, which has neither channel. */ private String configuredDriversDir() { - return context == null ? null : ConnectorConf.get(context, - IcebergConnectorProperties.CONF_DRIVERS_DIR, - IcebergConnectorProperties.ENV_JDBC_DRIVERS_DIR, null); + return IcebergConf.driversDir(context); } /** The FE install root. Engine-wide rather than this connector's, so it stays in the environment. */ private String configuredDorisHome() { return context == null ? null - : context.getEnvironment().get(IcebergConnectorProperties.ENV_DORIS_HOME); + : IcebergConf.dorisHome(context); } private void registerJdbcDriver(String driverUrl, String driverClassName) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java index 9fb98d8f75411d..a75da020914281 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java @@ -144,6 +144,7 @@ public class IcebergConnectorMetadata implements ConnectorMetadata { private static final String TABLE_COMMENT_PROP = "comment"; private final IcebergCatalogOps catalogOps; + private final IcebergCatalogProperties catalogProps; private final Map properties; // Every remote metadata READ is wrapped in context.executeAuthenticated(...) so the FE-injected // Kerberos UGI applies — legacy IcebergMetadataOps wrapped each call in executionAuthenticator.execute, @@ -175,36 +176,36 @@ public class IcebergConnectorMetadata implements ConnectorMetadata { private final ConnectorMetadataCache mvccPartitionViewCache; private final ConnectorMetadataCache> listPartitionsViewCache; - public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, + public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, IcebergCatalogProperties catalogProps, ConnectorContext context) { - this(catalogOps, properties, context, new IcebergLatestSnapshotCache(0L, 1), null, null); + this(catalogOps, catalogProps, context, new IcebergLatestSnapshotCache(0L, 1), null, null); } /** Convenience ctor without a cross-query table cache (tableCache null); used by MVCC/statistics tests. */ - public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, + public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, IcebergCatalogProperties catalogProps, ConnectorContext context, IcebergLatestSnapshotCache latestSnapshotCache) { - this(catalogOps, properties, context, latestSnapshotCache, null, null); + this(catalogOps, catalogProps, context, latestSnapshotCache, null, null); } /** Convenience ctor without a partition-view cache (partitionCache null). */ - public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, + public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, IcebergCatalogProperties catalogProps, ConnectorContext context, IcebergLatestSnapshotCache latestSnapshotCache, IcebergTableCache tableCache) { - this(catalogOps, properties, context, latestSnapshotCache, tableCache, null); + this(catalogOps, catalogProps, context, latestSnapshotCache, tableCache, null); } - public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, + public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, IcebergCatalogProperties catalogProps, ConnectorContext context, IcebergLatestSnapshotCache latestSnapshotCache, IcebergTableCache tableCache, IcebergPartitionCache partitionCache) { - this(catalogOps, properties, context, latestSnapshotCache, tableCache, partitionCache, null); + this(catalogOps, catalogProps, context, latestSnapshotCache, tableCache, partitionCache, null); } /** Convenience ctor without the PERF-06 derived partition-view caches (both null). */ - public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, + public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, IcebergCatalogProperties catalogProps, ConnectorContext context, IcebergLatestSnapshotCache latestSnapshotCache, IcebergTableCache tableCache, IcebergPartitionCache partitionCache, IcebergCommentCache commentCache) { - this(catalogOps, properties, context, latestSnapshotCache, tableCache, partitionCache, commentCache, + this(catalogOps, catalogProps, context, latestSnapshotCache, tableCache, partitionCache, commentCache, null, null); } @@ -215,14 +216,15 @@ public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map compute directly every call). */ - public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, Map properties, + public IcebergConnectorMetadata(IcebergCatalogOps catalogOps, IcebergCatalogProperties catalogProps, ConnectorContext context, IcebergLatestSnapshotCache latestSnapshotCache, IcebergTableCache tableCache, IcebergPartitionCache partitionCache, IcebergCommentCache commentCache, ConnectorMetadataCache mvccPartitionViewCache, ConnectorMetadataCache> listPartitionsViewCache) { this.catalogOps = catalogOps; - this.properties = properties; + this.catalogProps = catalogProps; + this.properties = catalogProps.getRaw(); this.context = context; this.latestSnapshotCache = latestSnapshotCache; this.tableCache = tableCache; @@ -844,8 +846,8 @@ public TTableDescriptor buildTableDescriptor( ConnectorSession session, long tableId, String tableName, String dbName, String remoteName, int numCols, long catalogId) { - if (IcebergConnectorProperties.TYPE_HMS.equalsIgnoreCase( - properties.get(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE))) { + if (IcebergCatalogProperties.TYPE_HMS.equalsIgnoreCase( + properties.get(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE))) { THiveTable tHiveTable = new THiveTable(dbName, tableName, new HashMap<>()); TTableDescriptor desc = new TTableDescriptor( tableId, TTableType.HIVE_TABLE, numCols, 0, tableName, dbName); @@ -1640,14 +1642,14 @@ private static boolean isComplexType(ConnectorType type) { return "ARRAY".equals(name) || "MAP".equals(name) || "STRUCT".equals(name); } - /** The configured {@code iceberg.catalog.type}, or {@code null} when unset. */ + /** The configured metastore backend, lower-cased, or {@code null} when the catalog names none. */ private String catalogType() { - return properties.get(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE); + return catalogProps.getFlavor(); } - /** Whether this is an HMS-backed iceberg catalog (case-insensitive, matching the read-path fork). */ + /** Whether this is an HMS-backed iceberg catalog (matching the read-path fork). */ private boolean isHmsCatalog() { - return IcebergConnectorProperties.TYPE_HMS.equalsIgnoreCase(catalogType()); + return IcebergCatalogProperties.TYPE_HMS.equals(catalogType()); } // ========== E7: System Tables (P6.5) ========== @@ -2218,12 +2220,8 @@ public ConnectorTableHandle applyTopnLazyMaterialization(ConnectorSession sessio private List parseSchema(Schema schema) { List fields = schema.columns(); List columns = new ArrayList<>(fields.size()); - boolean enableVarbinary = Boolean.parseBoolean( - properties.getOrDefault( - IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); - boolean enableTimestampTz = Boolean.parseBoolean( - properties.getOrDefault( - IcebergConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + boolean enableVarbinary = catalogProps.isEnableMappingVarbinary(); + boolean enableTimestampTz = catalogProps.isEnableMappingTimestampTz(); for (Types.NestedField field : fields) { // Legacy IcebergUtils.parseSchema parity (mirrors PaimonConnectorMetadata): the column name is diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProperties.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProperties.java deleted file mode 100644 index ac8289cb022256..00000000000000 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProperties.java +++ /dev/null @@ -1,189 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.iceberg; - -/** - * Property constants for Iceberg connector configuration. - * Mirrors keys from fe-core's Iceberg property classes without taking - * a compile-time dependency on fe-core. - */ -public final class IcebergConnectorProperties { - - private IcebergConnectorProperties() { - } - - // -- Deployment-level settings, read from this plugin's own iceberg.conf (named after - // ConnectorProvider.name()). Each falls back to the ENV_ name below it, which is the fe.conf key it - // used to live under and still works. - // - // Both are shared with other connectors at the fe.conf end -- one jdbc_drivers_dir and one - // hive_metastore_client_timeout_second serve jdbc, iceberg and paimon. A plugin conf cannot express - // that, so a deployment that moves to these files sets the value in each plugin's own conf. That is - // the accepted cost of a per-plugin file; the fe.conf keys stay as the shared fallback. -- - public static final String CONF_DRIVERS_DIR = "drivers_dir"; - public static final String CONF_METASTORE_CLIENT_TIMEOUT_SECOND = "metastore_client_timeout_second"; - - /** The fe.conf name of {@link #CONF_DRIVERS_DIR}, forwarded through the engine environment. */ - public static final String ENV_JDBC_DRIVERS_DIR = "jdbc_drivers_dir"; - /** The fe.conf name of {@link #CONF_METASTORE_CLIENT_TIMEOUT_SECOND}. */ - public static final String ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND = - "hive_metastore_client_timeout_second"; - /** Engine-wide, not this connector's: the FE install root. Stays in the engine environment. */ - public static final String ENV_DORIS_HOME = "doris_home"; - /** Legacy default when neither channel names a metastore client timeout. */ - public static final String DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND = "10"; - - // -- Catalog type (second-level dispatch) -- - public static final String ICEBERG_CATALOG_TYPE = "iceberg.catalog.type"; - - // -- Supported catalog type values -- - public static final String TYPE_REST = "rest"; - public static final String TYPE_HMS = "hms"; - public static final String TYPE_GLUE = "glue"; - public static final String TYPE_JDBC = "jdbc"; - public static final String TYPE_HADOOP = "hadoop"; - public static final String TYPE_S3_TABLES = "s3tables"; - - // -- Warehouse -- - public static final String WAREHOUSE = "warehouse"; - - // -- Type mapping options -- - // Dotted keys matching CatalogProperty.ENABLE_MAPPING_* — the exact spelling that real catalog - // property maps carry. The underscore spelling never matches a live catalog map and reads - // default-false (silent loss of the BINARY->VARBINARY / TIMESTAMP_TZ->TIMESTAMPTZ mapping). - public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; - public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; - - // -- REST catalog options -- - public static final String REST_NESTED_NAMESPACE_ENABLED = "iceberg.rest.nested-namespace-enabled"; - public static final String REST_VIEW_ENABLED = "iceberg.rest.view-enabled"; - - // -- REST per-user session (OIDC delegated credential; #63068 re-migration) -- - // iceberg.rest.session = none (default, one shared catalog identity) | user (project the querying user's - // delegated credential onto a per-request Iceberg REST SessionCatalog; requires security.type=oauth2 and - // gates the SUPPORTS_USER_SESSION capability). delegated-token-mode picks how the token is attached: - // access_token = verbatim OAuth2 bearer; token_exchange = typed token key so the REST server exchanges it. - public static final String REST_SESSION = "iceberg.rest.session"; - public static final String SESSION_NONE = "none"; - public static final String SESSION_USER = "user"; - public static final String REST_DELEGATED_TOKEN_MODE = "iceberg.rest.oauth2.delegated-token-mode"; - public static final String DELEGATED_TOKEN_MODE_ACCESS_TOKEN = "access_token"; - public static final String DELEGATED_TOKEN_MODE_TOKEN_EXCHANGE = "token_exchange"; - // Optional per-session OAuth2 AuthSession timeout (maps to CatalogProperties.AUTH_SESSION_TIMEOUT_MS). - public static final String REST_SESSION_TIMEOUT = "iceberg.rest.session-timeout"; - - // -- Namespace hierarchy (REST 3-level ..

) -- - // Mirrors legacy IcebergExternalCatalog.EXTERNAL_CATALOG_NAME: when present, this catalog level is - // appended to every namespace and roots database listing. - public static final String EXTERNAL_CATALOG_NAME = "external_catalog.name"; - - // -- Cache configuration -- - public static final String TABLE_CACHE_ENABLE = "meta.cache.iceberg.table.enable"; - public static final String TABLE_CACHE_TTL = "meta.cache.iceberg.table.ttl-second"; - public static final String TABLE_CACHE_CAPACITY = "meta.cache.iceberg.table.capacity"; - public static final String MANIFEST_CACHE_ENABLE = "meta.cache.iceberg.manifest.enable"; - public static final String MANIFEST_CACHE_TTL = "meta.cache.iceberg.manifest.ttl-second"; - public static final String MANIFEST_CACHE_CAPACITY = "meta.cache.iceberg.manifest.capacity"; - - // ===================================================================== - // Per-flavor INPUT alias keys + non-SDK literal EMITTED keys (T05). - // Mirror the legacy fe-core Iceberg*MetaStoreProperties @ConnectorProperty aliases and the - // literal catalog-option keys they emit. Keys that ARE iceberg-SDK constants - // (CatalogProperties / S3FileIOProperties / AwsProperties / AwsClientProperties / OAuth2Properties) - // are referenced via the SDK in IcebergCatalogFactory, not duplicated here. - // ===================================================================== - - // -- REST input aliases (legacy IcebergRestProperties @ConnectorProperty names) -- - public static final String REST_URI = "iceberg.rest.uri"; - public static final String URI = "uri"; - public static final String REST_PREFIX = "iceberg.rest.prefix"; - public static final String REST_SECURITY_TYPE = "iceberg.rest.security.type"; - public static final String REST_OAUTH2_TOKEN = "iceberg.rest.oauth2.token"; - public static final String REST_OAUTH2_CREDENTIAL = "iceberg.rest.oauth2.credential"; - public static final String REST_OAUTH2_SCOPE = "iceberg.rest.oauth2.scope"; - public static final String REST_OAUTH2_SERVER_URI = "iceberg.rest.oauth2.server-uri"; - public static final String REST_OAUTH2_TOKEN_REFRESH_ENABLED = "iceberg.rest.oauth2.token-refresh-enabled"; - public static final String REST_VENDED_CREDENTIALS_ENABLED = "iceberg.rest.vended-credentials-enabled"; - public static final String REST_SIGV4_ENABLED = "iceberg.rest.sigv4-enabled"; - public static final String REST_SIGNING_NAME = "iceberg.rest.signing-name"; - public static final String REST_SIGNING_REGION = "iceberg.rest.signing-region"; - public static final String REST_ACCESS_KEY_ID = "iceberg.rest.access-key-id"; - public static final String REST_SECRET_ACCESS_KEY = "iceberg.rest.secret-access-key"; - public static final String REST_SESSION_TOKEN = "iceberg.rest.session-token"; - public static final String REST_CREDENTIALS_PROVIDER_TYPE = "iceberg.rest.credentials_provider_type"; - public static final String REST_CONNECTION_TIMEOUT_MS = "iceberg.rest.connection-timeout-ms"; - public static final String REST_SOCKET_TIMEOUT_MS = "iceberg.rest.socket-timeout-ms"; - - // -- REST emitted literal keys / values / defaults (non-SDK) -- - public static final String REST_PREFIX_KEY = "prefix"; - public static final String REST_VENDED_CREDENTIALS_HEADER = "header.X-Iceberg-Access-Delegation"; - public static final String REST_VENDED_CREDENTIALS_VALUE = "vended-credentials"; - public static final String REST_CONNECTION_TIMEOUT_MS_KEY = "rest.client.connection-timeout-ms"; - public static final String REST_SOCKET_TIMEOUT_MS_KEY = "rest.client.socket-timeout-ms"; - public static final String REST_SIGNING_NAME_KEY = "rest.signing-name"; - public static final String REST_SIGV4_ENABLED_KEY = "rest.sigv4-enabled"; - public static final String REST_SIGNING_REGION_KEY = "rest.signing-region"; - public static final String DEFAULT_REST_CONNECTION_TIMEOUT_MS = "10000"; - public static final String DEFAULT_REST_SOCKET_TIMEOUT_MS = "60000"; - public static final String SECURITY_TYPE_OAUTH2 = "oauth2"; - public static final String SIGNING_NAME_GLUE = "glue"; - public static final String SIGNING_NAME_S3TABLES = "s3tables"; - public static final String PROVIDER_MODE_DEFAULT = "DEFAULT"; - - // -- GLUE input alias arrays (legacy AWSGlueMetaStoreBaseProperties / IcebergGlueMetaStoreProperties) -- - public static final String[] GLUE_ENDPOINT = {"glue.endpoint", "aws.endpoint", "aws.glue.endpoint"}; - public static final String[] GLUE_REGION = {"glue.region", "aws.region", "aws.glue.region"}; - public static final String[] GLUE_ACCESS_KEY = { - "glue.access_key", "aws.glue.access-key", "client.credentials-provider.glue.access_key"}; - public static final String[] GLUE_SECRET_KEY = { - "glue.secret_key", "aws.glue.secret-key", "client.credentials-provider.glue.secret_key"}; - public static final String[] GLUE_SESSION_TOKEN = {"aws.glue.session-token"}; - public static final String[] GLUE_IAM_ROLE = {"glue.role_arn"}; - public static final String[] GLUE_EXTERNAL_ID = {"glue.external_id"}; - - // -- GLUE emitted literal keys / values / defaults (non-SDK) -- - public static final String GLUE_CREDENTIALS_PROVIDER_KEY = "client.credentials-provider"; - public static final String GLUE_CREDENTIALS_PROVIDER_2X = - "org.apache.doris.connector.iceberg.glue.ConfigurationAWSCredentialsProvider2x"; - public static final String GLUE_CREDENTIALS_PROVIDER_ACCESS_KEY = "client.credentials-provider.glue.access_key"; - public static final String GLUE_CREDENTIALS_PROVIDER_SECRET_KEY = "client.credentials-provider.glue.secret_key"; - public static final String GLUE_CREDENTIALS_PROVIDER_SESSION_TOKEN = - "client.credentials-provider.glue.session_token"; - public static final String AWS_REGION_KEY = "aws.region"; - public static final String GLUE_CHECKED_WAREHOUSE = "s3://doris"; - public static final String GLUE_DEFAULT_REGION = "us-east-1"; - - // -- JDBC input aliases (legacy IcebergJdbcMetaStoreProperties @ConnectorProperty names) -- - public static final String[] JDBC_URI = {"uri", "iceberg.jdbc.uri"}; - public static final String JDBC_USER = "iceberg.jdbc.user"; - public static final String JDBC_PASSWORD = "iceberg.jdbc.password"; - public static final String JDBC_INIT_CATALOG_TABLES = "iceberg.jdbc.init-catalog-tables"; - public static final String JDBC_SCHEMA_VERSION = "iceberg.jdbc.schema-version"; - public static final String JDBC_STRICT_MODE = "iceberg.jdbc.strict-mode"; - public static final String JDBC_CATALOG_NAME = "iceberg.jdbc.catalog_name"; - public static final String JDBC_DRIVER_URL = "iceberg.jdbc.driver_url"; - public static final String JDBC_DRIVER_CLASS = "iceberg.jdbc.driver_class"; - - // -- JDBC emitted literal keys (non-SDK) -- - public static final String JDBC_PREFIX = "jdbc."; - public static final String JDBC_USER_KEY = "jdbc.user"; - public static final String JDBC_PASSWORD_KEY = "jdbc.password"; - public static final String JDBC_INIT_CATALOG_TABLES_KEY = "jdbc.init-catalog-tables"; - public static final String JDBC_SCHEMA_VERSION_KEY = "jdbc.schema-version"; - public static final String JDBC_STRICT_MODE_KEY = "jdbc.strict-mode"; -} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java index f03c858991e6dc..98b070d188c844 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorProvider.java @@ -17,8 +17,6 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.metastore.spi.MetaStoreProviders; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorProvider; @@ -57,47 +55,15 @@ public Set acceptedCreateTableEngineNames() { } /** - * Validates catalog properties at CREATE CATALOG time via the shared metastore parsers (P6-T10): the - * flavor is resolved from {@code iceberg.catalog.type} ({@link IcebergCatalogFactory#resolveFlavor}) and - * {@link MetaStoreProviders#bindForType} selects the iceberg backend, whose {@code validate()} enforces - * the per-flavor fail-fast rules — REST (security/creds enums, OAuth2, signing, AK/SK), Glue - * (AK/SK-together, endpoint https, at-least-one-credential), JDBC (uri/catalog_name/warehouse), and the - * shared HMS/DLF connection checks; hadoop/s3tables are no-op (their storage is validated upstream at - * fe-filesystem bind). Storage is not needed for validation, so an empty storage map is passed. A blank - * or unknown {@code iceberg.catalog.type} makes {@code bindForType} throw (no provider supports it). - * Throws {@link IllegalArgumentException}, which {@code PluginDrivenExternalCatalog.checkProperties} - * wraps into a DdlException. - * - *

The meta-cache knobs are validated first (restoring the legacy - * {@code IcebergExternalCatalog.checkProperties} fail-fast that was dropped at the SPI cutover), so a - * bad {@code meta.cache.iceberg.*} value is rejected at CREATE/ALTER instead of being silently - * coerced to a cache-disabling default. + * Validates catalog properties at CREATE CATALOG time. Everything this used to spell out lives on + * {@link IcebergCatalogProperties} now: the meta-cache knobs and the per-flavor backend rules alike are + * CREATE-time-only, so they belong next to the binding rather than beside it, and keeping them out of + * {@code of(Map)} is what lets a catalog created before a rule existed still come back after an FE + * restart. Throws {@link IllegalArgumentException}, which {@code PluginDrivenExternalCatalog + * .checkProperties} wraps into a DdlException. */ @Override public void validateProperties(Map properties) { - checkMetaCacheProperties(properties); - String flavor = IcebergCatalogFactory.resolveFlavor(properties); - MetaStoreProviders.bindForType(flavor, properties, Collections.emptyMap()).validate(); - } - - /** - * Byte-for-byte parity with the legacy {@code IcebergExternalCatalog.checkProperties}: table/manifest - * {@code enable} must be boolean, {@code ttl-second} must be a long ≥ -1 (the "no expiration" - * sentinel), {@code capacity} must be a long ≥ 0. Absent keys are skipped. - */ - private static void checkMetaCacheProperties(Map properties) { - CacheSpec.checkBooleanProperty(properties.get(IcebergConnectorProperties.TABLE_CACHE_ENABLE), - IcebergConnectorProperties.TABLE_CACHE_ENABLE); - CacheSpec.checkLongProperty(properties.get(IcebergConnectorProperties.TABLE_CACHE_TTL), - -1L, IcebergConnectorProperties.TABLE_CACHE_TTL); - CacheSpec.checkLongProperty(properties.get(IcebergConnectorProperties.TABLE_CACHE_CAPACITY), - 0L, IcebergConnectorProperties.TABLE_CACHE_CAPACITY); - - CacheSpec.checkBooleanProperty(properties.get(IcebergConnectorProperties.MANIFEST_CACHE_ENABLE), - IcebergConnectorProperties.MANIFEST_CACHE_ENABLE); - CacheSpec.checkLongProperty(properties.get(IcebergConnectorProperties.MANIFEST_CACHE_TTL), - -1L, IcebergConnectorProperties.MANIFEST_CACHE_TTL); - CacheSpec.checkLongProperty(properties.get(IcebergConnectorProperties.MANIFEST_CACHE_CAPACITY), - 0L, IcebergConnectorProperties.MANIFEST_CACHE_CAPACITY); + IcebergCatalogProperties.of(properties).checkCreateTimeOnlyRules(); } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java index d306facec300e2..a38af307a3d5a2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergScanPlanProvider.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorStorageContext; @@ -213,13 +214,11 @@ public class IcebergScanPlanProvider implements ConnectorScanPlanProvider { // (splitFiles). When enabled, planScan re-plans at the manifest level so the per-manifest data/delete-file // reads hit the connector-owned IcebergManifestCache. The .ttl-second/.capacity properties feed ONLY this // enable formula (legacy quirk); the cache itself is fixed no-TTL / capacity 100000. - private static final String MANIFEST_CACHE_ENABLE = "meta.cache.iceberg.manifest.enable"; - private static final String MANIFEST_CACHE_TTL_SECOND = "meta.cache.iceberg.manifest.ttl-second"; - private static final String MANIFEST_CACHE_CAPACITY = "meta.cache.iceberg.manifest.capacity"; private static final boolean DEFAULT_MANIFEST_CACHE_ENABLE = false; private static final long DEFAULT_MANIFEST_CACHE_TTL_SECOND = 48L * 60 * 60; private static final long DEFAULT_MANIFEST_CACHE_CAPACITY = 1024L; + private final IcebergCatalogProperties catalogProps; private final Map properties; // Per-request catalog-ops resolver: applied with the current ConnectorSession to obtain the IcebergCatalogOps // for that request. For a iceberg.rest.session=user catalog the connector passes this::newCatalogBackedOps so @@ -257,31 +256,31 @@ public class IcebergScanPlanProvider implements ConnectorScanPlanProvider { @VisibleForTesting int perFileInvariantComputeCount; - public IcebergScanPlanProvider(Map properties, IcebergCatalogOps catalogOps) { - this(properties, catalogOps, null, null); + public IcebergScanPlanProvider(IcebergCatalogProperties catalogProps, IcebergCatalogOps catalogOps) { + this(catalogProps, catalogOps, null, null); } - public IcebergScanPlanProvider(Map properties, IcebergCatalogOps catalogOps, + public IcebergScanPlanProvider(IcebergCatalogProperties catalogProps, IcebergCatalogOps catalogOps, ConnectorContext context) { - this(properties, catalogOps, context, null); + this(catalogProps, catalogOps, context, null); } - public IcebergScanPlanProvider(Map properties, IcebergCatalogOps catalogOps, + public IcebergScanPlanProvider(IcebergCatalogProperties catalogProps, IcebergCatalogOps catalogOps, ConnectorContext context, IcebergManifestCache manifestCache) { // Constant resolver: these ctors (offline tests + the pre-session connector paths) bind a single ops that // ignores the session, so existing behaviour/tests are byte-identical. No cross-query cache (tableCache // null) — the per-statement scope still dedups within a statement. - this(properties, session -> catalogOps, context, manifestCache, null); + this(catalogProps, session -> catalogOps, context, manifestCache, null); } /** * Session-aware convenience ctor without a cross-query table cache (tableCache null); used by the offline * session-routing tests. The per-statement scope still dedups within a statement. */ - public IcebergScanPlanProvider(Map properties, + public IcebergScanPlanProvider(IcebergCatalogProperties catalogProps, Function catalogOpsResolver, ConnectorContext context, IcebergManifestCache manifestCache) { - this(properties, catalogOpsResolver, context, manifestCache, null); + this(catalogProps, catalogOpsResolver, context, manifestCache, null); } /** @@ -290,10 +289,10 @@ public IcebergScanPlanProvider(Map properties, * resolves the querying user's per-request delegated catalog (the connector passes * {@code this::newCatalogBackedOps}); every other catalog resolves the single shared ops. */ - public IcebergScanPlanProvider(Map properties, + public IcebergScanPlanProvider(IcebergCatalogProperties catalogProps, Function catalogOpsResolver, ConnectorContext context, IcebergManifestCache manifestCache, IcebergTableCache tableCache) { - this(properties, catalogOpsResolver, context, manifestCache, tableCache, null); + this(catalogProps, catalogOpsResolver, context, manifestCache, tableCache, null); } /** @@ -301,11 +300,12 @@ public IcebergScanPlanProvider(Map properties, * inferred-file-format cache ({@code formatCache}). The 5-arg ctor delegates here with a null format cache * (offline tests + pre-cache paths resolve {@code file_format_type} live). */ - public IcebergScanPlanProvider(Map properties, + public IcebergScanPlanProvider(IcebergCatalogProperties catalogProps, Function catalogOpsResolver, ConnectorContext context, IcebergManifestCache manifestCache, IcebergTableCache tableCache, IcebergFormatCache formatCache) { - this.properties = properties; + this.catalogProps = catalogProps; + this.properties = catalogProps.getRaw(); this.catalogOpsResolver = catalogOpsResolver; this.context = context; this.manifestCache = manifestCache; @@ -976,8 +976,7 @@ private List doPlanPositionDeletesSystemTableScan(IcebergTab boolean partitionRequested = isPositionDeletesPartitionColumnRequested(columns); List outputPartitionFields = partitionRequested ? getPositionDeletesOutputPartitionFields(metadataTable) : Collections.emptyList(); - boolean enableMappingVarbinary = Boolean.parseBoolean( - properties.getOrDefault(IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); + boolean enableMappingVarbinary = catalogProps.isEnableMappingVarbinary(); ZoneId zone = resolveSessionZone(session); Map vendedToken = context != null ? extractVendedToken(metadataTable, restVendedCredentialsEnabled()) : Collections.emptyMap(); @@ -1538,8 +1537,11 @@ private boolean restVendedCredentialsEnabled() { * vended-credentials gate to its hadoop config / output path). Pure function of the catalog properties. */ static boolean restVendedCredentialsEnabled(Map properties) { - return IcebergConnectorProperties.TYPE_REST.equals(IcebergCatalogFactory.resolveFlavor(properties)) - && Boolean.parseBoolean(properties.get(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED)); + // Bound rather than read raw so the flag has ONE reader: the same holder the catalog assembly uses to + // decide whether to send the delegation header. The bind is behind the flavor check, so it happens only + // for a REST catalog, and only on the per-table paths below (never per split). + return IcebergCatalogProperties.TYPE_REST.equals(IcebergCatalogProperties.of(properties).getFlavor()) + && IcebergRestMetaStoreProperties.of(properties).isVendedCredentialsEnabled(); } /** @@ -1688,10 +1690,8 @@ public Map getScanNodeProperties( // FileQueryScanNode.classifyColumn categorizes it PARTITION_KEY, i.e. a NON-file slot filled from // columns_from_path. BE resolves through this dict only the slots it decodes FROM the file, so such a // column is never looked up in it whether or not the query projects it. - boolean enableVarbinary = Boolean.parseBoolean( - properties.getOrDefault(IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); - boolean enableTimestampTz = Boolean.parseBoolean( - properties.getOrDefault(IcebergConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + boolean enableVarbinary = catalogProps.isEnableMappingVarbinary(); + boolean enableTimestampTz = catalogProps.isEnableMappingTimestampTz(); if (!systemTable) { String dict; boolean appendRowLineage = getFormatVersion(table) >= 3; @@ -2677,9 +2677,9 @@ private boolean isManifestCacheEnabled() { return false; } CacheSpec spec = CacheSpec.fromProperties(properties, - MANIFEST_CACHE_ENABLE, DEFAULT_MANIFEST_CACHE_ENABLE, - MANIFEST_CACHE_TTL_SECOND, DEFAULT_MANIFEST_CACHE_TTL_SECOND, - MANIFEST_CACHE_CAPACITY, DEFAULT_MANIFEST_CACHE_CAPACITY); + IcebergConnector.MANIFEST_CACHE_ENABLE, DEFAULT_MANIFEST_CACHE_ENABLE, + IcebergConnector.MANIFEST_CACHE_TTL_SECOND, DEFAULT_MANIFEST_CACHE_TTL_SECOND, + IcebergConnector.MANIFEST_CACHE_CAPACITY, DEFAULT_MANIFEST_CACHE_CAPACITY); return CacheSpec.isCacheEnabled(spec.isEnable(), spec.getTtlSecond(), spec.getCapacity()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java index 2d09fe753a7536..02bb47ac08c544 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergWritePlanProvider.java @@ -130,6 +130,7 @@ private static ConnectorColumn buildRowIdColumn() { "Iceberg row position metadata", false, null, false).invisible(); } + private final IcebergCatalogProperties catalogProps; private final Map properties; // Per-request catalog-ops resolver: applied with the current ConnectorSession to obtain the IcebergCatalogOps // for that request. For a iceberg.rest.session=user catalog the connector passes this::newCatalogBackedOps so @@ -141,11 +142,11 @@ private static ConnectorColumn buildRowIdColumn() { private final Function catalogOpsResolver; private final ConnectorContext context; - public IcebergWritePlanProvider(Map properties, IcebergCatalogOps catalogOps, + public IcebergWritePlanProvider(IcebergCatalogProperties catalogProps, IcebergCatalogOps catalogOps, ConnectorContext context) { // Constant resolver: these ctors (offline tests) bind a single ops that ignores the session, so existing // behaviour/tests are byte-identical. - this(properties, session -> catalogOps, context); + this(catalogProps, session -> catalogOps, context); } /** @@ -154,10 +155,11 @@ public IcebergWritePlanProvider(Map properties, IcebergCatalogOp * resolves the querying user's per-request delegated catalog for the write-side read helpers (the connector * passes {@code this::newCatalogBackedOps}); every other catalog resolves the single shared ops. */ - public IcebergWritePlanProvider(Map properties, + public IcebergWritePlanProvider(IcebergCatalogProperties catalogProps, Function catalogOpsResolver, ConnectorContext context) { - this.properties = properties; + this.catalogProps = catalogProps; + this.properties = catalogProps.getRaw(); this.catalogOpsResolver = catalogOpsResolver; this.context = context; } @@ -274,10 +276,8 @@ private void validateBoundWriteColumns(Table table, IcebergWriteSchemaContext sc throw new DorisConnectorException("Iceberg table schema changed after the write was bound; retry the " + "statement with the latest schema"); } - boolean enableVarbinary = Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); - boolean enableTimestampTz = Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + boolean enableVarbinary = mappingVarbinaryEnabled(); + boolean enableTimestampTz = mappingTimestampTzEnabled(); for (int i = 0; i < currentColumns.size(); ++i) { NestedField current = currentColumns.get(i); ConnectorColumn bound = boundColumns.get(i); @@ -985,13 +985,11 @@ private IcebergWriteSchemaContext resolveWriteSchema(ConnectorSession session, } private boolean mappingVarbinaryEnabled() { - return Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); + return catalogProps.isEnableMappingVarbinary(); } private boolean mappingTimestampTzEnabled() { - return Boolean.parseBoolean(properties.getOrDefault( - IcebergConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + return catalogProps.isEnableMappingTimestampTz(); } private static TFileFormatType toTFileFormatType(FileFormat format) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergBuildTableDescriptorTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergBuildTableDescriptorTest.java index edb7ff49364a2a..5164a65af5810f 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergBuildTableDescriptorTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergBuildTableDescriptorTest.java @@ -56,9 +56,9 @@ public class IcebergBuildTableDescriptorTest { private static IcebergConnectorMetadata metadataWithCatalogType(String catalogType) { Map props = new HashMap<>(); if (catalogType != null) { - props.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, catalogType); + props.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, catalogType); } - return new IcebergConnectorMetadata(null, props, new RecordingConnectorContext()); + return new IcebergConnectorMetadata(null, IcebergCatalogProperties.of(props), new RecordingConnectorContext()); } private static TTableDescriptor build(IcebergConnectorMetadata metadata) { @@ -70,7 +70,7 @@ private static TTableDescriptor build(IcebergConnectorMetadata metadata) { public void buildsHiveTableDescriptorForHmsCatalog() { // hms branch: legacy IcebergSysExternalTable.toThrift / IcebergExternalTable.toThrift send // TTableType.HIVE_TABLE + THiveTable. MUTATION: dropping the hms fork -> ICEBERG_TABLE here -> red. - TTableDescriptor desc = build(metadataWithCatalogType(IcebergConnectorProperties.TYPE_HMS)); + TTableDescriptor desc = build(metadataWithCatalogType(IcebergCatalogProperties.TYPE_HMS)); Assertions.assertNotNull(desc, "buildTableDescriptor must return a typed descriptor, never null (null -> SCHEMA_TABLE fallback)"); @@ -89,7 +89,7 @@ public void buildsHiveTableDescriptorForHmsCatalog() { public void buildsIcebergTableDescriptorForRestCatalog() { // non-hms (rest) branch: legacy sends TTableType.ICEBERG_TABLE + TIcebergTable. // MUTATION: always emitting HIVE_TABLE (paimon-style single type) -> red here. - TTableDescriptor desc = build(metadataWithCatalogType(IcebergConnectorProperties.TYPE_REST)); + TTableDescriptor desc = build(metadataWithCatalogType(IcebergCatalogProperties.TYPE_REST)); Assertions.assertNotNull(desc, "non-hms catalog descriptor must not be null"); Assertions.assertEquals(TTableType.ICEBERG_TABLE, desc.getTableType(), diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java index 3db873d59c027b..b86600b248d7ed 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java @@ -17,6 +17,9 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.metastore.iceberg.glue.IcebergGlueMetaStoreProperties; +import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; +import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.filesystem.properties.StorageProperties; @@ -60,36 +63,24 @@ private static Map props(String... kv) { return m; } - // --------------------------------------------------------------------- - // resolveFlavor — lower-cased iceberg.catalog.type, null when absent/blank - // --------------------------------------------------------------------- - - @Test - public void resolveFlavorLowerCasesTheCatalogType() { - // WHY: the second-level dispatch keys off the flavor; the legacy code lower-cases the raw - // iceberg.catalog.type so a user who writes "REST"/"Hms" still routes correctly. MUTATION: - // returning the raw value (no toLowerCase) -> "REST" != "rest" -> red. - Assertions.assertEquals("rest", - IcebergCatalogFactory.resolveFlavor(props("iceberg.catalog.type", "REST"))); - Assertions.assertEquals("hms", - IcebergCatalogFactory.resolveFlavor(props("iceberg.catalog.type", "Hms"))); - Assertions.assertEquals("glue", - IcebergCatalogFactory.resolveFlavor(props("iceberg.catalog.type", "glue"))); + /** + * Binds the raw map the way {@code buildCatalogProperties} does before calling the rest appender, so these + * tests keep expressing their input as the properties a user writes. + */ + private static void appendRest(Map opts, Map raw, + Optional chosenS3) { + IcebergCatalogFactory.appendRestProperties(opts, IcebergRestMetaStoreProperties.of(raw), raw, chosenS3); } - @Test - public void resolveFlavorReturnsNullWhenAbsent() { - // WHY: an absent iceberg.catalog.type must resolve to null (the "no second-level flavor" - // signal), NOT throw or invent a default. MUTATION: defaulting to a flavor string -> red. - Assertions.assertNull(IcebergCatalogFactory.resolveFlavor(Collections.emptyMap())); + /** Same, for the glue appender. */ + private static void appendGlue(Map opts, Map raw, + Optional chosenS3) { + IcebergCatalogFactory.appendGlueProperties(opts, IcebergGlueMetaStoreProperties.of(raw), chosenS3); } - @Test - public void resolveFlavorReturnsNullWhenBlank() { - // WHY: a present-but-empty value is treated as absent (the production guard is - // null-or-isEmpty), so it must also fold to null. MUTATION: dropping the isEmpty() guard -> - // returns "" -> red. - Assertions.assertNull(IcebergCatalogFactory.resolveFlavor(props("iceberg.catalog.type", ""))); + /** Same, for the jdbc appender. */ + private static void appendJdbc(Map opts, Map raw) { + IcebergCatalogFactory.appendJdbcProperties(opts, IcebergJdbcMetaStoreProperties.of(raw)); } // --------------------------------------------------------------------- @@ -350,12 +341,12 @@ public void appendRestEmitsUriAlwaysWithAliasPriority() { // WHY: legacy puts CatalogProperties.URI UNCONDITIONALLY (field default ""), alias priority // iceberg.rest.uri > uri. MUTATION: only-if-nonblank put OR wrong alias order -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.uri", "https://rest", "uri", "https://other"), Optional.empty()); Assertions.assertEquals("https://rest", opts.get("uri")); Map empty = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(empty, props(), Optional.empty()); + appendRest(empty, props(), Optional.empty()); Assertions.assertEquals("", empty.get("uri"), "uri must be emitted as empty string when no alias is set"); } @@ -363,10 +354,10 @@ public void appendRestEmitsUriAlwaysWithAliasPriority() { public void appendRestEmitsPrefixOnlyWhenSet() { // WHY: legacy emits "prefix" only if iceberg.rest.prefix is non-blank. MUTATION: unconditional put -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, props("iceberg.rest.prefix", "p1"), Optional.empty()); + appendRest(opts, props("iceberg.rest.prefix", "p1"), Optional.empty()); Assertions.assertEquals("p1", opts.get("prefix")); Map none = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(none, props(), Optional.empty()); + appendRest(none, props(), Optional.empty()); Assertions.assertNull(none.get("prefix")); } @@ -376,11 +367,11 @@ public void appendRestEmitsVendedCredentialsHeaderWhenEnabled() { // Boolean.parseBoolean(iceberg.rest.vended-credentials-enabled). MUTATION: wrong header key/value or // emitting when disabled -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.vended-credentials-enabled", "true"), Optional.empty()); Assertions.assertEquals("vended-credentials", opts.get("header.X-Iceberg-Access-Delegation")); Map off = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(off, props(), Optional.empty()); + appendRest(off, props(), Optional.empty()); Assertions.assertNull(off.get("header.X-Iceberg-Access-Delegation")); } @@ -390,11 +381,11 @@ public void appendRestEmitsTimeoutsWithDefaults() { // keys rest.client.connection-timeout-ms / rest.client.socket-timeout-ms. MUTATION: wrong defaults or // wrong literal keys -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, props(), Optional.empty()); + appendRest(opts, props(), Optional.empty()); Assertions.assertEquals("10000", opts.get("rest.client.connection-timeout-ms")); Assertions.assertEquals("60000", opts.get("rest.client.socket-timeout-ms")); Map over = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(over, + appendRest(over, props("iceberg.rest.connection-timeout-ms", "5000", "iceberg.rest.socket-timeout-ms", "7000"), Optional.empty()); Assertions.assertEquals("5000", over.get("rest.client.connection-timeout-ms")); @@ -407,7 +398,7 @@ public void appendRestOAuth2CredentialBranchEmitsCredentialAndTokenRefreshDefaul // server-uri/scope + token-refresh-enabled (default "true" from OAuth2Properties default). // MUTATION: emitting token instead, wrong keys, or dropping token-refresh-enabled -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.security.type", "oauth2", "iceberg.rest.oauth2.credential", "id:secret", "iceberg.rest.oauth2.server-uri", "https://auth", @@ -425,7 +416,7 @@ public void appendRestOAuth2TokenBranchWhenNoCredential() { // WHY: oauth2 with no credential uses the pre-configured token flow: emit OAuth2Properties.TOKEN. // MUTATION: emitting credential / token-refresh-enabled here -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.security.type", "oauth2", "iceberg.rest.oauth2.token", "tok"), Optional.empty()); Assertions.assertEquals("tok", opts.get("token")); Assertions.assertNull(opts.get("credential")); @@ -437,7 +428,7 @@ public void appendRestOAuth2NotAppliedWhenSecurityNotOauth2() { // WHY: the oauth2 block is gated on security.type==oauth2 (default none). MUTATION: applying it // unconditionally would leak credential/token even for a non-oauth2 catalog -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.oauth2.credential", "id:secret"), Optional.empty()); Assertions.assertNull(opts.get("credential")); } @@ -448,7 +439,7 @@ public void appendRestSigningBlockEmitsSigningKeysAndS3ExplicitCredentials() { // glue/s3tables the credentials come from the chosen S3 store, EXPLICIT (static AK/SK) -> rest.* creds // (AwsProperties.REST_*). MUTATION: wrong signing keys, or sourcing creds from the wrong place -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.signing-name", "glue", "iceberg.rest.sigv4-enabled", "true", "iceberg.rest.signing-region", "us-east-1"), Optional.of(new FakeS3CompatibleStorageProperties("S3").accessKey("AK").secretKey("SK") @@ -467,7 +458,7 @@ public void appendRestSigningGlueAssumeRoleWhenNoStaticCreds() { // ARN the glue/s3tables signing path emits the assume-role block (client.factory + client.assume-role.*). // MUTATION: emitting rest.access-key-id from a blank AK, or skipping assume-role -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.signing-name", "s3tables", "iceberg.rest.sigv4-enabled", "true", "iceberg.rest.signing-region", "us-west-2"), Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-west-2") @@ -482,7 +473,7 @@ public void appendRestSigningOtherNameUsesIcebergRestCredentials() { // WHY: a signing-name NOT in {glue,s3tables} uses the iceberg.rest.* explicit creds (not the S3 store). // MUTATION: reading the S3 store here -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.signing-name", "custom", "iceberg.rest.access-key-id", "RAK", "iceberg.rest.secret-access-key", "RSK"), Optional.of(new FakeS3CompatibleStorageProperties("S3").accessKey("SHOULD_NOT_USE") @@ -497,7 +488,7 @@ public void appendRestSigningGlueProviderChainPinsNonDefaultProvider() { // s3.credentials_provider_type must pin client.credentials-provider to that provider class (was silently // dropped). MUTATION: dropping the else branch -> the key is absent -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.signing-name", "glue", "s3.credentials_provider_type", "anonymous"), Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-east-1"))); Assertions.assertEquals(AnonymousCredentialsProvider.class.getName(), @@ -505,7 +496,7 @@ public void appendRestSigningGlueProviderChainPinsNonDefaultProvider() { // DEFAULT / absent -> nothing emitted (SDK default chain, the common case). Map defaultOpts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(defaultOpts, + appendRest(defaultOpts, props("iceberg.rest.signing-name", "glue"), Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-east-1"))); Assertions.assertNull(defaultOpts.get("client.credentials-provider"), @@ -517,7 +508,7 @@ public void appendRestSigningOtherNameProviderChainPinsNonDefaultProvider() { // F14: a non-glue/s3tables signing-name with NO explicit iceberg.rest.* creds falls to PROVIDER_CHAIN; // iceberg.rest.credentials_provider_type pins the provider class. MUTATION: dropping the else -> absent. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, + appendRest(opts, props("iceberg.rest.signing-name", "custom", "iceberg.rest.credentials_provider_type", "web-identity"), Optional.of(new FakeS3CompatibleStorageProperties("S3"))); @@ -531,7 +522,7 @@ public void appendRestNoSigningBlockWhenSigningNameAbsent() { // WHY: the entire signing block is gated on a non-blank signing-name. MUTATION: emitting rest.signing-name // (even empty) without the gate -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendRestProperties(opts, props(), Optional.empty()); + appendRest(opts, props(), Optional.empty()); Assertions.assertNull(opts.get("rest.signing-name")); Assertions.assertNull(opts.get("rest.sigv4-enabled")); } @@ -546,7 +537,7 @@ public void appendGlueEmitsS3KeysUnconditionallyFromChosenStore() { // unset session token is written as "". MUTATION: blank-guarding these puts (like the base S3FileIO path) // -> the empty s3.session-token would be absent -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(opts, props("glue.access_key", "a", "glue.secret_key", "b", + appendGlue(opts, props("glue.access_key", "a", "glue.secret_key", "b", "glue.endpoint", "https://glue.us-east-1.amazonaws.com"), Optional.of(new FakeS3CompatibleStorageProperties("S3").accessKey("S3AK").secretKey("S3SK") .endpoint("https://s3").usePathStyle("true"))); @@ -563,7 +554,7 @@ public void appendGlueAccessKeyBranchEmitsProviderKeysAndWins() { // provider keys and RETURN (mutually exclusive with the IAM-role branch). MUTATION: wrong key/value, or // also emitting client.factory (IAM branch) -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(opts, + appendGlue(opts, props("glue.access_key", "GAK", "glue.secret_key", "GSK", "aws.glue.session-token", "GST", "glue.role_arn", "arn:aws:iam::1:role/should-not-fire", "glue.endpoint", "https://glue.us-east-1.amazonaws.com"), @@ -586,7 +577,7 @@ public void glueSessionTokenSurvivesToTheResolvedCredential() { // provider used to read only ak/sk, so a supplied token was dropped here and AWS then rejected the // temporary credentials. MUTATION: dropping the token read in create() -> AwsBasicCredentials -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(opts, + appendGlue(opts, props("glue.access_key", "GAK", "glue.secret_key", "GSK", "aws.glue.session-token", "GST", "glue.endpoint", "https://glue.us-east-1.amazonaws.com"), Optional.empty()); @@ -609,7 +600,7 @@ public void glueWithoutSessionTokenStaysBasicCredentials() { // -> AwsSessionCredentials.create accepts a blank token silently -> this turns red instead of AWS // rejecting it much later with a confusing 4xx. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(opts, + appendGlue(opts, props("glue.access_key", "GAK", "glue.secret_key", "GSK", "glue.endpoint", "https://glue.us-east-1.amazonaws.com"), Optional.empty()); @@ -629,7 +620,7 @@ public void appendGlueIamRoleBranchWhenNoAccessKey() { // aws.region + client.assume-role.arn/region + optional external-id). MUTATION: wrong keys or skipping // external-id -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(opts, + appendGlue(opts, props("glue.role_arn", "arn:aws:iam::1:role/r", "glue.external_id", "eid", "glue.region", "eu-west-1", "glue.endpoint", "https://glue.eu-west-1.amazonaws.com"), Optional.empty()); @@ -645,7 +636,7 @@ public void appendGlueEmitsEndpointAndClientRegionAlways() { // WHY: legacy always puts glue.endpoint (AwsProperties.GLUE_CATALOG_ENDPOINT) and client.region. MUTATION: // dropping either -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(opts, + appendGlue(opts, props("glue.access_key", "a", "glue.secret_key", "b", "glue.region", "ap-south-1", "glue.endpoint", "https://glue.ap-south-1.amazonaws.com"), Optional.empty()); @@ -658,14 +649,14 @@ public void appendGlueRegionFallsBackToEndpointRegexThenUsEast1() { // WHY: legacy resolves the region from glue.region first, else extracts it from the endpoint host via // ENDPOINT_PATTERN, else us-east-1. MUTATION: not extracting from the endpoint, or wrong fallback -> red. Map fromEndpoint = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(fromEndpoint, + appendGlue(fromEndpoint, props("glue.access_key", "a", "glue.secret_key", "b", "glue.endpoint", "https://glue-fips.ca-central-1.api.aws"), Optional.empty()); Assertions.assertEquals("ca-central-1", fromEndpoint.get("client.region")); Map defaulted = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(defaulted, + appendGlue(defaulted, props("glue.access_key", "a", "glue.secret_key", "b", "glue.endpoint", "https://not-a-glue-host"), Optional.empty()); Assertions.assertEquals("us-east-1", defaulted.get("client.region")); @@ -676,14 +667,14 @@ public void appendGluePutsWarehousePlaceholderOnlyWhenAbsent() { // WHY: legacy putIfAbsent(WAREHOUSE_LOCATION, "s3://doris") — fills the placeholder only when the user // did not supply a warehouse. MUTATION: an unconditional put would clobber the user's warehouse -> red. Map defaulted = new HashMap<>(); - IcebergCatalogFactory.appendGlueProperties(defaulted, + appendGlue(defaulted, props("glue.access_key", "a", "glue.secret_key", "b", "glue.endpoint", "https://glue.x.amazonaws.com"), Optional.empty()); Assertions.assertEquals("s3://doris", defaulted.get("warehouse")); Map userWh = new HashMap<>(); userWh.put("warehouse", "s3://mybucket/wh"); - IcebergCatalogFactory.appendGlueProperties(userWh, + appendGlue(userWh, props("glue.access_key", "a", "glue.secret_key", "b", "glue.endpoint", "https://glue.x.amazonaws.com"), Optional.empty()); Assertions.assertEquals("s3://mybucket/wh", userWh.get("warehouse")); @@ -697,7 +688,7 @@ public void appendGluePutsWarehousePlaceholderOnlyWhenAbsent() { public void appendJdbcEmitsUriWithAliasPriority() { // WHY: legacy uri alias priority is {uri, iceberg.jdbc.uri} (uri wins). MUTATION: wrong order -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendJdbcProperties(opts, + appendJdbc(opts, props("uri", "jdbc:mysql://h/db", "iceberg.jdbc.uri", "jdbc:other")); Assertions.assertEquals("jdbc:mysql://h/db", opts.get("uri")); } @@ -707,7 +698,7 @@ public void appendJdbcAddsDottedKeysOnlyWhenSet() { // WHY: legacy addIfNotBlank maps each iceberg.jdbc. to the dotted jdbc. only when non-blank. // MUTATION: wrong emitted key spelling or emitting a blank value -> red. Map opts = new HashMap<>(); - IcebergCatalogFactory.appendJdbcProperties(opts, + appendJdbc(opts, props("uri", "jdbc:mysql://h/db", "iceberg.jdbc.user", "u", "iceberg.jdbc.password", "p", "iceberg.jdbc.init-catalog-tables", "true", "iceberg.jdbc.schema-version", "V1", "iceberg.jdbc.strict-mode", "false")); @@ -718,7 +709,7 @@ public void appendJdbcAddsDottedKeysOnlyWhenSet() { Assertions.assertEquals("false", opts.get("jdbc.strict-mode")); Map bare = new HashMap<>(); - IcebergCatalogFactory.appendJdbcProperties(bare, props("uri", "jdbc:mysql://h/db")); + appendJdbc(bare, props("uri", "jdbc:mysql://h/db")); Assertions.assertNull(bare.get("jdbc.user"), "an unset jdbc.user must NOT be emitted"); } @@ -733,8 +724,7 @@ public void buildCatalogPropertiesSetsImplAndRemovesType() { // iceberg.catalog.type key is a separate raw key carried by copy-all and is harmless. MUTATION: not // setting impl, or leaving "type" -> red. Map opts = IcebergCatalogFactory.buildCatalogProperties( - props("iceberg.catalog.type", "hadoop", "warehouse", "s3://b/wh", "type", "hadoop"), - "hadoop", Optional.empty()); + IcebergCatalogProperties.of(props("iceberg.catalog.type", "hadoop", "warehouse", "s3://b/wh", "type", "hadoop")), Optional.empty()); Assertions.assertEquals("org.apache.iceberg.hadoop.HadoopCatalog", opts.get("catalog-impl")); Assertions.assertNull(opts.get("type"), "the SDK 'type' key must be removed before building"); } @@ -744,9 +734,8 @@ public void buildCatalogPropertiesRemovesJdbcCatalogNameFromMap() { // WHY: iceberg.jdbc.catalog_name is the positional catalog NAME, removed from the options map by legacy // initCatalog. MUTATION: leaving it in the map -> red (iceberg would treat it as an unknown option). Map opts = IcebergCatalogFactory.buildCatalogProperties( - props("iceberg.catalog.type", "jdbc", "uri", "jdbc:mysql://h/db", "warehouse", "s3://b/wh", - "iceberg.jdbc.catalog_name", "mycat"), - "jdbc", Optional.empty()); + IcebergCatalogProperties.of(props("iceberg.catalog.type", "jdbc", "uri", "jdbc:mysql://h/db", "warehouse", "s3://b/wh", + "iceberg.jdbc.catalog_name", "mycat")), Optional.empty()); Assertions.assertEquals("org.apache.iceberg.jdbc.JdbcCatalog", opts.get("catalog-impl")); Assertions.assertNull(opts.get("iceberg.jdbc.catalog_name"), "the jdbc catalog_name must be consumed positionally, not left in the options map"); @@ -757,7 +746,7 @@ public void buildCatalogPropertiesHmsEmitsNoS3FileIoKeys() { // WHY: legacy iceberg HMS does NOT call toFileIOProperties — object-store access rides the HiveConf, not // the s3.* options. MUTATION: appending the base S3FileIO for HMS -> s3.endpoint present -> red. Map opts = IcebergCatalogFactory.buildCatalogProperties( - props("iceberg.catalog.type", "hms"), "hms", + IcebergCatalogProperties.of(props("iceberg.catalog.type", "hms")), Optional.of(new FakeS3CompatibleStorageProperties("S3").endpoint("https://s3").accessKey("AK"))); Assertions.assertEquals("org.apache.iceberg.hive.HiveCatalog", opts.get("catalog-impl")); Assertions.assertNull(opts.get("s3.endpoint"), "HMS must not emit S3FileIO options"); @@ -772,10 +761,9 @@ public void buildCatalogPropertiesRestVendedPropagatesClientRegionWithoutBoundS3 // carried by copy-all is inert (iceberg reads client.region). Legacy toFileIOProperties supplied this in // its chosen==null branch. MUTATION: dropping the empty-chosenS3 region fallback -> client.region absent -> red. Map opts = IcebergCatalogFactory.buildCatalogProperties( - props("iceberg.catalog.type", "rest", "uri", "https://rest", + IcebergCatalogProperties.of(props("iceberg.catalog.type", "rest", "uri", "https://rest", "iceberg.rest.vended-credentials-enabled", "true", "s3.endpoint", "https://minio:9000", - "s3.region", "us-east-1"), - "rest", Optional.empty()); + "s3.region", "us-east-1")), Optional.empty()); Assertions.assertEquals("us-east-1", opts.get("client.region"), "vended REST (no bound S3) must still translate s3.region -> client.region"); } @@ -789,17 +777,15 @@ public void buildCatalogPropertiesRestVendedResolvesRegionFromWidenedAliases() { // RED before widening: AWS_REGION (uppercase) does not match the narrow lowercase aws.region and // iceberg.rest.signing-region is absent from the narrow 4-alias set -> client.region null. Map viaAwsRegion = IcebergCatalogFactory.buildCatalogProperties( - props("iceberg.catalog.type", "rest", "uri", "https://rest", - "iceberg.rest.vended-credentials-enabled", "true", "AWS_REGION", "us-east-1"), - "rest", Optional.empty()); + IcebergCatalogProperties.of(props("iceberg.catalog.type", "rest", "uri", "https://rest", + "iceberg.rest.vended-credentials-enabled", "true", "AWS_REGION", "us-east-1")), Optional.empty()); Assertions.assertEquals("us-east-1", viaAwsRegion.get("client.region"), "region supplied only via AWS_REGION must translate to client.region"); Map viaSigningRegion = IcebergCatalogFactory.buildCatalogProperties( - props("iceberg.catalog.type", "rest", "uri", "https://rest", + IcebergCatalogProperties.of(props("iceberg.catalog.type", "rest", "uri", "https://rest", "iceberg.rest.vended-credentials-enabled", "true", - "iceberg.rest.signing-region", "eu-west-1"), - "rest", Optional.empty()); + "iceberg.rest.signing-region", "eu-west-1")), Optional.empty()); Assertions.assertEquals("eu-west-1", viaSigningRegion.get("client.region"), "region supplied only via iceberg.rest.signing-region must translate to client.region"); } @@ -815,8 +801,8 @@ public void buildS3TablesCatalogPropertiesEmitsS3FileIoAndWarehouseNoImpl() { // manifest-cache + S3FileIO creds, and adds NEITHER "catalog-impl" NOR removes "type" (those are the // CatalogUtil path's concern). MUTATION: adding catalog-impl, removing type, or dropping S3FileIO -> red. Map opts = IcebergCatalogFactory.buildS3TablesCatalogProperties( - props("iceberg.catalog.type", "s3tables", "type", "iceberg", - "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b"), + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", "type", "iceberg", + "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b")), Optional.of(new FakeS3CompatibleStorageProperties("S3") .endpoint("https://s3.us-east-1.amazonaws.com").region("us-east-1") .accessKey("AK").secretKey("SK").sessionToken("TK").usePathStyle("true"))); @@ -841,7 +827,7 @@ public void buildS3TablesCatalogPropertiesEmitsAssumeRoleWhenNoStaticCreds() { // legacy putS3FileIOCredentialProperties ASSUME_ROLE branch emits. MUTATION: missing assume-role keys // OR leaking static AK/SK -> red. Map opts = IcebergCatalogFactory.buildS3TablesCatalogProperties( - props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-west-2:1:bucket/b"), + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-west-2:1:bucket/b")), Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-west-2") .roleArn("arn:aws:iam::1:role/r").externalId("eid"))); Assertions.assertEquals("org.apache.iceberg.aws.AssumeRoleAwsClientFactory", opts.get("client.factory")); @@ -858,8 +844,8 @@ public void buildS3TablesCatalogPropertiesEmitsProviderChainWhenNoStaticNoRole() // s3.credentials_provider_type pins client.credentials-provider (mirrors legacy putCredentialsProvider). // MUTATION: dropping the else branch in appendS3TablesFileIOProperties -> the key is absent -> red. Map opts = IcebergCatalogFactory.buildS3TablesCatalogProperties( - props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-west-2:1:bucket/b", - "s3.credentials_provider_type", "ENV"), + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-west-2:1:bucket/b", + "s3.credentials_provider_type", "ENV")), Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-west-2"))); Assertions.assertEquals(EnvironmentVariableCredentialsProvider.class.getName(), opts.get("client.credentials-provider")); @@ -874,7 +860,7 @@ public void buildS3TablesCatalogPropertiesExplicitStaticCredsSuppressAssumeRole( // The s3tables path must NOT reuse the always-emit appendS3FileIOProperties helper. MUTATION: emitting the // assume-role block (client.factory / client.assume-role.*) when static creds are present -> red. Map opts = IcebergCatalogFactory.buildS3TablesCatalogProperties( - props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-west-2:1:bucket/b"), + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-west-2:1:bucket/b")), Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-west-2") .accessKey("AK").secretKey("SK").roleArn("arn:aws:iam::1:role/r"))); Assertions.assertEquals("AK", opts.get("s3.access-key-id")); @@ -891,7 +877,7 @@ public void buildS3TablesCatalogPropertiesWithoutStorageOmitsS3FileIo() { // because there is no region to propagate. (When a region IS present it is now emitted; see // buildS3TablesCatalogPropertiesPropagatesClientRegionWithoutBoundS3.) MUTATION: fabricating any s3.* -> red. Map opts = IcebergCatalogFactory.buildS3TablesCatalogProperties( - props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b"), + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b")), Optional.empty()); Assertions.assertEquals("arn:aws:s3tables:us-east-1:1:bucket/b", opts.get("warehouse")); Assertions.assertNull(opts.get("s3.access-key-id")); @@ -906,8 +892,8 @@ public void buildS3TablesCatalogPropertiesPropagatesClientRegionWithoutBoundS3() // DefaultAwsRegionProviderChain. Parallels the REST vended-cred region test. RED at HEAD (the old // ifPresent-only path emitted nothing without a storage). No s3.* credential keys (none are bound). Map opts = IcebergCatalogFactory.buildS3TablesCatalogProperties( - props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b", - "s3.region", "us-east-1"), + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b", + "s3.region", "us-east-1")), Optional.empty()); Assertions.assertEquals("us-east-1", opts.get("client.region"), "no-storage s3tables must propagate s3.region -> client.region for the data-plane S3FileIO"); @@ -915,7 +901,8 @@ public void buildS3TablesCatalogPropertiesPropagatesClientRegionWithoutBoundS3() } // --------------------------------------------------------------------- - // resolveCatalogName — jdbc positional vs default + // resolveCatalogName — jdbc positional vs default (the flavor now comes from the properties, + // so each fixture names its own iceberg.catalog.type rather than passing one alongside) // --------------------------------------------------------------------- @Test @@ -923,7 +910,8 @@ public void resolveCatalogNameUsesJdbcCatalogNameForJdbc() { // WHY: legacy passes iceberg.jdbc.catalog_name as the catalog NAME (overriding the Doris catalog name). // MUTATION: returning the default name for jdbc -> red. Assertions.assertEquals("mycat", IcebergCatalogFactory.resolveCatalogName( - props("iceberg.jdbc.catalog_name", "mycat"), "jdbc", "doris_cat")); + IcebergCatalogProperties.of(props("iceberg.catalog.type", "jdbc", + "iceberg.jdbc.catalog_name", "mycat")), "doris_cat")); } @Test @@ -932,7 +920,8 @@ public void resolveCatalogNameThrowsWhenJdbcCatalogNameMissing() { // must fail loud rather than silently fall back to the Doris catalog name. MUTATION: returning the default // -> no exception -> red. DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, - () -> IcebergCatalogFactory.resolveCatalogName(props(), "jdbc", "doris_cat")); + () -> IcebergCatalogFactory.resolveCatalogName( + IcebergCatalogProperties.of(props("iceberg.catalog.type", "jdbc")), "doris_cat")); Assertions.assertTrue(ex.getMessage().contains("iceberg.jdbc.catalog_name")); } @@ -940,7 +929,8 @@ public void resolveCatalogNameThrowsWhenJdbcCatalogNameMissing() { public void resolveCatalogNameUsesDefaultForNonJdbc() { // WHY: every non-jdbc flavor uses the Doris catalog name. MUTATION: reading catalog_name for hms -> red. Assertions.assertEquals("doris_cat", IcebergCatalogFactory.resolveCatalogName( - props("iceberg.jdbc.catalog_name", "ignored"), "hms", "doris_cat")); + IcebergCatalogProperties.of(props("iceberg.catalog.type", "hms", + "iceberg.jdbc.catalog_name", "ignored")), "doris_cat")); } // --------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogOptionsSnapshotTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogOptionsSnapshotTest.java new file mode 100644 index 00000000000000..57649f8f79475d --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogOptionsSnapshotTest.java @@ -0,0 +1,607 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * WHOLE-MAP snapshots of {@link IcebergCatalogFactory#buildCatalogProperties} (and the bespoke + * {@link IcebergCatalogFactory#buildS3TablesCatalogProperties}), one per flavor and per emission branch. + * + *

Why a snapshot and not more per-key assertions. {@link IcebergCatalogFactoryTest} asserts that + * the keys it names are right; nothing there notices a key that appears, disappears, or is spelled + * differently. These options ARE the catalog connection: the iceberg SDK ignores an option it does not + * recognize, so a dropped or misspelled key does not throw — it produces a catalog that connects with + * different settings than the operator asked for. Asserting the ENTIRE map turns any drift into a diff. + * + *

They exist because the per-flavor assembly is being folded onto the bound + * {@code Iceberg*MetaStoreProperties} holders, retiring the parallel raw-map alias scan. Once the old path + * is gone there is nothing left to compare against, so the reference is captured HERE, before the change, + * and kept afterwards as the permanent guard that holder and assembly stay in agreement. + * + *

Every input map deliberately carries the awkward shapes as well as the ordinary ones: the alias form + * that must win over the plain one, a raw {@code jdbc.*} passthrough, and keys the base copy-all carries + * through untouched. + */ +public class IcebergCatalogOptionsSnapshotTest { + + private static Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + /** + * Sorted so an assertion failure renders as a readable diff of the whole option map. The flavor is no + * longer passed in: it is derived from the input map's {@code iceberg.catalog.type} exactly as a live + * catalog build derives it, so a fixture can no longer claim one flavor while its properties say another. + */ + private static void assertOptions(Map expected, Map input, + Optional chosenS3) { + Assertions.assertEquals(new TreeMap<>(expected), + new TreeMap<>(IcebergCatalogFactory.buildCatalogProperties(IcebergCatalogProperties.of(input), chosenS3))); + } + + private static Optional s3(FakeS3CompatibleStorageProperties fake) { + return Optional.of(fake); + } + + private static Optional noS3() { + return Optional.empty(); + } + + // --------------------------------------------------------------------- + // REST + // --------------------------------------------------------------------- + + /** + * The floor: a REST catalog that sets nothing optional still emits both client timeouts, because the + * legacy fields defaulted non-blank and the emission is unconditional. + */ + @Test + public void restMinimalSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "uri", "https://rest.example/api", + "warehouse", "s3://bucket/wh", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000"), + props("iceberg.catalog.type", "rest", + "uri", "https://rest.example/api", + "warehouse", "s3://bucket/wh"), noS3()); + } + + /** + * OAuth2 client-credentials flow with every optional REST knob set. Pins two things per-key tests miss: + * {@code iceberg.rest.uri} OVERWRITES the plain {@code uri} the copy-all base already carried, and the + * vended-credentials flag emits a header key rather than travelling as itself. + */ + @Test + public void restOAuth2CredentialFlowSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "iceberg.rest.uri", "https://rest/api", + "uri", "https://rest/api", + "iceberg.rest.prefix", "main", + "iceberg.rest.vended-credentials-enabled", "true", + "iceberg.rest.connection-timeout-ms", "5000", + "iceberg.rest.socket-timeout-ms", "15000", + "iceberg.rest.security.type", "oauth2", + "iceberg.rest.oauth2.credential", "cid:csecret", + "iceberg.rest.oauth2.server-uri", "https://auth/token", + "iceberg.rest.oauth2.scope", "catalog", + "iceberg.rest.oauth2.token-refresh-enabled", "false", + "s3.region", "us-west-2", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "prefix", "main", + "header.X-Iceberg-Access-Delegation", "vended-credentials", + "rest.client.connection-timeout-ms", "5000", + "rest.client.socket-timeout-ms", "15000", + "credential", "cid:csecret", + "oauth2-server-uri", "https://auth/token", + "scope", "catalog", + "token-refresh-enabled", "false", + "client.region", "us-west-2"), + props("iceberg.catalog.type", "rest", + "iceberg.rest.uri", "https://rest/api", + "uri", "https://plain-uri-loses-to-the-alias", + "iceberg.rest.prefix", "main", + "iceberg.rest.vended-credentials-enabled", "true", + "iceberg.rest.connection-timeout-ms", "5000", + "iceberg.rest.socket-timeout-ms", "15000", + "iceberg.rest.security.type", "oauth2", + "iceberg.rest.oauth2.credential", "cid:csecret", + "iceberg.rest.oauth2.server-uri", "https://auth/token", + "iceberg.rest.oauth2.scope", "catalog", + "iceberg.rest.oauth2.token-refresh-enabled", "false", + "s3.region", "us-west-2"), noS3()); + } + + /** Pre-configured token flow: no credential, so the token branch fires and no scope/server-uri appear. */ + @Test + public void restOAuth2TokenFlowSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.security.type", "oauth2", + "iceberg.rest.oauth2.token", "tkn-123", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000", + "token", "tkn-123"), + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.security.type", "oauth2", + "iceberg.rest.oauth2.token", "tkn-123"), noS3()); + } + + /** + * Surrounding whitespace is stripped from every {@code iceberg.rest.*} value. + * + *

This CHANGED when the assembly moved onto the bound holder, and the change is the point: the + * property binder has always trimmed, so a uri written with a trailing space was already validated + * trimmed at CREATE while the catalog was built from the untrimmed string. The two now agree. Note the + * raw keys themselves ride through the copy-all base untrimmed — only the values the assembly derives + * are normalized. + */ + @Test + public void restTrimsSurroundingWhitespaceSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "iceberg.rest.uri", " https://rest/api ", + "iceberg.rest.security.type", "oauth2", + "iceberg.rest.oauth2.token", " tkn-123 ", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000", + "uri", "https://rest/api", + "token", "tkn-123"), + props("iceberg.catalog.type", "rest", + "iceberg.rest.uri", " https://rest/api ", + "iceberg.rest.security.type", "oauth2", + "iceberg.rest.oauth2.token", " tkn-123 "), noS3()); + } + + /** + * signing-name=glue with a bound generic-S3 store: the signing block takes its credentials from the + * STORE (not the {@code iceberg.rest.*} keys), and the S3FileIO dialect is emitted on top. + */ + @Test + public void restSigningGlueWithBoundS3Snapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "glue", + "iceberg.rest.signing-region", "us-east-1", + "iceberg.rest.sigv4-enabled", "true", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000", + "rest.signing-name", "glue", + "rest.sigv4-enabled", "true", + "rest.signing-region", "us-east-1", + "rest.access-key-id", "AK", + "rest.secret-access-key", "SK", + "rest.session-token", "ST", + "s3.endpoint", "https://s3", + "s3.path-style-access", "true", + "client.region", "us-east-1", + "s3.access-key-id", "AK", + "s3.secret-access-key", "SK", + "s3.session-token", "ST"), + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "glue", + "iceberg.rest.signing-region", "us-east-1", + "iceberg.rest.sigv4-enabled", "true"), s3(new FakeS3CompatibleStorageProperties("S3").endpoint("https://s3") + .region("us-east-1").accessKey("AK").secretKey("SK").sessionToken("ST") + .usePathStyle("true"))); + } + + /** + * signing-name=glue with a role-only store: the signing block emits the assume-role keys, and the + * S3FileIO pass emits them a second time (same values) plus the dialect. Pinned because the two + * emitters are separate code paths that happen to agree. + */ + @Test + public void restSigningGlueAssumeRoleSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "glue", + "iceberg.rest.signing-region", "us-east-1", + "iceberg.rest.sigv4-enabled", "true", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000", + "rest.signing-name", "glue", + "rest.sigv4-enabled", "true", + "rest.signing-region", "us-east-1", + "client.factory", "org.apache.iceberg.aws.AssumeRoleAwsClientFactory", + "aws.region", "us-east-1", + "client.assume-role.region", "us-east-1", + "client.assume-role.arn", "arn:aws:iam::1:role/r", + "client.assume-role.external-id", "eid", + "client.region", "us-east-1"), + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "glue", + "iceberg.rest.signing-region", "us-east-1", + "iceberg.rest.sigv4-enabled", "true"), s3(new FakeS3CompatibleStorageProperties("S3").region("us-east-1") + .roleArn("arn:aws:iam::1:role/r").externalId("eid"))); + } + + /** + * A non-glue signing name takes the explicit {@code iceberg.rest.*} credentials instead. Also pins that + * the empty-storage region fallback reaches {@code iceberg.rest.signing-region} — it is one of the + * region aliases, so a catalog that names no S3 region at all still gets a client.region. + */ + @Test + public void restSigningOtherNameWithRestCredentialsSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "custom", + "iceberg.rest.signing-region", "cn-north-1", + "iceberg.rest.sigv4-enabled", "true", + "iceberg.rest.access-key-id", "RAK", + "iceberg.rest.secret-access-key", "RSK", + "iceberg.rest.session-token", "RST", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000", + "rest.signing-name", "custom", + "rest.sigv4-enabled", "true", + "rest.signing-region", "cn-north-1", + "rest.access-key-id", "RAK", + "rest.secret-access-key", "RSK", + "rest.session-token", "RST", + "client.region", "cn-north-1"), + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "custom", + "iceberg.rest.signing-region", "cn-north-1", + "iceberg.rest.sigv4-enabled", "true", + "iceberg.rest.access-key-id", "RAK", + "iceberg.rest.secret-access-key", "RSK", + "iceberg.rest.session-token", "RST"), noS3()); + } + + /** + * PROVIDER_CHAIN: no explicit REST credentials and a non-DEFAULT provider mode pins the provider class. + * The mode is read from the three-key alias set that spans the s3/glue/rest namespaces. + */ + @Test + public void restSigningProviderChainSnapshot() { + assertOptions( + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "custom", + "iceberg.rest.credentials_provider_type", "instance-profile", + "catalog-impl", "org.apache.iceberg.rest.RESTCatalog", + "rest.client.connection-timeout-ms", "10000", + "rest.client.socket-timeout-ms", "60000", + "rest.signing-name", "custom", + "rest.sigv4-enabled", "", + "rest.signing-region", "", + "client.credentials-provider", + "software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider"), + props("iceberg.catalog.type", "rest", + "uri", "https://rest/api", + "iceberg.rest.signing-name", "custom", + "iceberg.rest.credentials_provider_type", "instance-profile"), noS3()); + } + + // --------------------------------------------------------------------- + // GLUE + // --------------------------------------------------------------------- + + /** + * Glue with explicit AK/SK: the five s3.* FileIO keys come unconditionally from the bound store (empty + * strings included — legacy plain puts), the region is extracted from the endpoint host, and the + * credentials go through the Doris configuration provider rather than the SDK chain. + */ + @Test + public void glueStaticCredentialsSnapshot() { + assertOptions( + props("iceberg.catalog.type", "glue", + "warehouse", "s3://bucket/wh", + "glue.endpoint", "https://glue.us-east-1.amazonaws.com", + "glue.access_key", "GAK", + "glue.secret_key", "GSK", + "aws.glue.session-token", "GST", + "catalog-impl", "org.apache.iceberg.aws.glue.GlueCatalog", + "s3.access-key-id", "AK", + "s3.secret-access-key", "SK", + "s3.endpoint", "https://s3", + "s3.path-style-access", "false", + "s3.session-token", "ST", + "client.credentials-provider", + "org.apache.doris.connector.iceberg.glue.ConfigurationAWSCredentialsProvider2x", + "client.credentials-provider.glue.access_key", "GAK", + "client.credentials-provider.glue.secret_key", "GSK", + "client.credentials-provider.glue.session_token", "GST", + "client.region", "us-east-1"), + props("iceberg.catalog.type", "glue", + "warehouse", "s3://bucket/wh", + "glue.endpoint", "https://glue.us-east-1.amazonaws.com", + "glue.access_key", "GAK", + "glue.secret_key", "GSK", + "aws.glue.session-token", "GST"), s3(new FakeS3CompatibleStorageProperties("S3").endpoint("https://s3") + .accessKey("AK").secretKey("SK").sessionToken("ST").usePathStyle("false"))); + } + + /** + * Glue with an IAM role and no bound storage: the assume-role block replaces the credentials provider, + * and the warehouse placeholder appears because the catalog set none (glue does not use a warehouse but + * the SDK requires the key to be present). + */ + @Test + public void glueAssumeRoleWithoutWarehouseSnapshot() { + assertOptions( + props("iceberg.catalog.type", "glue", + "glue.endpoint", "https://glue.eu-west-1.amazonaws.com", + "glue.role_arn", "arn:aws:iam::1:role/r", + "glue.external_id", "eid", + "catalog-impl", "org.apache.iceberg.aws.glue.GlueCatalog", + "client.factory", "org.apache.iceberg.aws.AssumeRoleAwsClientFactory", + "aws.region", "eu-west-1", + "client.assume-role.arn", "arn:aws:iam::1:role/r", + "client.assume-role.region", "eu-west-1", + "client.assume-role.external-id", "eid", + "client.region", "eu-west-1", + "warehouse", "s3://doris"), + props("iceberg.catalog.type", "glue", + "glue.endpoint", "https://glue.eu-west-1.amazonaws.com", + "glue.role_arn", "arn:aws:iam::1:role/r", + "glue.external_id", "eid"), noS3()); + } + + /** An explicit region alias beats the endpoint-derived one; {@code aws.glue.region} is the last alias. */ + @Test + public void glueExplicitRegionAliasSnapshot() { + assertOptions( + props("iceberg.catalog.type", "glue", + "warehouse", "s3://bucket/wh", + "glue.endpoint", "https://glue.us-east-1.amazonaws.com", + "aws.glue.region", "ap-southeast-1", + "glue.access_key", "GAK", + "glue.secret_key", "GSK", + "catalog-impl", "org.apache.iceberg.aws.glue.GlueCatalog", + "client.credentials-provider", + "org.apache.doris.connector.iceberg.glue.ConfigurationAWSCredentialsProvider2x", + "client.credentials-provider.glue.access_key", "GAK", + "client.credentials-provider.glue.secret_key", "GSK", + "client.region", "ap-southeast-1"), + props("iceberg.catalog.type", "glue", + "warehouse", "s3://bucket/wh", + "glue.endpoint", "https://glue.us-east-1.amazonaws.com", + "aws.glue.region", "ap-southeast-1", + "glue.access_key", "GAK", + "glue.secret_key", "GSK"), noS3()); + } + + /** + * Neither an explicit region alias nor a parseable endpoint host: the region falls back to the legacy + * {@code us-east-1} literal rather than being omitted, which is what keeps the glue client buildable. + */ + @Test + public void glueRegionFallsBackToUsEast1Snapshot() { + assertOptions( + props("iceberg.catalog.type", "glue", + "warehouse", "s3://bucket/wh", + "glue.endpoint", "https://vpce-1234.glue.privatelink.aws", + "glue.role_arn", "arn:aws:iam::1:role/r", + "catalog-impl", "org.apache.iceberg.aws.glue.GlueCatalog", + "client.factory", "org.apache.iceberg.aws.AssumeRoleAwsClientFactory", + "aws.region", "us-east-1", + "client.assume-role.arn", "arn:aws:iam::1:role/r", + "client.assume-role.region", "us-east-1", + "client.region", "us-east-1"), + props("iceberg.catalog.type", "glue", + "warehouse", "s3://bucket/wh", + "glue.endpoint", "https://vpce-1234.glue.privatelink.aws", + "glue.role_arn", "arn:aws:iam::1:role/r"), noS3()); + } + + // --------------------------------------------------------------------- + // JDBC + // --------------------------------------------------------------------- + + /** + * A jdbc catalog that names no uri at all still gets the key, as an EMPTY string — the legacy field + * defaulted to "" and the put is unconditional. Pinned because "absent" and "present but empty" are + * different inputs to the SDK. + */ + @Test + public void jdbcWithoutUriEmitsEmptyUriSnapshot() { + assertOptions( + props("iceberg.catalog.type", "jdbc", + "warehouse", "s3://bucket/wh", + "uri", "", + "catalog-impl", "org.apache.iceberg.jdbc.JdbcCatalog"), + props("iceberg.catalog.type", "jdbc", + "warehouse", "s3://bucket/wh", + "iceberg.jdbc.catalog_name", "mycat"), noS3()); + } + + /** + * Every jdbc knob set: the dotted {@code jdbc.*} options are derived from the {@code iceberg.jdbc.*} + * aliases, a raw {@code jdbc.*} key rides through on the base copy-all, and the positional catalog_name + * is REMOVED from the map (it is passed as the catalog name argument instead). + */ + @Test + public void jdbcFullSnapshot() { + assertOptions( + props("iceberg.catalog.type", "jdbc", + "uri", "jdbc:postgresql://h/db", + "warehouse", "s3://bucket/wh", + "iceberg.jdbc.user", "u", + "iceberg.jdbc.password", "p", + "iceberg.jdbc.init-catalog-tables", "true", + "iceberg.jdbc.schema-version", "V1", + "iceberg.jdbc.strict-mode", "false", + "jdbc.useSSL", "false", + "catalog-impl", "org.apache.iceberg.jdbc.JdbcCatalog", + "jdbc.user", "u", + "jdbc.password", "p", + "jdbc.init-catalog-tables", "true", + "jdbc.schema-version", "V1", + "jdbc.strict-mode", "false"), + props("iceberg.catalog.type", "jdbc", + "uri", "jdbc:postgresql://h/db", + "warehouse", "s3://bucket/wh", + "iceberg.jdbc.catalog_name", "mycat", + "iceberg.jdbc.user", "u", + "iceberg.jdbc.password", "p", + "iceberg.jdbc.init-catalog-tables", "true", + "iceberg.jdbc.schema-version", "V1", + "iceberg.jdbc.strict-mode", "false", + "jdbc.useSSL", "false"), noS3()); + } + + /** + * The jdbc uri alias order is the REVERSE of the rest one: plain {@code uri} wins over + * {@code iceberg.jdbc.uri}. Getting this backwards would silently connect a live catalog to the other + * database, so it is pinned as a whole-map fact. + */ + @Test + public void jdbcUriAliasOrderSnapshot() { + assertOptions( + props("iceberg.catalog.type", "jdbc", + "uri", "jdbc:postgresql://plain/db", + "iceberg.jdbc.uri", "jdbc:postgresql://prefixed/db", + "warehouse", "s3://bucket/wh", + "catalog-impl", "org.apache.iceberg.jdbc.JdbcCatalog"), + props("iceberg.catalog.type", "jdbc", + "uri", "jdbc:postgresql://plain/db", + "iceberg.jdbc.uri", "jdbc:postgresql://prefixed/db", + "warehouse", "s3://bucket/wh", + "iceberg.jdbc.catalog_name", "mycat"), noS3()); + } + + // --------------------------------------------------------------------- + // HMS / HADOOP / S3TABLES + // --------------------------------------------------------------------- + + /** + * HMS emits NO S3FileIO options even with a fully-credentialed store bound: object-store access rides + * the HiveConf, which is a separate sink. A bound store leaking s3.* here would change what the live + * catalog authenticates with. + */ + @Test + public void hmsSnapshot() { + assertOptions( + props("iceberg.catalog.type", "hms", + "hive.metastore.uris", "thrift://h:9083", + "warehouse", "s3://bucket/wh", + "catalog-impl", "org.apache.iceberg.hive.HiveCatalog"), + props("iceberg.catalog.type", "hms", + "hive.metastore.uris", "thrift://h:9083", + "warehouse", "s3://bucket/wh"), s3(new FakeS3CompatibleStorageProperties("S3").endpoint("https://s3") + .accessKey("AK").secretKey("SK"))); + } + + /** + * Hadoop over a NON-generic store (OSS): the s3.* dialect is emitted but the assume-role block is not, + * because that block is reserved for the generic S3 provider. Blank getters emit nothing. + */ + @Test + public void hadoopWithNonGenericStoreSnapshot() { + assertOptions( + props("iceberg.catalog.type", "hadoop", + "warehouse", "oss://bucket/wh", + "catalog-impl", "org.apache.iceberg.hadoop.HadoopCatalog", + "s3.endpoint", "https://oss", + "client.region", "cn-hangzhou", + "s3.access-key-id", "AK", + "s3.secret-access-key", "SK"), + props("iceberg.catalog.type", "hadoop", + "warehouse", "oss://bucket/wh"), s3(new FakeS3CompatibleStorageProperties("OSS").endpoint("https://oss") + .region("cn-hangzhou").accessKey("AK").secretKey("SK"))); + } + + /** + * The manifest cache is derived, not copied: the FE-side {@code meta.cache.iceberg.manifest.*} spec + * turns into the iceberg {@code io.manifest.cache-enabled} option when enabled with a non-zero ttl and + * capacity. Only the derived key is added — the meta.cache keys themselves ride through on copy-all. + */ + @Test + public void manifestCacheDerivationSnapshot() { + assertOptions( + props("iceberg.catalog.type", "hadoop", + "warehouse", "s3://bucket/wh", + "meta.cache.iceberg.manifest.enable", "true", + "meta.cache.iceberg.manifest.ttl-second", "3600", + "meta.cache.iceberg.manifest.capacity", "512", + "catalog-impl", "org.apache.iceberg.hadoop.HadoopCatalog", + "io.manifest.cache-enabled", "true"), + props("iceberg.catalog.type", "hadoop", + "warehouse", "s3://bucket/wh", + "meta.cache.iceberg.manifest.enable", "true", + "meta.cache.iceberg.manifest.ttl-second", "3600", + "meta.cache.iceberg.manifest.capacity", "512"), noS3()); + } + + /** + * s3tables is built through the 3-arg initialize, so it gets NEITHER a catalog-impl NOR the {@code type} + * removal, and static credentials suppress the assume-role block entirely (EXPLICIT wins). + */ + @Test + public void s3TablesWithStaticCredentialsSnapshot() { + Map expected = props("iceberg.catalog.type", "s3tables", + "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b", + "s3.role_arn", "arn:aws:iam::1:role/ignored", + "client.region", "us-east-1", + "s3.access-key-id", "AK", + "s3.secret-access-key", "SK"); + Map actual = IcebergCatalogFactory.buildS3TablesCatalogProperties( + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", + "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b", + "s3.role_arn", "arn:aws:iam::1:role/ignored")), + Optional.of(new FakeS3CompatibleStorageProperties("S3").region("us-east-1") + .accessKey("AK").secretKey("SK").roleArn("arn:aws:iam::1:role/ignored"))); + Assertions.assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); + } + + /** No bound storage: the region still reaches client.region through the raw alias scan. */ + @Test + public void s3TablesWithoutBoundStorageSnapshot() { + Map expected = props("iceberg.catalog.type", "s3tables", + "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b", + "s3.region", "us-east-1", + "client.region", "us-east-1"); + Map actual = IcebergCatalogFactory.buildS3TablesCatalogProperties( + IcebergCatalogProperties.of(props("iceberg.catalog.type", "s3tables", + "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b", + "s3.region", "us-east-1")), + Optional.empty()); + Assertions.assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogPropertiesTest.java new file mode 100644 index 00000000000000..34070468c340dd --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogPropertiesTest.java @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for {@link IcebergCatalogProperties}, the connector-level (flavor-independent) catalog + * properties. + * + *

The three rules that matter are about WHEN things are allowed to fail, not about what the getters + * return. {@code of(Map)} runs on every connector build — including the lazy rebuild after an FE restart — + * so it must bind an existing catalog's properties without judging them; everything that judges belongs to + * {@code checkCreateTimeOnlyRules()}, which only a CREATE/ALTER statement reaches. A rule that drifts from + * the second method into the first does not fail here loudly — it fails in production, months later, as a + * catalog that stops coming back after a restart. + */ +public class IcebergCatalogPropertiesTest { + + private static Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + @Test + public void ofNeverThrowsSoAnExistingCatalogAlwaysRebuilds() { + // WHY: of() is on the rebuild path. Everything a live catalog could carry -- an unknown backend, a + // meta-cache value that fails validation, a key this connector does not model -- must still bind. + // MUTATION: moving any check out of checkCreateTimeOnlyRules() into of() -> red. + Assertions.assertDoesNotThrow(() -> IcebergCatalogProperties.of(props( + "iceberg.catalog.type", "no-such-backend", + "meta.cache.iceberg.table.ttl-second", "not-a-number", + "some.key.this.connector.never.heard.of", "x"))); + Assertions.assertDoesNotThrow(() -> IcebergCatalogProperties.of(Collections.emptyMap())); + } + + @Test + public void ofIsIdempotentAndDoesNotAliasTheCallersMap() { + // WHY: the connector binds once per build and hands the result to four consumers; a second bind of + // the same map must produce the same facts, and a later mutation of the caller's map must not change + // what an already-built connector sees. MUTATION: keeping the map by reference -> red. + Map raw = props("iceberg.catalog.type", "REST", "enable.mapping.varbinary", "true"); + IcebergCatalogProperties first = IcebergCatalogProperties.of(raw); + IcebergCatalogProperties second = IcebergCatalogProperties.of(raw); + Assertions.assertEquals(first.getFlavor(), second.getFlavor()); + Assertions.assertEquals(first.isEnableMappingVarbinary(), second.isEnableMappingVarbinary()); + + raw.put("iceberg.catalog.type", "hms"); + Assertions.assertEquals("rest", first.getFlavor(), "a bound instance must not follow the caller's map"); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> first.getRaw().put("k", "v"), "the exposed raw map must be unmodifiable"); + } + + @Test + public void flavorIsLowerCasedAndNullWhenTheCatalogNamesNone() { + // WHY: the flavor drives the second-level dispatch, and every switch on it compares lower-case. A + // catalog that names NO backend is a distinct state (the build reports "Missing ..."), so it must + // stay null rather than acquire a default. MUTATION: dropping toLowerCase, or defaulting to a + // flavor -> red. + Assertions.assertEquals("rest", IcebergCatalogProperties.of( + props("iceberg.catalog.type", "REST")).getFlavor()); + Assertions.assertEquals("hms", IcebergCatalogProperties.of( + props("iceberg.catalog.type", "Hms")).getFlavor()); + Assertions.assertNull(IcebergCatalogProperties.of(Collections.emptyMap()).getFlavor()); + Assertions.assertNull(IcebergCatalogProperties.of(props("iceberg.catalog.type", "")).getFlavor()); + // Surrounding whitespace is stripped by the binder, so a padded value names its backend rather + // than a nonexistent one. + Assertions.assertEquals("hms", IcebergCatalogProperties.of( + props("iceberg.catalog.type", " hms ")).getFlavor()); + } + + @Test + public void externalCatalogNameIsEmptyUnlessTheCatalogNamesOne() { + // WHY: the value becomes a namespace level applied to EVERY table resolution. An absent (or blank) + // key must yield an empty Optional -- an Optional.of("") would silently prepend an empty level and + // resolve every table in the wrong namespace. MUTATION: defaulting the field to "" -> red. + Assertions.assertFalse(IcebergCatalogProperties.of(Collections.emptyMap()) + .getExternalCatalogName().isPresent()); + Assertions.assertFalse(IcebergCatalogProperties.of(props("external_catalog.name", "")) + .getExternalCatalogName().isPresent()); + Assertions.assertEquals("prod", IcebergCatalogProperties.of(props("external_catalog.name", "prod")) + .getExternalCatalogName().orElse(null)); + } + + @Test + public void typeMappingSwitchesDefaultOffAndReadTheDottedKeys() { + // WHY: the dotted spelling is the only one a live catalog map carries; an underscore variant would + // read false and silently drop the BINARY->VARBINARY / TIMESTAMPTZ mapping for every table. + // MUTATION: binding an underscore key name -> red. + IcebergCatalogProperties off = IcebergCatalogProperties.of(Collections.emptyMap()); + Assertions.assertFalse(off.isEnableMappingVarbinary()); + Assertions.assertFalse(off.isEnableMappingTimestampTz()); + + IcebergCatalogProperties on = IcebergCatalogProperties.of(props( + "enable.mapping.varbinary", "true", "enable.mapping.timestamp_tz", "TRUE")); + Assertions.assertTrue(on.isEnableMappingVarbinary()); + Assertions.assertTrue(on.isEnableMappingTimestampTz(), "parsing must stay case-insensitive"); + } + + @Test + public void createTimeRulesRejectABadMetaCacheValue() { + // WHY: this restores the legacy fail-fast the SPI cutover dropped. Without it a typo'd ttl is + // silently coerced to a cache-disabling default, so the operator's tuning quietly does nothing. + // MUTATION: dropping checkMetaCacheProperties -> the statement is accepted -> red. + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergCatalogProperties.of(props( + "iceberg.catalog.type", "hadoop", "warehouse", "s3://b/wh", + "meta.cache.iceberg.table.ttl-second", "half-an-hour")).checkCreateTimeOnlyRules()); + Assertions.assertTrue(e.getMessage().contains("meta.cache.iceberg.table.ttl-second"), + "the message must name the offending key: " + e.getMessage()); + } + + @Test + public void createTimeRulesRejectAnUnknownBackendAndRunTheBackendsOwnRules() { + // WHY: the CREATE gate is where a catalog that could never work is refused. Two distinct failures: + // no backend answers to the named type, and a named backend's own fail-fast (here: a hadoop catalog + // cannot initialize without a warehouse). MUTATION: not dispatching -> both accepted -> red. + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergCatalogProperties.of(props("iceberg.catalog.type", "no-such-backend")) + .checkCreateTimeOnlyRules()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergCatalogProperties.of(props("iceberg.catalog.type", "hadoop")) + .checkCreateTimeOnlyRules(), + "a hadoop catalog without a warehouse must be refused at CREATE"); + Assertions.assertDoesNotThrow(() -> IcebergCatalogProperties.of(props( + "iceberg.catalog.type", "hadoop", "warehouse", "s3://bucket/wh")).checkCreateTimeOnlyRules()); + } + + @Test + public void toStringMasksSecretsCarriedInTheRawMap() { + // WHY: this object is logged. It declares no sensitive field of its own, but the masking helper is + // what keeps that true if one is ever added. MUTATION: switching to a plain field dump -> red. + String rendered = IcebergCatalogProperties.of(props( + "iceberg.catalog.type", "rest", "enable.mapping.varbinary", "true")).toString(); + Assertions.assertTrue(rendered.contains("rest"), "non-secret values stay readable: " + rendered); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java index 6020e321c9880b..10837dab723b85 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorCacheTest.java @@ -146,14 +146,14 @@ public void latestSnapshotCacheDisabledForSessionUser() { .latestSnapshotCacheForTest(), "a plain catalog builds the latest-snapshot cache"); Map vended = new HashMap<>(); - vended.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - vended.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + vended.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + vended.put("iceberg.rest.vended-credentials-enabled", "true"); Assertions.assertNotNull( new IcebergConnector(vended, new RecordingConnectorContext()).latestSnapshotCacheForTest(), "a vended-credentials catalog still builds the latest-snapshot cache (an id carries no token)"); Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); Assertions.assertNull( new IcebergConnector(session, new RecordingConnectorContext()).latestSnapshotCacheForTest(), "a session=user catalog must NOT build the latest-snapshot cache (per-user authz bypass)"); @@ -165,8 +165,8 @@ public void invalidateHooksAreNoThrowForSessionUserWithNulledCaches() { // still be no-throw (the invalidate* methods null-guard each cache). MUTATION: an unguarded invalidate call // on a nulled cache -> NPE -> red. Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); IcebergConnector connector = new IcebergConnector(session, new RecordingConnectorContext()); Assertions.assertNull(connector.latestSnapshotCacheForTest()); Assertions.assertNull(connector.partitionCacheForTest()); @@ -241,8 +241,8 @@ public void crossQueryTableCacheDisabledForVendedCredentials() { // would hand BE an expired token (403 mid-scan), so this layer MUST be off (null); the query-scoped fat // handle still dedups within one query. MUTATION: building the cache for a vended catalog -> non-null -> red. Map vended = new HashMap<>(); - vended.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - vended.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + vended.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + vended.put("iceberg.rest.vended-credentials-enabled", "true"); Assertions.assertNull( new IcebergConnector(vended, new RecordingConnectorContext()).tableCacheForTest(), "a REST vended-credentials catalog must NOT build the cross-query table cache"); @@ -254,8 +254,8 @@ public void crossQueryTableCacheDisabledForPerUserSession() { // across users would leak credentials. This layer MUST be off (null) — the fat handle keeps within-query // dedup. MUTATION: building the cache for a session=user catalog -> tableCacheForTest non-null -> red. Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); Assertions.assertNull( new IcebergConnector(session, new RecordingConnectorContext()).tableCacheForTest(), "a per-user session catalog must NOT build the cross-query table cache"); @@ -306,14 +306,14 @@ public void partitionCacheBuiltUnlessSessionUser() { new IcebergConnector(Collections.emptyMap(), new RecordingConnectorContext()).partitionCacheForTest(), "a plain catalog builds the partition cache"); Map vended = new HashMap<>(); - vended.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - vended.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + vended.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + vended.put("iceberg.rest.vended-credentials-enabled", "true"); Assertions.assertNotNull( new IcebergConnector(vended, new RecordingConnectorContext()).partitionCacheForTest(), "a vended-credentials catalog still builds the partition cache (metadata carries no credentials)"); Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); Assertions.assertNull( new IcebergConnector(session, new RecordingConnectorContext()).partitionCacheForTest(), "a session=user catalog must NOT build the partition cache (per-user authz must not be bypassed)"); @@ -360,14 +360,14 @@ public void formatCacheBuiltUnlessSessionUser() { new IcebergConnector(Collections.emptyMap(), new RecordingConnectorContext()).formatCacheForTest(), "a plain catalog builds the format cache"); Map vended = new HashMap<>(); - vended.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - vended.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + vended.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + vended.put("iceberg.rest.vended-credentials-enabled", "true"); Assertions.assertNotNull( new IcebergConnector(vended, new RecordingConnectorContext()).formatCacheForTest(), "a vended-credentials catalog still builds the format cache (a format name carries no credentials)"); Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); Assertions.assertNull( new IcebergConnector(session, new RecordingConnectorContext()).formatCacheForTest(), "a session=user catalog must NOT build the format cache (per-user authz must not be bypassed)"); @@ -401,12 +401,12 @@ public void refreshHooksInvalidateFormatCache() { private static Map restProps(boolean vended, boolean sessionUser) { Map m = new HashMap<>(); - m.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); + m.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); if (vended) { - m.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + m.put("iceberg.rest.vended-credentials-enabled", "true"); } if (sessionUser) { - m.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + m.put("iceberg.rest.session", "user"); } return m; } @@ -473,16 +473,16 @@ public void partitionViewCacheBuiltUnlessSessionUser() { Assertions.assertNotNull(plain.listPartitionsViewCacheForTest(), "a plain catalog builds the listPartitions view cache"); Map vended = new HashMap<>(); - vended.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - vended.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + vended.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + vended.put("iceberg.rest.vended-credentials-enabled", "true"); IcebergConnector vendedConn = new IcebergConnector(vended, new RecordingConnectorContext()); Assertions.assertNotNull(vendedConn.mvccPartitionViewCacheForTest(), "a vended-credentials catalog still builds the MVCC view cache (metadata carries no credentials)"); Assertions.assertNotNull(vendedConn.listPartitionsViewCacheForTest(), "a vended-credentials catalog still builds the listPartitions view cache"); Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); IcebergConnector sessionConn = new IcebergConnector(session, new RecordingConnectorContext()); Assertions.assertNull(sessionConn.mvccPartitionViewCacheForTest(), "a session=user catalog must NOT build the MVCC view cache (per-user authz must not be bypassed)"); @@ -541,8 +541,8 @@ public void invalidateHooksNoThrowForSessionUserPartitionViewCaches() { // Under session=user cache A's two instances are null; the invalidate hooks must null-guard them (no NPE). // MUTATION: an unguarded view-cache invalidate on a nulled field -> NPE -> red. Map session = new HashMap<>(); - session.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - session.put(IcebergConnectorProperties.REST_SESSION, IcebergConnectorProperties.SESSION_USER); + session.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + session.put("iceberg.rest.session", "user"); IcebergConnector connector = new IcebergConnector(session, new RecordingConnectorContext()); Assertions.assertNull(connector.mvccPartitionViewCacheForTest()); Assertions.assertNull(connector.listPartitionsViewCacheForTest()); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorConfTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorConfTest.java index 4fd6e69b213e76..f166b94eaa0b82 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorConfTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorConfTest.java @@ -17,7 +17,6 @@ package org.apache.doris.connector.iceberg; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.junit.jupiter.api.Assertions; @@ -40,51 +39,37 @@ public class IcebergConnectorConfTest { @Test public void driversDirPrefersThePluginConfThenFeConf() { - Assertions.assertEquals("/from/plugin/conf", ConnectorConf.get( - context(Collections.singletonMap(IcebergConnectorProperties.CONF_DRIVERS_DIR, - "/from/plugin/conf"), - Collections.singletonMap(IcebergConnectorProperties.ENV_JDBC_DRIVERS_DIR, - "/from/fe/conf")), - IcebergConnectorProperties.CONF_DRIVERS_DIR, - IcebergConnectorProperties.ENV_JDBC_DRIVERS_DIR, null)); - - Assertions.assertEquals("/from/fe/conf", ConnectorConf.get( + Assertions.assertEquals("/from/plugin/conf", IcebergConf.driversDir( + context(Collections.singletonMap(IcebergConf.CONF_DRIVERS_DIR, "/from/plugin/conf"), + Collections.singletonMap(IcebergConf.ENV_JDBC_DRIVERS_DIR, "/from/fe/conf")))); + + Assertions.assertEquals("/from/fe/conf", IcebergConf.driversDir( context(Collections.emptyMap(), - Collections.singletonMap(IcebergConnectorProperties.ENV_JDBC_DRIVERS_DIR, - "/from/fe/conf")), - IcebergConnectorProperties.CONF_DRIVERS_DIR, - IcebergConnectorProperties.ENV_JDBC_DRIVERS_DIR, null)); - - Assertions.assertNull(ConnectorConf.get(context(Collections.emptyMap(), Collections.emptyMap()), - IcebergConnectorProperties.CONF_DRIVERS_DIR, - IcebergConnectorProperties.ENV_JDBC_DRIVERS_DIR, null)); + Collections.singletonMap(IcebergConf.ENV_JDBC_DRIVERS_DIR, "/from/fe/conf")))); + + Assertions.assertNull(IcebergConf.driversDir( + context(Collections.emptyMap(), Collections.emptyMap()))); + + // A null context is a direct-construction unit test: neither channel exists, and the driver + // support falls back to /plugins/jdbc_drivers as it always has. + Assertions.assertNull(IcebergConf.driversDir(null)); } @Test public void metastoreTimeoutPrefersThePluginConfThenFeConfThenTen() { - Assertions.assertEquals("30", ConnectorConf.get( - context(Collections.singletonMap( - IcebergConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, "30"), + Assertions.assertEquals("30", IcebergConf.metastoreClientTimeoutSecond( + context(Collections.singletonMap(IcebergConf.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, "30"), Collections.singletonMap( - IcebergConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")), - IcebergConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND)); + IcebergConf.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")))); - Assertions.assertEquals("20", ConnectorConf.get( + Assertions.assertEquals("20", IcebergConf.metastoreClientTimeoutSecond( context(Collections.emptyMap(), Collections.singletonMap( - IcebergConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")), - IcebergConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND)); + IcebergConf.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")))); // The literal the call site used before this channel existed; keeping it is what makes a // deployment with neither file behave exactly as it did. - Assertions.assertEquals("10", ConnectorConf.get( - context(Collections.emptyMap(), Collections.emptyMap()), - IcebergConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - IcebergConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND)); + Assertions.assertEquals("10", IcebergConf.metastoreClientTimeoutSecond( + context(Collections.emptyMap(), Collections.emptyMap()))); } @Test @@ -96,6 +81,17 @@ public void theConfTemplateIsNamedAfterTheProvider() { "the plugin must ship " + expected + " on its classpath"); } + @Test + public void dorisHomeComesFromTheEngineEnvironmentOnly() { + // WHY: the FE install root is engine-wide, not this connector's setting, so it has no iceberg.conf + // key -- only the environment. MUTATION: adding a conf key for it -> a deployment could set a + // per-plugin doris_home that disagrees with the engine's. + Assertions.assertEquals("/opt/doris", IcebergConf.dorisHome( + context(Collections.singletonMap(IcebergConf.ENV_DORIS_HOME, "/from/plugin/conf"), + Collections.singletonMap(IcebergConf.ENV_DORIS_HOME, "/opt/doris")))); + Assertions.assertNull(IcebergConf.dorisHome(null)); + } + private static ConnectorContext context(Map conf, Map env) { Map confCopy = new HashMap<>(conf); Map envCopy = new HashMap<>(env); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorDeriveStoragePropertiesTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorDeriveStoragePropertiesTest.java index 8679312a5652cb..568bb1ffb6e8fb 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorDeriveStoragePropertiesTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorDeriveStoragePropertiesTest.java @@ -37,10 +37,10 @@ public class IcebergConnectorDeriveStoragePropertiesTest { private static Map props(String catalogType, String warehouse) { Map m = new HashMap<>(); if (catalogType != null) { - m.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, catalogType); + m.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, catalogType); } if (warehouse != null) { - m.put(IcebergConnectorProperties.WAREHOUSE, warehouse); + m.put("warehouse", warehouse); } return m; } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataColumnEvolutionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataColumnEvolutionTest.java index 452be206fb63f6..da1a8e3557b214 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataColumnEvolutionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataColumnEvolutionTest.java @@ -50,12 +50,12 @@ public class IcebergConnectorMetadataColumnEvolutionTest { private static Map props() { Map p = new HashMap<>(); - p.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); + p.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); return p; } private static IcebergConnectorMetadata metadata(RecordingIcebergCatalogOps ops, RecordingConnectorContext ctx) { - return new IcebergConnectorMetadata(ops, props(), ctx); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(props()), ctx); } private static ConnectorColumn col(String name, String type) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java index 9fae1d210a9022..7220358f6bc9de 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataDdlTest.java @@ -51,14 +51,14 @@ public class IcebergConnectorMetadataDdlTest { private static Map props(String catalogType) { Map p = new HashMap<>(); if (catalogType != null) { - p.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, catalogType); + p.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, catalogType); } return p; } private static IcebergConnectorMetadata metadata(RecordingIcebergCatalogOps ops, RecordingConnectorContext ctx, String catalogType) { - return new IcebergConnectorMetadata(ops, props(catalogType), ctx); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(props(catalogType)), ctx); } // ---------- createDatabase ---------- @@ -68,7 +68,7 @@ public void testCreateDatabaseHmsWithPropertiesIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); Map dbProps = Collections.singletonMap("location", "s3://wh/db"); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_HMS).createDatabase(null, "db1", dbProps); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_HMS).createDatabase(null, "db1", dbProps); Assertions.assertEquals("db1", ops.lastCreateDb); Assertions.assertEquals(dbProps, ops.lastCreateDbProps); Assertions.assertEquals(1, ctx.authCount, "createDatabase must run inside executeAuthenticated"); @@ -78,7 +78,7 @@ public void testCreateDatabaseHmsWithPropertiesIsAuthWrapped() { public void testCreateDatabaseNonHmsWithPropertiesFailsLoud() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); - IcebergConnectorMetadata md = metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST); + IcebergConnectorMetadata md = metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST); DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, () -> md.createDatabase(null, "db1", Collections.singletonMap("k", "v"))); Assertions.assertTrue(ex.getMessage().contains("rest")); @@ -91,7 +91,7 @@ public void testCreateDatabaseNonHmsWithPropertiesFailsLoud() { public void testCreateDatabaseNonHmsEmptyPropertiesSucceeds() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .createDatabase(null, "db1", Collections.emptyMap()); Assertions.assertEquals("db1", ops.lastCreateDb); Assertions.assertEquals(1, ctx.authCount); @@ -102,7 +102,7 @@ public void testCreateDatabaseAuthFailureWraps() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; - IcebergConnectorMetadata md = metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST); + IcebergConnectorMetadata md = metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST); Assertions.assertThrows(DorisConnectorException.class, () -> md.createDatabase(null, "db1", Collections.emptyMap())); // failAuth throws WITHOUT running the task -> the seam create must not have run. @@ -117,7 +117,7 @@ public void testDropDatabaseForceCascadesAndCleansHms() { ops.tables = Arrays.asList("t1", "t2"); ops.namespaceLocation = Optional.of("s3://wh/db1"); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_HMS).dropDatabase(null, "db1", false, true); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_HMS).dropDatabase(null, "db1", false, true); // location captured BEFORE drop, then the tables cascade-dropped, then the (empty) view list probed, // then the namespace dropped. Assertions.assertEquals(Arrays.asList( @@ -138,7 +138,7 @@ public void testDropDatabaseForceCascadesViewsAfterTables() { ops.tables = Collections.singletonList("t1"); ops.views = Arrays.asList("v1", "v2"); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).dropDatabase(null, "db1", false, true); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).dropDatabase(null, "db1", false, true); // WHY: iceberg VIEWS live in their own namespace (listTableNames subtracts them), so a force drop // must cascade them too — AFTER the tables and BEFORE dropNamespace — or the dropDatabase below would // fail loud "namespace not empty". MUTATION: dropping the view cascade -> the dropView entries vanish @@ -157,7 +157,7 @@ public void testDropDatabaseNonForceNonHmsNoCascadeNoCleanup() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.tables = Arrays.asList("t1", "t2"); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).dropDatabase(null, "db1", false, false); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).dropDatabase(null, "db1", false, false); // No location load (non-HMS), no cascade (non-force), just the namespace drop. Assertions.assertEquals(Collections.singletonList("dropDatabase:db1"), ops.log); Assertions.assertTrue(ctx.cleanedLocations.isEmpty()); @@ -175,7 +175,7 @@ public void testDropDatabaseForceToleratesAlreadyDeletedNamespaceNonHms() { // and lost that tolerance. This asserts FORCE no longer throws. MUTATION: removing the // catch(NoSuchNamespaceException) re-surfaces it as DorisConnectorException -> red. Assertions.assertDoesNotThrow(() -> - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).dropDatabase(null, "db1", false, true)); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).dropDatabase(null, "db1", false, true)); // The missing namespace surfaces at the first cascade probe (listTableNames); the namespace drop is skipped. Assertions.assertTrue(ops.log.contains("listTableNames:db1"), ops.log.toString()); Assertions.assertFalse(ops.log.contains("dropDatabase:db1"), ops.log.toString()); @@ -192,7 +192,7 @@ public void testDropDatabaseForceToleratesAlreadyDeletedNamespaceHms() { // flavor), or an HMS FORCE-drop of an already-gone namespace would still fail. MUTATION: scoping the // catch to only the cascade (excluding loadNamespaceLocation) makes this red. Assertions.assertDoesNotThrow(() -> - metadata(ops, ctx, IcebergConnectorProperties.TYPE_HMS).dropDatabase(null, "db1", false, true)); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_HMS).dropDatabase(null, "db1", false, true)); Assertions.assertTrue(ops.log.contains("loadNamespaceLocation:db1"), ops.log.toString()); Assertions.assertFalse(ops.log.contains("dropDatabase:db1"), ops.log.toString()); // Tolerated drop returns no location -> the managed-location cleanup hook must not run. @@ -209,7 +209,7 @@ public void testDropDatabaseNonForceDoesNotTolerateMissingNamespace() { // still fail loud. MUTATION: dropping the `if (!force) throw e;` guard (always tolerate) makes this // assertThrows red. Assertions.assertThrows(DorisConnectorException.class, () -> - metadata(ops, ctx, IcebergConnectorProperties.TYPE_HMS).dropDatabase(null, "db1", false, false)); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_HMS).dropDatabase(null, "db1", false, false)); } // ---------- createTable ---------- @@ -228,7 +228,7 @@ public void testCreateTableBuildsArtifactsAndCallsSeam() { new ConnectorPartitionField("id", "bucket", Collections.singletonList(8))))) .sortOrder(Collections.singletonList(new ConnectorSortField("id", true, true))) .build(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).createTable(null, request); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).createTable(null, request); Assertions.assertEquals("db1", ops.lastCreateTableDb); Assertions.assertEquals("t1", ops.lastCreateTableName); @@ -253,7 +253,7 @@ public void testCreateTableUnsupportedTypeFailsBeforeRemote() { .columns(Collections.singletonList( new ConnectorColumn("t", ConnectorType.of("TINYINT"), "", true, null, false))) .build(); - IcebergConnectorMetadata md = metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST); + IcebergConnectorMetadata md = metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST); Assertions.assertThrows(DorisConnectorException.class, () -> md.createTable(null, request)); // Schema build is pure + runs before the auth/remote create -> the seam never ran. Assertions.assertTrue(ops.log.isEmpty()); @@ -277,7 +277,7 @@ public void testCreateTableRejectsReservedRowLineageColumnAtV3() { .properties(Collections.singletonMap(TableProperties.FORMAT_VERSION, "3")) .build(); DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).createTable(null, request), + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).createTable(null, request), reserved + " must be rejected on a v3 table"); Assertions.assertTrue(ex.getMessage().contains("reserved row lineage column"), ex.getMessage()); Assertions.assertTrue(ops.log.isEmpty(), "reject must run before the remote seam"); @@ -296,7 +296,7 @@ public void testCreateTableAllowsReservedRowLineageNameBelowV3() { .columns(Collections.singletonList( new ConnectorColumn("_row_id", ConnectorType.of("BIGINT"), "", true, null, false))) .build(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).createTable(null, request); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).createTable(null, request); Assertions.assertEquals("t1", ops.lastCreateTableName); Assertions.assertNotNull(ops.lastCreateSchema.findField("_row_id")); } @@ -318,9 +318,9 @@ private static void assertCatalogLevelV3Rejects(String catalogFormatVersionKey) RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); Map catalogProps = new HashMap<>(); - catalogProps.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); + catalogProps.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); catalogProps.put(catalogFormatVersionKey, "3"); - IcebergConnectorMetadata md = new IcebergConnectorMetadata(ops, catalogProps, ctx); + IcebergConnectorMetadata md = new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(catalogProps), ctx); ConnectorCreateTableRequest request = ConnectorCreateTableRequest.builder() .dbName("db1").tableName("t1") .columns(Collections.singletonList( @@ -341,7 +341,7 @@ public void testDropTableHmsCapturesLocationAndCleans() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.tableLocation = Optional.of("s3://wh/db1/t1"); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_HMS) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_HMS) .dropTable(null, new IcebergTableHandle("db1", "t1")); // location captured BEFORE the purge-drop. Assertions.assertEquals(Arrays.asList( @@ -356,7 +356,7 @@ public void testDropTableHmsCapturesLocationAndCleans() { public void testDropTableNonHmsNoLocationNoCleanup() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .dropTable(null, new IcebergTableHandle("db1", "t1")); Assertions.assertEquals(Collections.singletonList("dropTable:db1.t1:purge=true"), ops.log); Assertions.assertTrue(ctx.cleanedLocations.isEmpty()); @@ -368,7 +368,7 @@ public void testDropTableNonHmsNoLocationNoCleanup() { public void testDropViewRoutesToSeamAndIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).dropView(null, "db1", "v1"); + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).dropView(null, "db1", "v1"); // WHY: PluginDrivenExternalCatalog.dropTable routes a flipped iceberg view here; it must reach the seam // with the (db, view) names verbatim, INSIDE the auth context (mirrors legacy performDropView under the // executionAuthenticator). MUTATION: dropping the delegation / hoisting it outside the auth wrap -> red. @@ -386,7 +386,7 @@ public void testDropViewAuthFailureWraps() { // WHY: like the other write ops, a remote/auth failure must surface as a DorisConnectorException so // PluginDrivenExternalCatalog.dropTable can rewrap it as a DdlException; the seam must NOT be reached. DorisConnectorException ex = Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).dropView(null, "db1", "v1")); + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).dropView(null, "db1", "v1")); Assertions.assertTrue(ex.getMessage().contains("Failed to drop Iceberg view"), ex.getMessage()); Assertions.assertFalse(ops.log.contains("dropView:db1.v1"), "the seam must not be reached when the auth wrap throws"); @@ -398,7 +398,7 @@ public void testDropViewAuthFailureWraps() { public void testRenameTableRoutesByHandleAndIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .renameTable(null, new IcebergTableHandle("db1", "t1"), "t2"); Assertions.assertEquals(Collections.singletonList("renameTable:db1.t1->t2"), ops.log); Assertions.assertEquals("db1", ops.lastRenameTableDb); @@ -413,7 +413,7 @@ public void testRenameTableAuthFailureWraps() { RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .renameTable(null, new IcebergTableHandle("db1", "t1"), "t2")); Assertions.assertTrue(ops.log.isEmpty()); } @@ -426,7 +426,7 @@ public void testCreateOrReplaceBranchRoutesByHandleAndIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); BranchChange branch = new BranchChange("b1", true, false, false, 7L, null, null, null); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .createOrReplaceBranch(null, new IcebergTableHandle("db1", "t1"), branch); Assertions.assertEquals(Collections.singletonList("createOrReplaceBranch:db1.t1:b1"), ops.log); Assertions.assertEquals("db1", ops.lastBranchTagDb); @@ -441,7 +441,7 @@ public void testCreateOrReplaceBranchAuthFailureWraps() { RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).createOrReplaceBranch( + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).createOrReplaceBranch( null, new IcebergTableHandle("db1", "t1"), new BranchChange("b1", true, false, false, null, null, null, null))); Assertions.assertTrue(ops.log.isEmpty()); @@ -452,7 +452,7 @@ public void testCreateOrReplaceTagRoutesByHandleAndIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); TagChange tag = new TagChange("v1", true, false, false, 7L, null); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .createOrReplaceTag(null, new IcebergTableHandle("db1", "t1"), tag); Assertions.assertEquals(Collections.singletonList("createOrReplaceTag:db1.t1:v1"), ops.log); Assertions.assertSame(tag, ops.lastTag); @@ -465,7 +465,7 @@ public void testCreateOrReplaceTagAuthFailureWraps() { RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).createOrReplaceTag( + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).createOrReplaceTag( null, new IcebergTableHandle("db1", "t1"), new TagChange("v1", true, false, false, null, null))); Assertions.assertTrue(ops.log.isEmpty()); @@ -476,7 +476,7 @@ public void testDropBranchRoutesByHandleAndIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); DropRefChange drop = new DropRefChange("b1", true); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .dropBranch(null, new IcebergTableHandle("db1", "t1"), drop); Assertions.assertEquals(Collections.singletonList("dropBranch:db1.t1:b1"), ops.log); Assertions.assertSame(drop, ops.lastDropBranch); @@ -488,7 +488,7 @@ public void testDropTagRoutesByHandleAndIsAuthWrapped() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); RecordingConnectorContext ctx = new RecordingConnectorContext(); DropRefChange drop = new DropRefChange("v1", false); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .dropTag(null, new IcebergTableHandle("db1", "t1"), drop); Assertions.assertEquals(Collections.singletonList("dropTag:db1.t1:v1"), ops.log); Assertions.assertSame(drop, ops.lastDropTag); @@ -501,7 +501,7 @@ public void testDropTagAuthFailureWraps() { RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).dropTag( + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).dropTag( null, new IcebergTableHandle("db1", "t1"), new DropRefChange("v1", false))); Assertions.assertTrue(ops.log.isEmpty()); } @@ -514,7 +514,7 @@ public void testAddPartitionFieldRoutesByHandleAndIsAuthWrapped() { RecordingConnectorContext ctx = new RecordingConnectorContext(); PartitionFieldChange change = new PartitionFieldChange("bucket", 8, "id", "id_b", null, null, null, null); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .addPartitionField(null, new IcebergTableHandle("db1", "t1"), change); Assertions.assertEquals(Collections.singletonList("addPartitionField:db1.t1:id"), ops.log); Assertions.assertEquals("db1", ops.lastPartitionFieldDb); @@ -529,7 +529,7 @@ public void testAddPartitionFieldAuthFailureWraps() { RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).addPartitionField( + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).addPartitionField( null, new IcebergTableHandle("db1", "t1"), new PartitionFieldChange(null, null, "id", null, null, null, null, null))); Assertions.assertTrue(ops.log.isEmpty()); @@ -541,7 +541,7 @@ public void testDropPartitionFieldRoutesByHandleAndIsAuthWrapped() { RecordingConnectorContext ctx = new RecordingConnectorContext(); PartitionFieldChange change = new PartitionFieldChange(null, null, null, "p_id", null, null, null, null); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .dropPartitionField(null, new IcebergTableHandle("db1", "t1"), change); Assertions.assertEquals(Collections.singletonList("dropPartitionField:db1.t1:p_id"), ops.log); Assertions.assertSame(change, ops.lastDropPartitionField); @@ -554,7 +554,7 @@ public void testReplacePartitionFieldRoutesByHandleAndIsAuthWrapped() { RecordingConnectorContext ctx = new RecordingConnectorContext(); PartitionFieldChange change = new PartitionFieldChange("bucket", 4, "id", "p2", "p", null, null, null); - metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST) + metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST) .replacePartitionField(null, new IcebergTableHandle("db1", "t1"), change); Assertions.assertEquals(Collections.singletonList("replacePartitionField:db1.t1:id"), ops.log); Assertions.assertSame(change, ops.lastReplacePartitionField); @@ -567,7 +567,7 @@ public void testReplacePartitionFieldAuthFailureWraps() { RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; Assertions.assertThrows(DorisConnectorException.class, - () -> metadata(ops, ctx, IcebergConnectorProperties.TYPE_REST).replacePartitionField( + () -> metadata(ops, ctx, IcebergCatalogProperties.TYPE_REST).replacePartitionField( null, new IcebergTableHandle("db1", "t1"), new PartitionFieldChange(null, null, "id", null, "p", null, null, null))); Assertions.assertTrue(ops.log.isEmpty()); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java index 3aa8c9ea82564b..be8514b11faacb 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataMvccTest.java @@ -104,7 +104,7 @@ private static DataFile dataFile(String path) { private static IcebergConnectorMetadata metadataFor(Table table, RecordingIcebergCatalogOps ops) { ops.table = table; - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } private static ConnectorTableHandle handle() { @@ -156,7 +156,7 @@ public void beginQuerySnapshotEnabledCachePinsStableAndLoadsOnce() { // An ENABLED cache (TTL 100s) injected via the 4-arg ctor — the production wiring (IcebergConnector // injects its per-catalog cache here). T08. IcebergConnectorMetadata md = new IcebergConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext(), + ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext(), new IcebergLatestSnapshotCache(100, 1000)); Optional first = md.beginQuerySnapshot(null, handle()); Optional second = md.beginQuerySnapshot(null, handle()); @@ -489,7 +489,7 @@ public void getMvccPartitionViewFailsLoudWhenTableMissing() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.throwNoSuchTableOnLoadTable = true; IcebergConnectorMetadata md = - new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); Assertions.assertThrows(RuntimeException.class, () -> md.getMvccPartitionView(null, handle())); } @@ -501,7 +501,7 @@ public void getMvccPartitionViewRunsInsideAuthContext() { ops.table = dayPartitionedTable(); RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; - IcebergConnectorMetadata md = new IcebergConnectorMetadata(ops, Collections.emptyMap(), ctx); + IcebergConnectorMetadata md = new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), ctx); Assertions.assertThrows(RuntimeException.class, () -> md.getMvccPartitionView(null, handle())); Assertions.assertEquals(1, ctx.authCount); Assertions.assertFalse(ops.log.contains("loadTable:db1.t1"), "loadTable must sit inside executeAuthenticated"); @@ -520,7 +520,7 @@ public void listPartitionNamesDegradesToEmptyWhenTableMissing() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.throwNoSuchTableOnLoadTable = true; List names = - new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()) + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()) .listPartitionNames(null, handle()); Assertions.assertTrue(names.isEmpty()); } @@ -531,7 +531,7 @@ public void listPartitionNamesRunsInsideAuthContext() { ops.table = dayPartitionedTable(); RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; - IcebergConnectorMetadata md = new IcebergConnectorMetadata(ops, Collections.emptyMap(), ctx); + IcebergConnectorMetadata md = new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), ctx); Assertions.assertThrows(RuntimeException.class, () -> md.listPartitionNames(null, handle())); Assertions.assertEquals(1, ctx.authCount); Assertions.assertFalse(ops.log.contains("loadTable:db1.t1"), "loadTable must sit inside executeAuthenticated"); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java index df241b1181952c..8bec2551e770b8 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java @@ -94,13 +94,13 @@ private static IcebergConnectorMetadata metadataWithMvccCache(RecordingIcebergCa ConnectorMetadataCache cache) { // 9-arg ctor: disabled latest-snapshot cache + null table/partition/comment caches so the loadTable count // reflects cache A alone; only the mvcc view cache under test is injected. - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext(), + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext(), new IcebergLatestSnapshotCache(0L, 1), null, null, null, cache, null); } private static IcebergConnectorMetadata metadataWithListCache(RecordingIcebergCatalogOps ops, ConnectorMetadataCache> cache) { - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext(), + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext(), new IcebergLatestSnapshotCache(0L, 1), null, null, null, null, cache); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataStatisticsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataStatisticsTest.java index af50681e834571..bfea7820434c26 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataStatisticsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataStatisticsTest.java @@ -91,7 +91,7 @@ private static DeleteFile equalityDeletes(String path, long records) { private static IcebergConnectorMetadata metadataFor(Table table, RecordingIcebergCatalogOps ops) { ops.table = table; - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } private static ConnectorTableHandle handle() { @@ -184,7 +184,7 @@ public void loadFailureDegradesToUnknown() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.throwOnLoadTable = true; IcebergConnectorMetadata metadata = - new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); // WHY: a statistics miss must NEVER break query planning. MUTATION: letting the exception propagate // makes assertDoesNotThrow fail. diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataSysTableTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataSysTableTest.java index d2680142122d0a..b62b69462480f1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataSysTableTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataSysTableTest.java @@ -72,12 +72,12 @@ public class IcebergConnectorMetadataSysTableTest { private static IcebergConnectorMetadata metadataWith(RecordingIcebergCatalogOps ops) { - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } private static IcebergConnectorMetadata metadataWith( RecordingIcebergCatalogOps ops, RecordingConnectorContext ctx) { - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), ctx); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), ctx); } private static IcebergTableHandle baseHandle() { @@ -384,10 +384,10 @@ public void getTableSchemaForSysHandleThreadsMappingFlags() { IcebergConnectorMetadata mdDefault = metadataWith(seamWith(inMemoryBaseTable())); Map tzProps = new HashMap<>(); - tzProps.put(IcebergConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true"); + tzProps.put(IcebergCatalogProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true"); RecordingIcebergCatalogOps opsTz = seamWith(inMemoryBaseTable()); IcebergConnectorMetadata mdTz = - new IcebergConnectorMetadata(opsTz, tzProps, new RecordingConnectorContext()); + new IcebergConnectorMetadata(opsTz, IcebergCatalogProperties.of(tzProps), new RecordingConnectorContext()); ConnectorColumn committedDefault = committedAtColumn(mdDefault); ConnectorColumn committedTz = committedAtColumn(mdTz); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java index 83a2303d95ac06..5e98e90fe0a2cb 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java @@ -70,12 +70,12 @@ private static IcebergConnectorMetadata metadataWith(RecordingIcebergCatalogOps private static IcebergConnectorMetadata metadataWith( RecordingIcebergCatalogOps ops, Map props) { - return new IcebergConnectorMetadata(ops, props, new RecordingConnectorContext()); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(props), new RecordingConnectorContext()); } private static IcebergConnectorMetadata metadataWith( RecordingIcebergCatalogOps ops, RecordingConnectorContext ctx) { - return new IcebergConnectorMetadata(ops, Collections.emptyMap(), ctx); + return new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), ctx); } /** A simple 2-column unpartitioned schema (id required, name optional). */ @@ -168,7 +168,7 @@ public void getTableCommentIsMemoizedAcrossQueriesWhenCommentCachePresent() { ops.table = new FakeIcebergTable( "t1", idNameSchema(), PartitionSpec.unpartitioned(), "s3://bucket/db1/t1", props); IcebergCommentCache cache = new IcebergCommentCache(100, 1000); - IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, Collections.emptyMap(), + IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext(), new IcebergLatestSnapshotCache(0L, 1), null, null, cache); Assertions.assertEquals("sales fact", metadata.getTableComment(null, "db1", "t1")); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTestConnectionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTestConnectionTest.java index b9c26cba61fa12..dc7a83d9c3e3aa 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTestConnectionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTestConnectionTest.java @@ -103,18 +103,18 @@ public void toS3LocationNormalizesAndFilters() { */ @Test public void probesMetastoreOnlyForRemoteMetastoreTypes() { - Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergConnectorProperties.TYPE_REST)); - Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergConnectorProperties.TYPE_HMS)); - Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergConnectorProperties.TYPE_GLUE)); - Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergConnectorProperties.TYPE_S3_TABLES)); - Assertions.assertFalse(IcebergConnector.probesMetastore(IcebergConnectorProperties.TYPE_HADOOP)); + Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergCatalogProperties.TYPE_REST)); + Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergCatalogProperties.TYPE_HMS)); + Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergCatalogProperties.TYPE_GLUE)); + Assertions.assertTrue(IcebergConnector.probesMetastore(IcebergCatalogProperties.TYPE_S3_TABLES)); + Assertions.assertFalse(IcebergConnector.probesMetastore(IcebergCatalogProperties.TYPE_HADOOP)); Assertions.assertFalse(IcebergConnector.probesMetastore("")); } /** An HMS-backed catalog must name HMS in the failure, which is what the CREATE CATALOG regression asserts. */ @Test public void metaFailureMessageTagsTheCatalogType() { - String msg = IcebergConnector.metaFailureMessage(IcebergConnectorProperties.TYPE_HMS, + String msg = IcebergConnector.metaFailureMessage(IcebergCatalogProperties.TYPE_HMS, new RuntimeException("connection refused")); Assertions.assertTrue(msg.contains("Iceberg HMS"), msg); Assertions.assertTrue(msg.contains("connectivity test failed"), msg); @@ -222,8 +222,8 @@ public void testConnectionSucceedsWhenNothingToProbe() { // probesMetastore) and the storage probe is skipped (no s3.* creds), so testConnection // succeeds without any I/O. Map props = new HashMap<>(); - props.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, - IcebergConnectorProperties.TYPE_HADOOP); + props.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, + IcebergCatalogProperties.TYPE_HADOOP); try (IcebergConnector connector = new IcebergConnector(props, CTX)) { ConnectorTestResult result = connector.testConnection(null); Assertions.assertTrue(result.isSuccess(), result.getMessage()); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java index 77c45d0c833f55..cdc1a341f40cc1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTransactionTest.java @@ -825,7 +825,7 @@ public void mergeFromResolvedEmptySnapshotRejectsConcurrentFirstAppend() throws props("format-version", "2", "write.format.default", "parquet")); RecordingIcebergCatalogOps ops = opsReturning(empty); RecordingConnectorContext context = new RecordingConnectorContext(); - IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, Collections.emptyMap(), context); + IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), context); CountDownLatch mergeReadResolved = new CountDownLatch(1); CountDownLatch concurrentInsertCommitted = new CountDownLatch(1); @@ -857,7 +857,7 @@ public void mergeFromResolvedEmptySnapshotRejectsConcurrentFirstAppend() throws Assertions.assertNull(insertFailure.get(), "the concurrent INSERT must commit at the barrier"); ops.table = catalog.loadTable(id); - IcebergScanPlanProvider scanProvider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + IcebergScanPlanProvider scanProvider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops); Assertions.assertTrue(scanProvider.planScan(null, ConnectorScanRequest.builder(emptyRead, Collections.emptyList()).build()).isEmpty(), "MERGE must keep reading the empty snapshot after the concurrent INSERT"); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCreateTableValidationTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCreateTableValidationTest.java index 078e2e9b568518..40db02b89e008a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCreateTableValidationTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCreateTableValidationTest.java @@ -53,7 +53,7 @@ public class IcebergCreateTableValidationTest { private IcebergConnectorMetadata metadata() { - return new IcebergConnectorMetadata(null, null, null); + return new IcebergConnectorMetadata(null, IcebergCatalogProperties.of(Collections.emptyMap()), null); } private ConnectorColumn col(String name, String type) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLiveConnectivityTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLiveConnectivityTest.java index 61764073d799d3..7121beba8839fd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLiveConnectivityTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergLiveConnectivityTest.java @@ -72,7 +72,7 @@ public void liveMetadataRoundTrip() { String warehouse = System.getenv("ICEBERG_WAREHOUSE"); Map props = new HashMap<>(); - props.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); + props.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); props.put("iceberg.rest.uri", restUri); if (warehouse != null && !warehouse.isEmpty()) { props.put("warehouse", warehouse); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProviderSessionRoutingTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProviderSessionRoutingTest.java index 55e03a0b91a862..ef17543e33bccf 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProviderSessionRoutingTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergProviderSessionRoutingTest.java @@ -70,7 +70,7 @@ private static Function failClosedResolver( @Test public void scanProviderAppliesResolverWithCallSessionAndFailsClosed() { List seen = new ArrayList<>(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), failClosedResolver(seen, new RecordingIcebergCatalogOps()), null, null, null); RoutingSession noCred = new RoutingSession(null); @@ -91,7 +91,7 @@ public void scanProviderRoutesCredentialedSessionToPerUserOps() { // Record the loadTable call, then stop the method — the routing target is all this test asserts, not the // downstream split planning (which would need a live table). perUserOps.throwOnLoadTable = true; - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), failClosedResolver(seen, perUserOps), null, null, null); RoutingSession cred = new RoutingSession(CRED); @@ -107,7 +107,7 @@ public void scanProviderRoutesCredentialedSessionToPerUserOps() { @Test public void writeProviderAppliesResolverWithCallSessionAndFailsClosed() { List seen = new ArrayList<>(); - IcebergWritePlanProvider provider = new IcebergWritePlanProvider(Collections.emptyMap(), + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), failClosedResolver(seen, new RecordingIcebergCatalogOps()), null); RoutingSession noCred = new RoutingSession(null); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderClassifyColumnTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderClassifyColumnTest.java index 6dbb821fd9dda8..bdca87a49365ee 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderClassifyColumnTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderClassifyColumnTest.java @@ -42,7 +42,7 @@ public class IcebergScanPlanProviderClassifyColumnTest { private static final IcebergScanPlanProvider PROVIDER = - new IcebergScanPlanProvider(Collections.emptyMap(), null); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), null); @Test public void hiddenRowIdIsSynthesized() { diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderKerberosScanIoTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderKerberosScanIoTest.java index c5c3f0bd26635d..66c5c05bc09699 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderKerberosScanIoTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderKerberosScanIoTest.java @@ -115,7 +115,7 @@ public void kerberosPlanScanRoutesManifestReadsThroughPluginDoAs() { TcclPinningConnectorContext context = new TcclPinningConnectorContext( new RecordingConnectorContext(), getClass().getClassLoader(), () -> auth); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -139,7 +139,7 @@ public void nonKerberosPlanScanKeepsDelegatePathAndNoWrap() { TcclPinningConnectorContext context = new TcclPinningConnectorContext( delegate, getClass().getClassLoader(), () -> null); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -159,7 +159,7 @@ public void wrapTableForScanWrapsIoFactoryCallsInPluginDoAs() { TcclPinningConnectorContext context = new TcclPinningConnectorContext( new RecordingConnectorContext(), getClass().getClassLoader(), () -> auth); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); Table wrapped = provider.wrapTableForScan(table); @@ -178,7 +178,7 @@ public void plainContextWrapIsIdentityPassThrough() { Table table = oneFileTable(); // A non-TcclPinning context (offline tests / fe-core fakes) must never be wrapped. IcebergScanPlanProvider provider = new IcebergScanPlanProvider( - Collections.emptyMap(), opsReturning(table), new RecordingConnectorContext()); + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), new RecordingConnectorContext()); Assertions.assertSame(table, provider.wrapTableForScan(table)); } @@ -188,7 +188,7 @@ public void sysTablePlanningRunsInsideAuthScope() { Table table = oneFileTable(); RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder( diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderPartitionCountTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderPartitionCountTest.java index 0820fefcc3154a..cbc71e8530ac4e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderPartitionCountTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderPartitionCountTest.java @@ -75,7 +75,7 @@ public void scannedPartitionCountReturnsDistinctPartitions() { // FIX-L12 THE load-bearing RED assertion: three files over TWO distinct partitions (two files in // partition [1] + one in [2]) count 2, not 3. A mutation dropping the override (default // OptionalLong.empty()) makes this red. - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), null); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), null); List ranges = Arrays.asList( range("/t/a.parquet", 0, "[\"1\"]"), range("/t/b.parquet", 0, "[\"1\"]"), @@ -87,7 +87,7 @@ public void scannedPartitionCountReturnsDistinctPartitions() { public void scannedPartitionCountEmptyForUnpartitionedTable() { // No range carries a partition key (unpartitioned) -> report nothing so the engine keeps its count. // (Same value as the un-overridden default; documents the fall-through, not RED-able.) - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), null); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), null); List ranges = Arrays.asList( range("/t/a.parquet", null, null), range("/t/b.parquet", null, null)); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java index 80e0da6816a103..55f6c087f08931 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergScanPlanProviderTest.java @@ -171,7 +171,7 @@ private static ConnectorExpression eqInt(String col, int value) { @Test public void ignorePartitionPruneShortCircuitIsTrue() { IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), new RecordingIcebergCatalogOps()); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingIcebergCatalogOps()); // WHY: iceberg is predicate-driven — it re-plans through its own SDK from the pushed predicate and // never consults requiredPartitions (same as the legacy IcebergScanNode / paimon). So a GENUINE FE // prune-to-zero must scan-all rather than short-circuit to zero rows. MUTATION: default false -> red. @@ -181,7 +181,7 @@ public void ignorePartitionPruneShortCircuitIsTrue() { @Test public void supportsSystemTableTimeTravelIsTrue() { IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), new RecordingIcebergCatalogOps()); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingIcebergCatalogOps()); // WHY: iceberg metadata tables legally time-travel (t$snapshots FOR TIME AS OF ..., t$files@branch). // legacy IcebergScanNode.createTableScan honors useRef/useSnapshot for sys tables with no // isSystemTable gate, and this provider retains+honors the pin. So the generic @@ -196,7 +196,7 @@ public void planScanResolvesTableViaSeamAndEmptyTableReturnsNoSplits() { // An empty table (no snapshot) plans no files -> no ranges; proves the (db, table) coordinates were // threaded through the seam to loadTable, and the real scan path tolerates an empty table. RecordingIcebergCatalogOps ops = opsReturning(createTable("t1", SCHEMA, PartitionSpec.unpartitioned())); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -214,7 +214,7 @@ public void planScanResolvesTableInsideAuthContext() { RecordingIcebergCatalogOps ops = opsReturning(createTable("t1", SCHEMA, PartitionSpec.unpartitioned())); RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), ops, context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops, context); provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -235,8 +235,8 @@ public void planningPassLoadsSameTableOnceViaSharedScope() { Table empty = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); RecordingIcebergCatalogOps ops = opsReturning(empty); IcebergConnectorMetadata metadata = - new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops); IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); ConnectorSession session = new FakeScanSession("UTC", Collections.emptyMap()) .withScope(new TestStatementScope()); @@ -256,8 +256,8 @@ public void planningPassSystemTableLoadsSameBaseOnceViaSharedScope() { dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); RecordingIcebergCatalogOps ops = opsReturning(table); IcebergConnectorMetadata metadata = - new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops); IcebergTableHandle handle = IcebergTableHandle.forSystemTable( "db1", "t1", "snapshots", -1L, null, -1L); ConnectorSession session = new FakeScanSession("UTC", Collections.emptyMap()) @@ -280,8 +280,8 @@ public void planningPassWithoutSharedScopeLoadsEachTime() { Table empty = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); RecordingIcebergCatalogOps ops = opsReturning(empty); IcebergConnectorMetadata metadata = - new IcebergConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops); IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); ConnectorSession none = new FakeScanSession("UTC", Collections.emptyMap()); @@ -301,7 +301,7 @@ public void planScanEnumeratesOneRangePerDataFile() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 2048, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -329,7 +329,7 @@ public void planScanRewriteFileScopeKeepsOnlyRawScopedFiles() { .appendFile(dataFile(table.spec(), "oss://b/db/t1/f3.parquet", 4096, null, null)) .commit(); IcebergScanPlanProvider provider = new IcebergScanPlanProvider( - Collections.emptyMap(), opsReturning(table), new RecordingConnectorContext()); + IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), new RecordingConnectorContext()); // A rewrite group bin-packed to f1 + f3 only; f2 must be dropped. IcebergTableHandle scoped = new IcebergTableHandle("db1", "t1").withRewriteFileScope( @@ -361,7 +361,7 @@ public void planScanNullRewriteScopeReadsAllFiles() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 2048, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // WHY: a bare handle (no rewrite scope) is EVERY normal scan; it must read all files, not zero. MUTATION: // the scope guard firing on a null scope (e.g. `scope.isEmpty()` instead of `scope != null`) -> 0 ranges @@ -381,7 +381,7 @@ public void planScanSplitsLargeFileByFileSplitSizeSessionVar() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/big.parquet", 96 * mb, Arrays.asList(0L, 32 * mb, 64 * mb), null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // file_split_size = 32MB forces splitting at that granularity (mirrors SessionVariable override path). ConnectorSession session = new FakeScanSession("UTC", Collections.singletonMap("file_split_size", Long.toString(32 * mb))); @@ -419,7 +419,7 @@ public void planScanMemoizesPerFileInvariantsAcrossByteSlices() { .appendFile(dataFile(spec, "s3://b/db/pt/p=7/big.parquet", 96 * mb, Arrays.asList(0L, 32 * mb, 64 * mb), "p=7")) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorSession session = new FakeScanSession("UTC", Collections.singletonMap("file_split_size", Long.toString(32 * mb))); @@ -459,7 +459,7 @@ public void planScanComputesPerFileInvariantsOncePerDistinctFile() { .appendFile(dataFile(spec, "s3://b/db/pt/p=2/b.parquet", 96 * mb, Arrays.asList(0L, 32 * mb, 64 * mb), "p=2")) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorSession session = new FakeScanSession("UTC", Collections.singletonMap("file_split_size", Long.toString(32 * mb))); @@ -494,7 +494,7 @@ public void planScanRangesCarrySizeProportionalWeight() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 2048, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -525,7 +525,7 @@ public void planScanSelfSplitWeightIncludesDeleteFileSizes() { table.newRowDelta() .addDeletes(positionDeleteFile("s3://b/db/t1/pos.parquet", FileFormat.PARQUET, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(new FakeScanSession("UTC", Collections.emptyMap()), ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -550,7 +550,7 @@ public void planScanFileSplitSizeUsesFileSplitSizeAsWeightDenominator() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/big.parquet", 96 * mb, Arrays.asList(0L, 32 * mb, 64 * mb), null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorSession session = new FakeScanSession("UTC", Collections.singletonMap("file_split_size", Long.toString(16 * mb))); @@ -573,7 +573,7 @@ public void planScanPushesPredicateAndPrunesPartition() { .appendFile(dataFile(spec, "s3://b/db/pt/p1.parquet", 512, null, "p=1")) .appendFile(dataFile(spec, "s3://b/db/pt/p2.parquet", 512, null, "p=2")) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // WHERE p = 1 must push to the scan and prune the p=2 file -> only the p=1 data file is enumerated. // This proves the converted predicate reaches scan.filter and is honoured by iceberg planning. @@ -597,7 +597,7 @@ public void planScanUnpushablePredicateScansAllFiles() { table.newAppend() .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // A predicate the converter drops (id = 'abc' : string -> INTEGER fails) leaves no filter on the scan, // so all files are scanned (safe over-approximation) rather than crashing or pruning everything. @@ -644,7 +644,7 @@ public void planScanPopulatesPerFilePartitionAndFormatCarriers() { .appendFile(dataFile(spec, "s3://b/db/pt/p=1/a.parquet", 512, null, "p=1", FileFormat.PARQUET)) .appendFile(dataFile(spec, "s3://b/db/pt/p=2/b.orc", 512, null, "p=2", FileFormat.ORC)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "pt"), Collections.emptyList()) @@ -680,7 +680,7 @@ public void getScanNodePropertiesEmitsPathPartitionKeysAndRealDataFormat() { Types.NestedField.required(3, "region", Types.StringType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).identity("P").identity("region").build(); Table table = createTable("pt", schema, spec); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "pt"), Collections.emptyList(), Optional.empty()); @@ -703,7 +703,7 @@ public void getScanNodePropertiesResolvesOrcTableFormat() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); Table table = createTable("orct", schema, PartitionSpec.unpartitioned(), Collections.singletonMap(TableProperties.DEFAULT_FILE_FORMAT, "orc")); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "orct"), Collections.emptyList(), Optional.empty()); @@ -717,7 +717,7 @@ public void getScanNodePropertiesResolvesOrcTableFormat() { @Test public void getScanNodePropertiesOmitsPathPartitionKeysForUnpartitionedTable() { Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -734,7 +734,7 @@ public void getScanNodePropertiesEmitsFieldIdSchemaEvolutionDictKeyedOffRequeste // it BE name-matches schema-evolved files (NULL/garbage on rename) or DCHECK-aborts. MUTATION: not // emitting the prop -> absent -> red. MUTATION: keying off all columns -> the entry includes "name" -> red. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List columns = Collections.singletonList(new IcebergColumnHandle("id", 1)); @@ -762,7 +762,7 @@ public void populateScanLevelParamsAdvertisesIcebergScanSemanticsV2() throws Exc // .enableCurrentIcebergScanSemantics(). MUTATION: drop the setIcebergScanSemanticsVersion call -> unset // -> red. The marker must be stamped independently of the dict, so exercise it with an EMPTY props map. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); TFileScanRangeParams params = new TFileScanRangeParams(); provider.populateScanLevelParams(params, Collections.emptyMap()); @@ -794,7 +794,7 @@ public void getScanNodePropertiesForcesEqualityDeleteKeyColumnIntoDict() throws // Sanity: the identifier field must survive table creation (iceberg reassigns ids but keeps the key). Assertions.assertFalse(table.schema().identifierFieldIds().isEmpty(), "the declared identifier field must be preserved on the created table"); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // Project ONLY the non-key data column "name". List columns = Collections.singletonList(new IcebergColumnHandle("name", 2)); @@ -1041,7 +1041,7 @@ public void getScanNodePropertiesEmitsSchemaEvolutionDictForPartitionedTableToo( Types.NestedField.required(2, "p", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).identity("p").build(); Table table = createTable("pt", schema, spec); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "pt"), Collections.emptyList(), Optional.empty()); @@ -1063,7 +1063,7 @@ public void getScanNodePropertiesForUnpinnedSysHandleSkipsDictAndPathKeysWithout // throws here -> red. PartitionSpec spec = PartitionSpec.builderFor(PART_SCHEMA).identity("p").build(); Table table = createTable("pt", PART_SCHEMA, spec); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // Requested columns are metadata-table columns, NOT present in the base (id, p) schema. List metaColumns = Arrays.asList( new IcebergColumnHandle("committed_at", 1), @@ -1091,7 +1091,7 @@ public void getScanNodePropertiesForPinnedSysHandleSkipsDict() { Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); long s1 = table.currentSnapshot().snapshotId(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, IcebergTableHandle.forSystemTable("db1", "t1", "files", s1, null, -1L), @@ -1118,7 +1118,7 @@ public void getScanNodePropertiesForSysHandleStillEmitsLocationCreds() { beStatic.put("AWS_SECRET_KEY", "sk"); context.storageProperties = Collections.singletonList(fakeBackendStorage(beStatic)); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); // Requested columns are metadata columns absent from the base schema (the dict-throw trap). List metaColumns = Arrays.asList( new IcebergColumnHandle("committed_at", 1), @@ -1151,7 +1151,7 @@ public void planSystemTableScanProjectsRequestedColumnsExcludingReadableMetrics( table.newAppend() .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder( @@ -1188,7 +1188,7 @@ public void planSystemTableScanProjectionPreservesRequestedColumnOrderForPositio .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .commit(); long snapshotId = table.currentSnapshot().snapshotId(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // $snapshots table order is committed_at, snapshot_id, parent_id, operation, ...; request operation // BEFORE snapshot_id so a table-order projection would visibly disagree with the requested order. @@ -1253,7 +1253,7 @@ private static TFileRangeDesc positionDeleteRangeDesc(List r } private static List planPositionDeletes(Table table, List cols) { - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); return provider.planScan(new FakeScanSession("UTC", Collections.emptyMap()), ConnectorScanRequest.builder( IcebergTableHandle.forSystemTable("db1", "t1", "position_deletes", -1L, null, -1L), @@ -1349,7 +1349,7 @@ public void getScanNodePropertiesForPositionDeletesSysHandleEmitsDict() { // MUTATION: widening the guard back to `if (!systemTable)` -> no dict -> red. Table table = tableWithPositionDelete( positionDeleteFile("s3://b/db/t1/pos.parquet", FileFormat.PARQUET, null, null)); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // Metadata-table columns, absent from the base (id, name) schema — the dict-throw trap. List metaColumns = Arrays.asList( new IcebergColumnHandle("file_path", 1), @@ -1378,7 +1378,7 @@ public void getScanNodePropertiesForPositionDeletesLoadsTheBaseTableOnlyOnce() { Table table = tableWithPositionDelete( positionDeleteFile("s3://b/db/t1/pos.parquet", FileFormat.PARQUET, null, null)); RecordingIcebergCatalogOps ops = opsReturning(table); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), ops); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops); Map props = provider.getScanNodeProperties( null, IcebergTableHandle.forSystemTable("db1", "t1", "position_deletes", -1L, null, -1L), @@ -1401,7 +1401,7 @@ public void planScanPinnedToOlderSnapshotReadsOnlyThatSnapshotsFiles() { long s1 = table.currentSnapshot().snapshotId(); long schemaIdS1 = table.currentSnapshot().schemaId(); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List latest = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -1429,7 +1429,7 @@ public void planScanResolvedEmptySnapshotIgnoresConcurrentFirstAppend() { table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List rows = provider.planScan(null, ConnectorScanRequest.builder(emptyRead, Collections.emptyList()).build()); @@ -1447,7 +1447,7 @@ public void streamSplitsResolvedEmptySnapshotIgnoresConcurrentFirstAppend() thro IcebergTableHandle emptyRead = new IcebergTableHandle("db1", "t1") .withSnapshot(-1L, null, table.schema().schemaId()); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); @@ -1465,7 +1465,7 @@ public void streamSplitsResolvedEmptySnapshotIgnoresConcurrentFirstAppend() thro public void resolvedEmptyMetadataLogEntriesStillReturnsCreationEntry() throws Exception { Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); IcebergTableHandle handle = IcebergTableHandle.forSystemTable( "db1", "t1", "metadata_log_entries", -1L, null, table.schema().schemaId(), true); @@ -1485,7 +1485,7 @@ public void resolvedEmptySnapshotDerivedMetadataTablesIgnoreConcurrentFirstAppen table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Assertions.assertTrue(provider.planScan(null, ConnectorScanRequest.builder(emptyRead, Collections.emptyList()).build()).isEmpty(), @@ -1501,7 +1501,7 @@ public void resolvedEmptyAllFilesDoesNotExposeConcurrentFirstAppend() { table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/concurrent.parquet", 1024, null, null)).commit(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Assertions.assertTrue(provider.planScan(null, ConnectorScanRequest.builder(emptyAllFiles, Collections.emptyList()).build()).isEmpty(), @@ -1519,7 +1519,7 @@ public void planScanPinnedToTagReadsViaUseRefNotSnapshotId() { table.manageSnapshots().createTag("tag1", s1).commit(); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 1024, null, null)).commit(); long s2 = table.currentSnapshot().snapshotId(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List pinned = provider.planScan(null, ConnectorScanRequest.builder( @@ -1539,7 +1539,7 @@ public void countPushdownFollowsTheSnapshotPin() { long s1 = table.currentSnapshot().snapshotId(); long schemaIdS1 = table.currentSnapshot().schemaId(); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 2000, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List pinned = provider.planScan(null, ConnectorScanRequest.builder( @@ -1563,7 +1563,7 @@ public void getScanNodePropertiesUnderPinEmitsFullPinnedSchemaDict() throws Exce long schemaIdS1 = table.currentSnapshot().schemaId(); table.updateSchema().renameColumn("name", "fullname").commit(); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1").withSnapshot(s1, null, schemaIdS1), @@ -1587,7 +1587,7 @@ public void getScanNodePropertiesUnderPinConvertsPredicatesWithPinnedSchema() { long s1 = table.currentSnapshot().snapshotId(); long schemaIdS1 = table.currentSnapshot().schemaId(); table.updateSchema().renameColumn("name", "fullname").commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorExpression oldNamePredicate = new ConnectorComparison(ConnectorComparison.Operator.EQ, new ConnectorColumnRef("name", ConnectorType.of("VARCHAR")), new ConnectorLiteral(ConnectorType.of("VARCHAR"), "alice")); @@ -1612,7 +1612,7 @@ public void getScanNodePropertiesUnderTopnLazyMatEmitsFullLatestSchemaDict() thr // must carry "name". MUTATION: dropping the isTopnLazyMaterialize() branch -> keyed off `columns` -> // dict has only "id" -> red. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1").withTopnLazyMaterialize(true), @@ -1639,7 +1639,7 @@ public void getScanNodePropertiesPinTakesPrecedenceOverTopnLazyMat() throws Exce long s1 = table.currentSnapshot().snapshotId(); long schemaIdS1 = table.currentSnapshot().schemaId(); table.updateSchema().renameColumn("name", "fullname").commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1").withSnapshot(s1, null, schemaIdS1) @@ -1661,7 +1661,7 @@ public void planScanReadsRealFormatVersionAndEmitsV3RowLineage() { table.newAppend() .appendFile(dataFile(table.spec(), "s3://b/db/v3t/f.parquet", 512, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "v3t"), Collections.emptyList()) @@ -1694,7 +1694,7 @@ public void planScanAccumulatesRewritableDeletesKeyedByRawDataFilePathForV3() { .addDeletes(deletionVectorFile("s3://b/db/t1/dv.puffin", 16L, 64L)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorSession session = new FakeScanSession("UTC", Collections.emptyMap()) .withScope(new TestStatementScope()); provider.planScan(session, @@ -1725,7 +1725,7 @@ public void planScanDoesNotAccumulateForVersionTwo() { .addDeletes(positionDeleteFile("s3://b/db/t1/pos.parquet", FileFormat.PARQUET, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorSession session = new FakeScanSession("UTC", Collections.emptyMap()) .withScope(new TestStatementScope()); provider.planScan(session, @@ -1749,7 +1749,7 @@ public void planScanUnderNoneScopeIsInert() { table.newRowDelta() .addDeletes(deletionVectorFile("s3://b/db/t1/dv.puffin", 16L, 64L)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(new FakeScanSession("UTC", Collections.emptyMap()), ConnectorScanRequest.builder(new IcebergTableHandle("db1", "v3ns"), Collections.emptyList()) @@ -1766,7 +1766,7 @@ public void planScanRejectsUnsupportedFileFormatFailLoud() { table.newAppend() .appendFile(dataFile(table.spec(), "s3://b/db/t1/f.avro", 512, null, null, FileFormat.AVRO)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // MUTATION: leaving the else-branch silent (FORMAT_JNI default) -> no throw -> red. IllegalStateException ex = Assertions.assertThrows(IllegalStateException.class, () -> provider.planScan(null, @@ -1801,7 +1801,7 @@ private static Table threeFileTable() { } private static IcebergScanPlanProvider providerOver(Table table) { - return new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + return new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); } private static ConnectorSession emptySession() { @@ -1851,7 +1851,7 @@ public void streamingSplitEstimateDisabledBySessionVarStaysSynchronous() { public void streamingSplitEstimateEmptyTableStaysSynchronous() { // No snapshot (no append) -> nothing to stream. MUTATION: dropping the snapshot==null guard -> NPE/red. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); long estimate = provider.streamingSplitEstimate(batchSession(0, true), new IcebergTableHandle("db1", "t1"), Optional.empty(), false); Assertions.assertEquals(-1, estimate, "empty table must not stream"); @@ -1873,7 +1873,7 @@ public void streamingSplitEstimateV3StaysSynchronous() { // write-plan time; streaming would fill it too late (BE-pull time) and resurrect deleted rows. So v3 is // gated onto the eager path. MUTATION: `>= 3` -> `> 3` (or dropping the guard) -> 3 -> red (correctness). Table table = threeFileTable(Collections.singletonMap("format-version", "3")); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); long estimate = provider.streamingSplitEstimate(batchSession(2, true), new IcebergTableHandle("db1", "t1"), Optional.empty(), false); Assertions.assertEquals(-1, estimate, "v3 (deletion-vector) tables must not stream"); @@ -1909,7 +1909,7 @@ public void streamSplitsRewriteScopeSkipsUnscopedFilesViaLookahead() throws IOEx // dropping the rewrite-scope skip -> f2 leaks (3 ranges) -> red; a broken look-ahead (no skip) would // surface a null -> red. Table table = threeFileTable(Collections.emptyMap()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); IcebergTableHandle scoped = new IcebergTableHandle("db1", "t1").withRewriteFileScope( ImmutableSet.of("s3://b/db/t1/f1.parquet", "s3://b/db/t1/f3.parquet")); List ranges = drain(provider.streamSplits(emptySession(), @@ -2058,7 +2058,7 @@ private static DeleteFile equalityDeleteFile(String path, FileFormat format, int } private static IcebergScanPlanProvider provider() { - return new IcebergScanPlanProvider(Collections.emptyMap(), new RecordingIcebergCatalogOps()); + return new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingIcebergCatalogOps()); } private static TIcebergDeleteFileDesc deleteDesc(String path, int content) { @@ -2194,7 +2194,7 @@ public void convertDeleteNormalizesDeletePathViaContext() { // rows. MUTATION: handing BE the raw oss:// path -> red. RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), new RecordingIcebergCatalogOps(), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingIcebergCatalogOps(), context); DeleteFile delete = positionDeleteFile("oss://bucket/db/t1/pos.parquet", FileFormat.PARQUET, null, null); TIcebergDeleteFileDesc d = @@ -2251,7 +2251,7 @@ public void planScanAttachesRealPositionDeleteEndToEnd() { .withReferencedDataFile("s3://b/db/t1/f1.parquet") .build(); table.newRowDelta().addDeletes(posDelete).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -2283,7 +2283,7 @@ public void planScanNormalizesDataFilePathButKeepsOriginalFilePathRaw() { .commit(); RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -2321,7 +2321,7 @@ public void countPushdownCollapsesToSingleRangeWithTotalRecords() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 2048, null, null)) .appendFile(dataFile(table.spec(), "s3://b/db/t1/f3.parquet", 3072, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = planCount(provider, null, true); @@ -2349,7 +2349,7 @@ public void countPushdownNotAppliedWithEqualityDeletesScansAll() { table.newRowDelta() .addDeletes(equalityDeleteFile("s3://b/db/t1/eq.parquet", FileFormat.PARQUET, 1)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = planCount(provider, null, true); @@ -2378,7 +2378,7 @@ public void countPushdownWithPositionDeletesNetsOutWhenIgnoringDangling() { .withReferencedDataFile("s3://b/db/t1/f1.parquet") .build(); table.newRowDelta().addDeletes(posDelete).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); ConnectorSession session = new FakeScanSession("UTC", Collections.singletonMap("ignore_iceberg_dangling_delete", "true")); @@ -2407,7 +2407,7 @@ public void countPushdownWithPositionDeletesScansAllWhenNotIgnoringDangling() { .withReferencedDataFile("s3://b/db/t1/f1.parquet") .build(); table.newRowDelta().addDeletes(posDelete).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // No ignore flag (null session -> default false): position deletes present -> not pushable. List ranges = planCount(provider, null, true); @@ -2425,7 +2425,7 @@ public void countPushdownEmptyTableProducesNoRanges() { // 0 ranges -> COUNT returns 0 (legacy returns empty splits too). MUTATION: emitting a synthetic count // range with no path -> red (no file to build from). Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = planCount(provider, null, true); @@ -2440,7 +2440,7 @@ public void countPushdownFalseDoesNormalMultiRangeScan() { .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)) .appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 1024, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = planCount(provider, null, false); @@ -2462,7 +2462,7 @@ private static Map manifestCacheProps() { /** A provider whose manifest-level path (and IcebergManifestCache) is enabled. */ private static IcebergScanPlanProvider manifestProvider(Map props, Table table, IcebergManifestCache cache) { - return new IcebergScanPlanProvider(props, opsReturning(table), null, cache); + return new IcebergScanPlanProvider(IcebergCatalogProperties.of(props), opsReturning(table), null, cache); } private static List sortedPaths(List ranges) { @@ -2485,7 +2485,7 @@ public void planScanManifestCacheEnabledMatchesSdkPathAndConsumesCache() { IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); // Gate OFF (default): the iceberg SDK splitFiles path (T02). - List sdk = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)) + List sdk = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) .planScan(null, ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); // Gate ON: the manifest-level path that reads manifests through the cache. @@ -2513,7 +2513,7 @@ public void planScanManifestCachePrunesPartitionLikeSdk() { IcebergTableHandle handle = new IcebergTableHandle("db1", "pt"); Optional wherePeq1 = Optional.of(eqInt("p", 1)); - List sdk = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)) + List sdk = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) .planScan(null, ConnectorScanRequest.builder(handle, Collections.emptyList()).filter(wherePeq1) .build()); IcebergManifestCache cache = new IcebergManifestCache(); @@ -2544,7 +2544,7 @@ public void streamSplitsManifestCacheEnabledMatchesSdkPathAndConsumesCache() thr IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); // Cache OFF: the streaming SDK planFiles() path (the pre-PERF-04 behavior). - List sdk = drain(new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)) + List sdk = drain(new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) .streamSplits(emptySession(), handle, Collections.emptyList(), Optional.empty(), -1L)); // Cache ON: streaming must now read manifests THROUGH the cache and yield the SAME files (PERF-04 C17). // MUTATION: streaming still bypasses the cache (scan.planFiles()) -> cache stays empty -> red. @@ -2568,7 +2568,7 @@ public void streamSplitsManifestCachePrunesPartitionLikeSdk() throws IOException IcebergTableHandle handle = new IcebergTableHandle("db1", "pt"); Optional wherePeq1 = Optional.of(eqInt("p", 1)); - List sdk = drain(new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)) + List sdk = drain(new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) .streamSplits(emptySession(), handle, Collections.emptyList(), wherePeq1, -1L)); IcebergManifestCache cache = new IcebergManifestCache(); List cached = drain(manifestProvider(manifestCacheProps(), table, cache) @@ -2613,7 +2613,7 @@ public void countPushdownManifestCacheMatchesCountAndReadsLazily() { Assertions.assertTrue(totalManifests >= 2, "precondition: multiple data manifests"); List sdk = planCount( - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)), null, true); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)), null, true); IcebergManifestCache cache = new IcebergManifestCache(); List cached = planCount(manifestProvider(manifestCacheProps(), table, cache), emptySession(), true); @@ -2697,7 +2697,7 @@ public void planScanManifestCacheAssociatesDeletesLikeSdk() { .commit(); IcebergTableHandle handle = new IcebergTableHandle("db1", "t1"); - List sdk = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)) + List sdk = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)) .planScan(null, ConnectorScanRequest.builder(handle, Collections.emptyList()).build()); IcebergManifestCache cache = new IcebergManifestCache(); List manifest = manifestProvider(manifestCacheProps(), table, cache) @@ -2774,7 +2774,7 @@ public void getScanNodePropertiesEmitsStaticStorageCredsAsLocation() { beStatic.put("AWS_ENDPOINT", "ep"); context.storageProperties = Collections.singletonList(fakeBackendStorage(beStatic)); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -2804,7 +2804,7 @@ public void getScanNodePropertiesOverlaysVendedCredsOverStatic() { vended.put("AWS_ENDPOINT", "vended-ep"); context.vendedBeProps = vended; IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(restVendedFlagOn(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(restVendedFlagOn()), opsReturning(table), context); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -2826,7 +2826,7 @@ public void getScanNodePropertiesOmitsVendedWhenFlagDisabled() { context.vendedBeProps = Collections.singletonMap("AWS_ACCESS_KEY", "vended-ak"); // No vended flag -> default false. IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -2846,10 +2846,10 @@ public void getScanNodePropertiesOmitsVendedWhenFlagSetButNonRestFlavor() { // The vended flag is set, but the catalog flavor is HMS (not REST). Legacy isVendedCredentialsEnabled is // `instanceof IcebergRestProperties && flag`, so a non-REST catalog NEVER vends even with the flag set. Map hmsWithRestFlag = new HashMap<>(); - hmsWithRestFlag.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_HMS); - hmsWithRestFlag.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + hmsWithRestFlag.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_HMS); + hmsWithRestFlag.put("iceberg.rest.vended-credentials-enabled", "true"); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(hmsWithRestFlag, opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(hmsWithRestFlag), opsReturning(table), context); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -2866,7 +2866,7 @@ public void getScanNodePropertiesNoContextEmitsNoLocation() { Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); // 2-arg ctor -> context == null (offline harness path). IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -2888,7 +2888,7 @@ public void getScanNodePropertiesSkipsStorageWithoutBackendModelAndMergesRest() // real object-store backend: exercises the .ifPresent skip and the multi-entry putAll merge. context.storageProperties = Arrays.asList(fakeStorageWithoutBackend(), fakeBackendStorage(beMap)); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); Map props = provider.getScanNodeProperties( null, new IcebergTableHandle("db1", "t1"), Collections.emptyList(), Optional.empty()); @@ -2917,7 +2917,7 @@ public void planScanThreadsVendedTokenIntoDataAndDeletePathNormalize() { // PLUMBING: planScan routes BOTH the data and delete paths through the 2-arg normalizeStorageUri, // passing the per-table vended token (empty here, but the MAP, not null). IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -2951,7 +2951,7 @@ public void planScanDerivesUriNormalizerOncePerScanNotPerFile() { .commit(); RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "t1"), Collections.emptyList()) @@ -2966,7 +2966,7 @@ public void planScanDerivesUriNormalizerOncePerScanNotPerFile() { public void convertDeleteNormalizesDeletePathViaVendedToken() { RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), new RecordingIcebergCatalogOps(), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), new RecordingIcebergCatalogOps(), context); DeleteFile delete = positionDeleteFile("oss://bucket/db/t1/pos.parquet", FileFormat.PARQUET, null, null); Map token = Collections.singletonMap("s3.access-key-id", "ak"); @@ -2993,7 +2993,7 @@ public void planScanDefaultSplitHeuristicTilesAndMaxFileSplitNumCapCollapses() { table.newAppend() .appendFile(dataFile(table.spec(), "s3://b/db/t1/big.parquet", 96 * mb, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // (1) DEFAULT heuristic (NO file_split_size override): total 96MB << maxSplitSize*maxInitialSplitNum // (12.8GB) so no escalation -> 32MB initial target; the 100000-file cap is far below 32MB -> the file @@ -3039,7 +3039,7 @@ public void planScanPartitionBearingFileWithNoIdentityValuesEmitsNoColumnsFromPa table.newAppend() .appendFile(dataFile(spec, "s3://b/db/ev/f.parquet", 512, null, "id_bucket=0", FileFormat.PARQUET)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "ev"), Collections.emptyList()) @@ -3125,7 +3125,7 @@ public void planScanCombinesPartitionPruneDeleteAndPathNormalizeOnOneRange() { table.newRowDelta().addDeletes(posDelete).commit(); RecordingConnectorContext context = new RecordingConnectorContext(); IcebergScanPlanProvider provider = - new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table), context); + new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table), context); List ranges = provider.planScan(null, ConnectorScanRequest.builder(new IcebergTableHandle("db1", "pt"), Collections.emptyList()) @@ -3167,8 +3167,8 @@ private static FakeIcebergTable fakeTable(String name) { /** Catalog props with BOTH the REST flavor and the vended flag on — the two-part legacy gate. */ private static Map restVendedFlagOn() { Map props = new HashMap<>(); - props.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - props.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + props.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + props.put("iceberg.rest.vended-credentials-enabled", "true"); return props; } @@ -3262,7 +3262,7 @@ public void planScanForSystemTableSerializesEachFileScanTaskAsJniSplit() { // serialized_split -> red. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder( @@ -3293,7 +3293,7 @@ public void planScanForSystemTableSplitDeserializesThroughTheBeJniReaderPath() t // asDataTask() fails or yields the wrong schema -> red. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder( @@ -3334,7 +3334,7 @@ public void planScanForSystemTableHonorsTheSnapshotPin() throws Exception { table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); long s1 = table.currentSnapshot().snapshotId(); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); long latestRows = countSerializedSplitRows(provider.planScan(null, ConnectorScanRequest.builder( @@ -3356,7 +3356,7 @@ public void planScanForStaticSystemTableDoesNotSelectSnapshot() throws Exception Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder( @@ -3375,7 +3375,7 @@ public void planScanForAllMetadataTablesDoesNotSelectSnapshotOrRef() throws Exce Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); table.newAppend().appendFile( dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); for (String systemTable : Arrays.asList( "all_data_files", "all_delete_files", "all_files", "all_manifests", "all_entries")) { @@ -3404,7 +3404,7 @@ public void planScanForSystemTableLoadsMetadataInsideTheAuthScope() { table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); RecordingIcebergCatalogOps ops = opsReturning(table); RecordingConnectorContext context = new RecordingConnectorContext(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), ops, context); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), ops, context); provider.planScan(null, ConnectorScanRequest.builder( @@ -3432,7 +3432,7 @@ public void planScanForSystemTableCarriesPredicateAsResidualForBe() throws Excep .appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1000, null, null)) .appendFile(dataFile(table.spec(), "s3://b/db/t1/f2.parquet", 100, null, null)) .commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); // record_count is an iceberg LONG field of the $files metadata schema; the converter resolves it by // name and pushes a BIGINT equality. @@ -3476,7 +3476,7 @@ public void planScanForSystemTableSetsDummyPathOnEverySplit() { // path != "/dummyPath" -> red. Table table = createTable("t1", SCHEMA, PartitionSpec.unpartitioned()); table.newAppend().appendFile(dataFile(table.spec(), "s3://b/db/t1/f1.parquet", 1024, null, null)).commit(); - IcebergScanPlanProvider provider = new IcebergScanPlanProvider(Collections.emptyMap(), opsReturning(table)); + IcebergScanPlanProvider provider = new IcebergScanPlanProvider(IcebergCatalogProperties.of(Collections.emptyMap()), opsReturning(table)); List ranges = provider.planScan(null, ConnectorScanRequest.builder( diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java index 342c97f91d6caa..bcfaf94a932a44 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergWritePlanProviderTest.java @@ -216,7 +216,7 @@ private static RecordingConnectorContext contextWithBroker(List properties) { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = table; - ConnectorSinkPlan plan = new IcebergWritePlanProvider(properties, ops, ctx) + ConnectorSinkPlan plan = new IcebergWritePlanProvider(IcebergCatalogProperties.of(properties), ops, ctx) .planWrite(sessionFor(table, ctx), handle); Assertions.assertEquals(TDataSinkType.ICEBERG_TABLE_SINK, plan.getDataSink().getType()); return plan.getDataSink().getIcebergTableSink(); @@ -493,7 +493,7 @@ public void planWriteAcceptsDorisPhysicalScalarAliases() { // Doris physical decimal widths and unbounded binary encoding must not look like schema evolution. Map properties = new HashMap<>(NON_REST_PROPS); - properties.put(IcebergConnectorProperties.ENABLE_MAPPING_VARBINARY, "true"); + properties.put(IcebergCatalogProperties.ENABLE_MAPPING_VARBINARY, "true"); Assertions.assertDoesNotThrow(() -> planSink(table, contextWithStorage(), new WriteHandle(new IcebergTableHandle("db1", "scalars")).boundTargetColumns(boundSchema), properties)); @@ -835,7 +835,7 @@ public void planWriteOverlaysVendedCredentials() { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = vendedTable; - IcebergWritePlanProvider provider = new IcebergWritePlanProvider(restVendedProps(), ops, ctx); + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(IcebergCatalogProperties.of(restVendedProps()), ops, ctx); WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); ConnectorSinkPlan plan = provider.planWrite(session, @@ -852,8 +852,8 @@ public void planWriteOverlaysVendedCredentials() { private static Map restVendedProps() { Map props = new HashMap<>(); - props.put(IcebergConnectorProperties.ICEBERG_CATALOG_TYPE, IcebergConnectorProperties.TYPE_REST); - props.put(IcebergConnectorProperties.REST_VENDED_CREDENTIALS_ENABLED, "true"); + props.put(IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST); + props.put("iceberg.rest.vended-credentials-enabled", "true"); return props; } @@ -1011,7 +1011,7 @@ public void planWriteRejectsSortOrderEvolutionAfterPhysicalShaping() { RecordingConnectorContext ctx = contextWithStorage(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = plannedTable; - IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(IcebergCatalogProperties.of(NON_REST_PROPS), ops, ctx); WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); String boundMetadataIdentity = provider.getWriteMetadataIdentity(session, tableHandle); @@ -1036,7 +1036,7 @@ public void planWriteRejectsPartitionFieldRenameAfterStaticOverwriteBinding() { RecordingConnectorContext ctx = contextWithStorage(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = plannedTable; - IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(IcebergCatalogProperties.of(NON_REST_PROPS), ops, ctx); WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t1"); String boundMetadataIdentity = provider.getWriteMetadataIdentity(session, tableHandle); @@ -1073,7 +1073,7 @@ private static void assertRowLevelModeEvolutionRejected(String modeProperty, Wri RecordingConnectorContext ctx = contextWithStorage(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = table; - IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(IcebergCatalogProperties.of(NON_REST_PROPS), ops, ctx); WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); IcebergTableHandle tableHandle = new IcebergTableHandle("db1", "t2"); table.updateProperties().set(modeProperty, "merge-on-read").commit(); @@ -1117,7 +1117,7 @@ public void planWriteRejectsDropRecreateBeforeFirstWritableTableLoad() { RecordingConnectorContext ctx = contextWithStorage(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = original; - IcebergWritePlanProvider provider = new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + IcebergWritePlanProvider provider = new IcebergWritePlanProvider(IcebergCatalogProperties.of(NON_REST_PROPS), ops, ctx); WriteSession session = new WriteSession(new IcebergConnectorTransaction(42L, ops, ctx)); catalog.dropTable(identifier, false); @@ -1411,7 +1411,7 @@ public void planWriteDeleteSinkFormatVersionThree() { private static IcebergWritePlanProvider supplyProvider(Table table, RecordingConnectorContext ctx) { RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = table; - return new IcebergWritePlanProvider(NON_REST_PROPS, ops, ctx); + return new IcebergWritePlanProvider(IcebergCatalogProperties.of(NON_REST_PROPS), ops, ctx); } private static WriteSession supplySession(Table table, RecordingConnectorContext ctx, String queryId) { @@ -1579,7 +1579,7 @@ public void planMergePreservesExplicitlyEmptyReadAcrossConcurrentFirstAppend() { RecordingConnectorContext ctx = contextWithStorage(); RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps(); ops.table = empty; - IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, Collections.emptyMap(), ctx); + IcebergConnectorMetadata metadata = new IcebergConnectorMetadata(ops, IcebergCatalogProperties.of(Collections.emptyMap()), ctx); ConnectorMvccSnapshot emptySnapshot = metadata.beginQuerySnapshot(null, new IcebergTableHandle("db1", "tv2")).orElseThrow(AssertionError::new); IcebergTableHandle emptyPinnedHandle = (IcebergTableHandle) metadata.applySnapshot( diff --git a/fe/fe-connector/fe-connector-jdbc/pom.xml b/fe/fe-connector/fe-connector-jdbc/pom.xml index 716a87ba1f9807..80532cc71313b6 100644 --- a/fe/fe-connector/fe-connector-jdbc/pom.xml +++ b/fe/fe-connector/fe-connector-jdbc/pom.xml @@ -46,6 +46,13 @@ under the License. ${project.version} + + + ${project.groupId} + fe-foundation + ${project.version} + + ${project.groupId} diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcCatalogProperties.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcCatalogProperties.java new file mode 100644 index 00000000000000..05cc546de4ee0e --- /dev/null +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcCatalogProperties.java @@ -0,0 +1,418 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.jdbc; + +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; +import org.apache.doris.foundation.property.ParamRules; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.UnaryOperator; + +/** + * Everything a user writes in {@code CREATE CATALOG} for a JDBC catalog, bound and checked. + * + *

fe-core parses no connector properties, so this class is where they are interpreted. Settings + * that are one-per-FE rather than one-per-catalog are not here -- they live in the plugin's + * {@code jdbc.conf} and are read through {@link JdbcConf}. + * + *

{@link #of} is deliberately thin, and {@link #checkCreateTimeOnlyRules()} is not. This + * connector has always validated a great deal more at {@code CREATE}/{@code ALTER} than at run time: + * the pool sizes had bounds no reader ever enforced, the boolean properties had to be spelled + * {@code true}/{@code false} where the reader would have taken anything, and the database-list + * consistency rule was checked once and never again. Those are rules about a statement, not + * invariants of a working catalog -- a stored catalog that breaks them runs today. of() runs on every + * rebuild, including on an FE replaying the edit log, so moving them into it would take such catalogs + * away from their owners with no statement able to repair them. They stay in the method the + * interactive doors call. + * + *

Unknown keys are accepted, always. The same map carries engine keys ({@code type}, + * {@code meta.cache.*}, ...) and storage keys, and {@code ALTER CATALOG} merges properties: it can + * overwrite a key but never remove one, so a key refused here would leave a catalog that no statement + * could repair. Bad values are refused; unrecognized names are not. + */ +public final class JdbcCatalogProperties { + + /** + * The prefix a user may put on any key, from when these were written as {@code jdbc.jdbc_url}. It + * is stripped from every key alike, so it is not an alias of any one property. + */ + public static final String JDBC_PROPERTIES_PREFIX = "jdbc."; + + // -- connection -- + + public static final String JDBC_URL = "jdbc_url"; + public static final String USER = "user"; + public static final String PASSWORD = "password"; + + // -- driver -- + + public static final String DRIVER_CLASS = "driver_class"; + + /** The driver jar: a bare file name (resolved under {@link JdbcConf#CONF_DRIVERS_DIR}) or a URL. */ + public static final String DRIVER_URL = "driver_url"; + + /** MD5 of the driver jar. Computed and stored at CREATE when the user names none. */ + public static final String DRIVER_CHECKSUM = "checksum"; + + // -- connection pool -- + + public static final String CONNECTION_POOL_MIN_SIZE = "connection_pool_min_size"; + public static final String CONNECTION_POOL_MAX_SIZE = "connection_pool_max_size"; + public static final String CONNECTION_POOL_MAX_WAIT_TIME = "connection_pool_max_wait_time"; + public static final String CONNECTION_POOL_MAX_LIFE_TIME = "connection_pool_max_life_time"; + public static final String CONNECTION_POOL_KEEP_ALIVE = "connection_pool_keep_alive"; + + // -- metadata filtering -- + + public static final String ONLY_SPECIFIED_DATABASE = "only_specified_database"; + public static final String INCLUDE_DATABASE_LIST = "include_database_list"; + public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; + + // -- type mapping options -- + + public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; + public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; + + // -- identifier mapping -- + + public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; + public static final String META_NAMES_MAPPING = "meta_names_mapping"; + + /** + * Rejected as a catalog property: it was replaced by {@link #LOWER_CASE_META_NAMES}. The same name + * is also a session variable that {@code JdbcConnectorMetadata} reads, which is a different + * thing that happens to be spelled the same. + */ + public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; + + /** + * Owned by the engine, which parses the rules; this connector reads the same key to decide what it + * may push down. The literal is copied rather than shared because the engine's classes are not on + * this plugin's compile path. + */ + public static final String FUNCTION_RULES = "function_rules"; + + /** + * Owned by the engine, which decides whether to run a connectivity test at CREATE. Validated here + * (it is this connector's answer that a bad value is refused) but never read here, so it gets no + * field -- mirroring a key's real owner rather than copying it. + */ + public static final String TEST_CONNECTION = "test_connection"; + + private static final int MAX_POOL_WAIT_TIME = 30000; + private static final int MIN_POOL_LIFE_TIME = 150000; + + @ConnectorProperty(names = {JDBC_URL}, description = "the JDBC connection URL") + private String jdbcUrl; + + @ConnectorProperty(names = {USER}, required = false, description = "remote user name") + private String user = ""; + + @ConnectorProperty(names = {PASSWORD}, required = false, sensitive = true, + description = "remote password") + private String password = ""; + + @ConnectorProperty(names = {DRIVER_CLASS}, required = false, + description = "the JDBC driver class name") + private String driverClass = ""; + + @ConnectorProperty(names = {DRIVER_URL}, required = false, + description = "the driver jar: a bare file name under drivers_dir, or a URL") + private String driverUrl = ""; + + @ConnectorProperty(names = {DRIVER_CHECKSUM}, required = false, + description = "MD5 of the driver jar") + private String driverChecksum = ""; + + @ConnectorProperty(names = {CONNECTION_POOL_MIN_SIZE}, required = false, + description = "connections kept open when idle") + private int connectionPoolMinSize = 1; + + @ConnectorProperty(names = {CONNECTION_POOL_MAX_SIZE}, required = false, + description = "the most connections one FE may open to the source") + private int connectionPoolMaxSize = 30; + + @ConnectorProperty(names = {CONNECTION_POOL_MAX_WAIT_TIME}, required = false, + description = "milliseconds a query waits for a free connection") + private int connectionPoolMaxWaitTime = 5000; + + @ConnectorProperty(names = {CONNECTION_POOL_MAX_LIFE_TIME}, required = false, + description = "milliseconds before a pooled connection is retired") + private int connectionPoolMaxLifeTime = 1800000; + + @ConnectorProperty(names = {CONNECTION_POOL_KEEP_ALIVE}, required = false, + description = "keep idle connections alive rather than letting them time out") + private boolean connectionPoolKeepAlive; + + @ConnectorProperty(names = {ONLY_SPECIFIED_DATABASE}, required = false, + description = "expose only the database named in the URL") + private boolean onlySpecifiedDatabase; + + @ConnectorProperty(names = {INCLUDE_DATABASE_LIST}, required = false, + description = "comma-separated databases to expose") + private String includeDatabaseList = ""; + + @ConnectorProperty(names = {EXCLUDE_DATABASE_LIST}, required = false, + description = "comma-separated databases to hide") + private String excludeDatabaseList = ""; + + @ConnectorProperty(names = {ENABLE_MAPPING_VARBINARY}, required = false, + description = "map the source's binary types to VARBINARY") + private boolean enableMappingVarbinary; + + @ConnectorProperty(names = {ENABLE_MAPPING_TIMESTAMP_TZ}, required = false, + description = "map the source's timestamp-with-time-zone types") + private boolean enableMappingTimestampTz; + + @ConnectorProperty(names = {FUNCTION_RULES}, required = false, + description = "engine-owned: which functions may be pushed down") + private String functionRules = ""; + + @ConnectorProperty(names = {LOWER_CASE_META_NAMES}, required = false, + description = "lower-case the source's database, table and column names") + private boolean lowerCaseMetaNames; + + @ConnectorProperty(names = {META_NAMES_MAPPING}, required = false, + description = "explicit name mapping, as JSON") + private String metaNamesMapping = ""; + + private final Map raw; + + private JdbcCatalogProperties(Map stripped) { + this.raw = Collections.unmodifiableMap(new LinkedHashMap<>(stripped)); + } + + /** For the validation doors, which care that a URL is present rather than what it normalizes to. */ + public static JdbcCatalogProperties of(Map properties) { + return of(properties, UnaryOperator.identity()); + } + + /** + * Binds and validates, with {@code urlNormalizer} applied to {@link #JDBC_URL}. + * + *

The normalization is a parameter rather than something this class does, because it depends on + * both halves of the configuration at once: the URL is a per-catalog property, but whether a SQL + * Server URL gets {@code encrypt=false} appended is a per-FE setting + * ({@link JdbcConf#forceSqlServerEncryptFalse}). A holder of per-catalog properties has no business + * reading deployment config, and a class of deployment config has no business rewriting a catalog's + * URL, so the connector -- which has both -- supplies the function. + */ + public static JdbcCatalogProperties of(Map properties, + UnaryOperator urlNormalizer) { + JdbcCatalogProperties p = new JdbcCatalogProperties(stripJdbcPrefix(properties)); + ConnectorPropertiesUtils.bindConnectorProperties(p, p.raw); + new ParamRules() + .require(p.jdbcUrl, "Required property '" + JDBC_URL + "' is missing") + .validate(); + p.jdbcUrl = urlNormalizer.apply(p.jdbcUrl); + return p; + } + + /** + * The rules that apply to a statement a user is writing now, but not to a catalog already stored. + * + *

Every one of them has only ever run at {@code CREATE}/{@code ALTER}: no reader enforces the + * pool bounds, no reader minds a boolean spelled {@code yes}, and the database-list rule is checked + * nowhere else. Running them from {@link #of} instead would make a stored catalog that breaks one + * of them unbuildable on the next FE restart, when it works today. See the class javadoc. + */ + public JdbcCatalogProperties checkCreateTimeOnlyRules() { + new ParamRules() + .require(driverUrl, "Required property '" + DRIVER_URL + "' is missing") + .require(driverClass, "Required property '" + DRIVER_CLASS + "' is missing") + .validate(); + + if (raw.containsKey(LOWER_CASE_TABLE_NAMES)) { + throw new IllegalArgumentException( + "Jdbc catalog property lower_case_table_names is not supported," + + " please use lower_case_meta_names instead"); + } + + // Spelled-out booleans: the binder's parseBoolean would read anything else as false, which is + // how a typo becomes a silently disabled option. + checkBoolean(ONLY_SPECIFIED_DATABASE); + checkBoolean(LOWER_CASE_META_NAMES); + checkBoolean(CONNECTION_POOL_KEEP_ALIVE); + checkBoolean(TEST_CONNECTION); + + if (!onlySpecifiedDatabase + && (!includeDatabaseList.isEmpty() || !excludeDatabaseList.isEmpty())) { + throw new IllegalArgumentException( + "include_database_list and exclude_database_list " + + "cannot be set when only_specified_database is false"); + } + + new ParamRules() + // ParamRules.check throws when the condition holds, so each states the FAILING case. + .check(() -> connectionPoolMinSize < 0, + "connection_pool_min_size must be greater than or equal to 0") + .check(() -> connectionPoolMaxSize < 1, + "connection_pool_max_size must be greater than or equal to 1") + .check(() -> connectionPoolMaxSize < connectionPoolMinSize, + "connection_pool_max_size must be greater than or equal to connection_pool_min_size") + .check(() -> connectionPoolMaxWaitTime < 0, + "connection_pool_max_wait_time must be greater than or equal to 0") + .check(() -> connectionPoolMaxWaitTime > MAX_POOL_WAIT_TIME, + "connection_pool_max_wait_time must be less than or equal to " + MAX_POOL_WAIT_TIME) + .check(() -> connectionPoolMaxLifeTime < MIN_POOL_LIFE_TIME, + "connection_pool_max_life_time must be greater than or equal to " + MIN_POOL_LIFE_TIME) + .validate(); + + if (!metaNamesMapping.isEmpty()) { + try { + // Both states of the session's lower_case_table_names, since it may enable it at runtime + // and a mapping collision can appear only after lower-casing. + new JdbcIdentifierMapper(false, lowerCaseMetaNames, metaNamesMapping); + new JdbcIdentifierMapper(true, lowerCaseMetaNames, metaNamesMapping); + } catch (RuntimeException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + return this; + } + + private void checkBoolean(String key) { + String value = raw.get(key); + if (value != null && !value.isEmpty() + && !value.equalsIgnoreCase("true") && !value.equalsIgnoreCase("false")) { + throw new IllegalArgumentException(key + " must be true or false"); + } + } + + /** + * Returns the properties with {@link #JDBC_PROPERTIES_PREFIX} removed from every key that carries + * it, leaving an unprefixed key already present untouched -- the short spelling wins, as it did in + * both places that used to resolve these keys. + */ + private static Map stripJdbcPrefix(Map properties) { + Map stripped = new LinkedHashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + if (!entry.getKey().startsWith(JDBC_PROPERTIES_PREFIX)) { + stripped.put(entry.getKey(), entry.getValue()); + } + } + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey(); + if (key.startsWith(JDBC_PROPERTIES_PREFIX)) { + String shortKey = key.substring(JDBC_PROPERTIES_PREFIX.length()); + if (!stripped.containsKey(shortKey)) { + stripped.put(shortKey, entry.getValue()); + } else { + // Keep the prefixed spelling too: it was left in the map before this class, and an + // unrecognized key must survive so ALTER CATALOG can still address it. + stripped.put(key, entry.getValue()); + } + } + } + return stripped; + } + + /** The URL, normalized when this instance was built by the connector. */ + public String getJdbcUrl() { + return jdbcUrl; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public String getDriverClass() { + return driverClass; + } + + public String getDriverUrl() { + return driverUrl; + } + + public String getDriverChecksum() { + return driverChecksum; + } + + public int getConnectionPoolMinSize() { + return connectionPoolMinSize; + } + + public int getConnectionPoolMaxSize() { + return connectionPoolMaxSize; + } + + public int getConnectionPoolMaxWaitTime() { + return connectionPoolMaxWaitTime; + } + + public int getConnectionPoolMaxLifeTime() { + return connectionPoolMaxLifeTime; + } + + public boolean isConnectionPoolKeepAlive() { + return connectionPoolKeepAlive; + } + + public boolean isOnlySpecifiedDatabase() { + return onlySpecifiedDatabase; + } + + public String getIncludeDatabaseList() { + return includeDatabaseList; + } + + public String getExcludeDatabaseList() { + return excludeDatabaseList; + } + + public boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + + public String getFunctionRules() { + return functionRules; + } + + public boolean isLowerCaseMetaNames() { + return lowerCaseMetaNames; + } + + public String getMetaNamesMapping() { + return metaNamesMapping; + } + + /** + * The catalog's properties with the {@code jdbc.} prefix resolved, unmodifiable. + * + *

It contains the password. Pass it only to code that keeps it internal. + */ + public Map getRaw() { + return raw; + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConf.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConf.java new file mode 100644 index 00000000000000..0f579f0eb88d88 --- /dev/null +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConf.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.jdbc; + +import org.apache.doris.connector.spi.ConnectorConf; +import org.apache.doris.connector.spi.ConnectorContext; + +/** + * The deployment-level settings of this plugin: one per FE, not one per catalog. + * + *

They are read from the plugin's own {@code jdbc.conf} (named after the provider), each falling + * back to the {@code fe.conf} key it used to live under, which still works. Per-catalog settings are + * the other half and live in {@link JdbcCatalogProperties}. + */ +public final class JdbcConf { + + /** Where a bare {@code driver_url} file name is resolved. */ + public static final String CONF_DRIVERS_DIR = "drivers_dir"; + + /** The fe.conf name of {@link #CONF_DRIVERS_DIR}, forwarded through the engine environment. */ + public static final String ENV_DRIVERS_DIR = "jdbc_drivers_dir"; + + /** + * Whether a SQL Server URL gets {@code encrypt=false} appended when it names no encryption. It + * exists because a driver upgrade changed that default, and a deployment may not have TLS on its + * SQL Server yet. + */ + public static final String CONF_FORCE_SQLSERVER_ENCRYPT_FALSE = "force_sqlserver_encrypt_false"; + + /** The fe.conf name of {@link #CONF_FORCE_SQLSERVER_ENCRYPT_FALSE}. */ + public static final String ENV_FORCE_SQLSERVER_ENCRYPT_FALSE = "force_sqlserver_jdbc_encrypt_false"; + + /** Engine-wide, not this connector's: the FE install root. Stays in the engine environment. */ + public static final String ENV_DORIS_HOME = "doris_home"; + + private JdbcConf() { + } + + /** The drivers directory, or null when neither the plugin conf nor fe.conf names one. */ + public static String driversDir(ConnectorContext context) { + return ConnectorConf.get(context, CONF_DRIVERS_DIR, ENV_DRIVERS_DIR, null); + } + + public static String dorisHome(ConnectorContext context) { + return context.getEnvironment().get(ENV_DORIS_HOME); + } + + public static boolean forceSqlServerEncryptFalse(ConnectorContext context) { + return Boolean.parseBoolean(ConnectorConf.get(context, CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, + ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "false")); + } +} diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java index 80d4870f02d543..4ee67840d5deff 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadata.java @@ -60,23 +60,22 @@ public class JdbcConnectorMetadata implements ConnectorMetadata, ConnectorPassth static final String COLUMNS_NAMESPACE = "jdbc.columns"; private final JdbcConnectorClient client; - private final Map properties; + private final JdbcCatalogProperties props; - public JdbcConnectorMetadata(JdbcConnectorClient client, Map properties) { + public JdbcConnectorMetadata(JdbcConnectorClient client, JdbcCatalogProperties props) { this.client = client; - this.properties = properties; + this.props = props; } private JdbcIdentifierMapper getIdentifierMapper(ConnectorSession session) { - boolean isLowerCaseMetaNames = Boolean.parseBoolean( - properties.getOrDefault( - JdbcConnectorProperties.LOWER_CASE_META_NAMES, "false")); - String metaNamesMapping = properties.getOrDefault( - JdbcConnectorProperties.META_NAMES_MAPPING, ""); + boolean isLowerCaseMetaNames = props.isLowerCaseMetaNames(); + String metaNamesMapping = props.getMetaNamesMapping(); int lowerCaseTableNames = 0; if (session != null) { String val = session.getSessionProperties().get( - JdbcConnectorProperties.LOWER_CASE_TABLE_NAMES); + // The catalog property of this name is rejected; this is the session variable + // that happens to share it. + JdbcCatalogProperties.LOWER_CASE_TABLE_NAMES); if (val != null) { try { lowerCaseTableNames = Integer.parseInt(val); @@ -120,7 +119,7 @@ public ConnectorTableSchema getTableSchema( ConnectorSession session, ConnectorTableHandle handle) { if (handle instanceof PassthroughQueryTableHandle) { return new ConnectorTableSchema("__passthrough_query__", - Collections.emptyList(), "JDBC", properties); + Collections.emptyList(), "JDBC", props.getRaw()); } JdbcTableHandle jdbcHandle = (JdbcTableHandle) handle; String dbName = jdbcHandle.getRemoteDbName(); @@ -143,7 +142,7 @@ public ConnectorTableSchema getTableSchema( null, true)); } - return new ConnectorTableSchema(tableName, columns, "JDBC", properties); + return new ConnectorTableSchema(tableName, columns, "JDBC", props.getRaw()); } @Override @@ -190,29 +189,19 @@ public org.apache.doris.thrift.TTableDescriptor buildTableDescriptor( String remoteName, int numCols, long catalogId) { org.apache.doris.thrift.TJdbcTable tJdbcTable = new org.apache.doris.thrift.TJdbcTable(); tJdbcTable.setCatalogId(catalogId); - tJdbcTable.setJdbcUrl(properties.getOrDefault(JdbcConnectorProperties.JDBC_URL, "")); - tJdbcTable.setJdbcUser(properties.getOrDefault(JdbcConnectorProperties.USER, "")); - tJdbcTable.setJdbcPassword(properties.getOrDefault(JdbcConnectorProperties.PASSWORD, "")); + tJdbcTable.setJdbcUrl(props.getJdbcUrl()); + tJdbcTable.setJdbcUser(props.getUser()); + tJdbcTable.setJdbcPassword(props.getPassword()); tJdbcTable.setJdbcTableName(remoteName); - tJdbcTable.setJdbcDriverClass(properties.getOrDefault(JdbcConnectorProperties.DRIVER_CLASS, "")); - tJdbcTable.setJdbcDriverUrl(properties.getOrDefault(JdbcConnectorProperties.DRIVER_URL, "")); + tJdbcTable.setJdbcDriverClass(props.getDriverClass()); + tJdbcTable.setJdbcDriverUrl(props.getDriverUrl()); tJdbcTable.setJdbcResourceName(""); - tJdbcTable.setJdbcDriverChecksum(properties.getOrDefault(JdbcConnectorProperties.DRIVER_CHECKSUM, "")); - tJdbcTable.setConnectionPoolMinSize(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MIN_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE)); - tJdbcTable.setConnectionPoolMaxSize(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MAX_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE)); - tJdbcTable.setConnectionPoolMaxWaitTime(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MAX_WAIT_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME)); - tJdbcTable.setConnectionPoolMaxLifeTime(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MAX_LIFE_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME)); - tJdbcTable.setConnectionPoolKeepAlive(Boolean.parseBoolean(properties.getOrDefault( - JdbcConnectorProperties.CONNECTION_POOL_KEEP_ALIVE, - String.valueOf(JdbcConnectorProperties.DEFAULT_POOL_KEEP_ALIVE)))); + tJdbcTable.setJdbcDriverChecksum(props.getDriverChecksum()); + tJdbcTable.setConnectionPoolMinSize(props.getConnectionPoolMinSize()); + tJdbcTable.setConnectionPoolMaxSize(props.getConnectionPoolMaxSize()); + tJdbcTable.setConnectionPoolMaxWaitTime(props.getConnectionPoolMaxWaitTime()); + tJdbcTable.setConnectionPoolMaxLifeTime(props.getConnectionPoolMaxLifeTime()); + tJdbcTable.setConnectionPoolKeepAlive(props.isConnectionPoolKeepAlive()); org.apache.doris.thrift.TTableDescriptor desc = new org.apache.doris.thrift.TTableDescriptor( tableId, org.apache.doris.thrift.TTableType.JDBC_TABLE, @@ -275,7 +264,7 @@ public ConnectorTableSchema getColumnsFromQuery(ConnectorSession session, String null, true)); } - return new ConnectorTableSchema("query_result", columns, "JDBC", properties); + return new ConnectorTableSchema("query_result", columns, "JDBC", props.getRaw()); } // ========= ConnectorWriteOps ========= diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProperties.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProperties.java deleted file mode 100644 index 6641eec5ed1c0b..00000000000000 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProperties.java +++ /dev/null @@ -1,104 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.jdbc; - -import java.util.Map; - -/** - * Property constants for JDBC connector configuration. - * Mirrors keys from fe-core's {@code JdbcResource} and {@code CatalogProperty} - * without taking a compile-time dependency on fe-core. - */ -public final class JdbcConnectorProperties { - - private JdbcConnectorProperties() { - } - - // -- connection -- - // -- deployment-level settings, read from this plugin's own jdbc.conf (named after - // ConnectorProvider.name()). Each falls back to the ENV_ name below it, which is the fe.conf key it - // used to live under and still works. -- - public static final String CONF_DRIVERS_DIR = "drivers_dir"; - public static final String CONF_FORCE_SQLSERVER_ENCRYPT_FALSE = "force_sqlserver_encrypt_false"; - - /** The fe.conf name of {@link #CONF_DRIVERS_DIR}, forwarded through the engine environment. */ - public static final String ENV_DRIVERS_DIR = "jdbc_drivers_dir"; - /** The fe.conf name of {@link #CONF_FORCE_SQLSERVER_ENCRYPT_FALSE}. */ - public static final String ENV_FORCE_SQLSERVER_ENCRYPT_FALSE = "force_sqlserver_jdbc_encrypt_false"; - /** Engine-wide, not this connector's: the FE install root. Stays in the engine environment. */ - public static final String ENV_DORIS_HOME = "doris_home"; - - public static final String JDBC_URL = "jdbc_url"; - public static final String USER = "user"; - public static final String PASSWORD = "password"; - public static final String DRIVER_CLASS = "driver_class"; - public static final String DRIVER_URL = "driver_url"; - public static final String TYPE = "type"; - - // -- connection pool -- - public static final String CONNECTION_POOL_MIN_SIZE = "connection_pool_min_size"; - public static final String CONNECTION_POOL_MAX_SIZE = "connection_pool_max_size"; - public static final String CONNECTION_POOL_MAX_WAIT_TIME = "connection_pool_max_wait_time"; - public static final String CONNECTION_POOL_MAX_LIFE_TIME = "connection_pool_max_life_time"; - public static final String CONNECTION_POOL_KEEP_ALIVE = "connection_pool_keep_alive"; - - // -- defaults -- - public static final int DEFAULT_POOL_MIN_SIZE = 1; - public static final int DEFAULT_POOL_MAX_SIZE = 30; - public static final int DEFAULT_POOL_MAX_WAIT_TIME = 5000; - public static final int DEFAULT_POOL_MAX_LIFE_TIME = 1800000; - public static final boolean DEFAULT_POOL_KEEP_ALIVE = false; - - // -- metadata filtering -- - public static final String ONLY_SPECIFIED_DATABASE = "only_specified_database"; - public static final String INCLUDE_DATABASE_LIST = "include_database_list"; - public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; - - // -- connectivity test -- - public static final String TEST_CONNECTION = "test_connection"; - - // -- driver -- - public static final String DRIVER_CHECKSUM = "checksum"; - - // -- type mapping options -- - public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; - public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; - - // -- function pushdown -- - public static final String FUNCTION_RULES = "function_rules"; - - // -- identifier mapping -- - public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; - public static final String META_NAMES_MAPPING = "meta_names_mapping"; - public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; - - /** - * Parse an integer property with a default value. - */ - public static int getInt(Map props, String key, int defaultVal) { - String value = props.get(key); - if (value == null || value.isEmpty()) { - return defaultVal; - } - try { - return Integer.parseInt(value.trim()); - } catch (NumberFormatException e) { - return defaultVal; - } - } -} diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProvider.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProvider.java index 03be060e17ca63..2d6c34d8f9de68 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProvider.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcConnectorProvider.java @@ -20,10 +20,7 @@ import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorProvider; -import org.apache.doris.connector.spi.DorisConnectorException; -import java.util.Arrays; -import java.util.List; import java.util.Map; /** @@ -31,12 +28,6 @@ */ public class JdbcConnectorProvider implements ConnectorProvider { - private static final List REQUIRED_PROPERTIES = Arrays.asList( - JdbcConnectorProperties.JDBC_URL, - JdbcConnectorProperties.DRIVER_URL, - JdbcConnectorProperties.DRIVER_CLASS - ); - @Override public String getType() { return "jdbc"; @@ -47,140 +38,15 @@ public Connector create(Map properties, ConnectorContext context return new JdbcDorisConnector(properties, context); } - @Override - public void validateProperties(Map properties) { - // 1. Required properties - for (String required : REQUIRED_PROPERTIES) { - String value = resolve(properties, required); - if (value == null || value.isEmpty()) { - throw new IllegalArgumentException( - "Required property '" + required + "' is missing"); - } - } - - // 1b. Mandatory, non-configurable driver_url security rule. checkProperties() runs this on - // both CREATE and ALTER CATALOG (both !isReplay), so a malicious driver_url cannot be - // introduced by either; metadata replay of existing catalogs is never affected. - JdbcDorisConnector.checkDriverUrlSecurityRule(resolve(properties, JdbcConnectorProperties.DRIVER_URL)); - - // 2. Reject deprecated lower_case_table_names - if (properties.containsKey(JdbcConnectorProperties.LOWER_CASE_TABLE_NAMES) - || properties.containsKey( - JdbcDorisConnector.JDBC_PROPERTIES_PREFIX - + JdbcConnectorProperties.LOWER_CASE_TABLE_NAMES)) { - throw new IllegalArgumentException( - "Jdbc catalog property lower_case_table_names is not supported," - + " please use lower_case_meta_names instead"); - } - - // 3. Boolean property validation - checkBooleanProperty(properties, JdbcConnectorProperties.ONLY_SPECIFIED_DATABASE); - checkBooleanProperty(properties, JdbcConnectorProperties.LOWER_CASE_META_NAMES); - checkBooleanProperty(properties, JdbcConnectorProperties.CONNECTION_POOL_KEEP_ALIVE); - checkBooleanProperty(properties, JdbcConnectorProperties.TEST_CONNECTION); - - // 4. Database list consistency: include/exclude cannot be set when only_specified_database=false - String onlySpecified = resolve(properties, JdbcConnectorProperties.ONLY_SPECIFIED_DATABASE); - if (onlySpecified == null || !onlySpecified.equalsIgnoreCase("true")) { - String includeList = resolve(properties, JdbcConnectorProperties.INCLUDE_DATABASE_LIST); - String excludeList = resolve(properties, JdbcConnectorProperties.EXCLUDE_DATABASE_LIST); - if ((includeList != null && !includeList.isEmpty()) - || (excludeList != null && !excludeList.isEmpty())) { - throw new IllegalArgumentException( - "include_database_list and exclude_database_list " - + "cannot be set when only_specified_database is false"); - } - } - - // 5. Connection pool settings - checkConnectionPoolProperties(properties); - - // 6. Validate meta_names_mapping with actual lower_case_meta_names setting. - // At runtime, JdbcConnectorMetadata builds the mapper with lower_case_meta_names - // from catalog properties and lower_case_table_names from session. We validate - // with the real lower_case_meta_names and both possible lower_case_table_names - // states to catch mapping collisions that only appear after lowercasing. - String metaNamesMapping = resolve(properties, JdbcConnectorProperties.META_NAMES_MAPPING); - if (metaNamesMapping != null && !metaNamesMapping.isEmpty()) { - boolean isLowerCaseMetaNames = Boolean.parseBoolean( - resolve(properties, JdbcConnectorProperties.LOWER_CASE_META_NAMES)); - try { - // Validate with lower_case_table_names=false - new JdbcIdentifierMapper(false, isLowerCaseMetaNames, metaNamesMapping); - // Also validate with lower_case_table_names=true since the session - // variable may enable it at runtime - new JdbcIdentifierMapper(true, isLowerCaseMetaNames, metaNamesMapping); - } catch (DorisConnectorException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - } - } - - private static void checkBooleanProperty(Map properties, String key) { - String value = resolve(properties, key); - if (value != null && !value.isEmpty() - && !value.equalsIgnoreCase("true") && !value.equalsIgnoreCase("false")) { - throw new IllegalArgumentException(key + " must be true or false"); - } - } - - private static void checkConnectionPoolProperties(Map properties) { - int minSize = resolveInt(properties, JdbcConnectorProperties.CONNECTION_POOL_MIN_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE); - int maxSize = resolveInt(properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE); - int maxWaitTime = resolveInt(properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_WAIT_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME); - int maxLifeTime = resolveInt(properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_LIFE_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME); - - if (minSize < 0) { - throw new IllegalArgumentException( - "connection_pool_min_size must be greater than or equal to 0"); - } - if (maxSize < 1) { - throw new IllegalArgumentException( - "connection_pool_max_size must be greater than or equal to 1"); - } - if (maxSize < minSize) { - throw new IllegalArgumentException( - "connection_pool_max_size must be greater than or equal to connection_pool_min_size"); - } - if (maxWaitTime < 0) { - throw new IllegalArgumentException( - "connection_pool_max_wait_time must be greater than or equal to 0"); - } - if (maxWaitTime > 30000) { - throw new IllegalArgumentException( - "connection_pool_max_wait_time must be less than or equal to 30000"); - } - if (maxLifeTime < 150000) { - throw new IllegalArgumentException( - "connection_pool_max_life_time must be greater than or equal to 150000"); - } - } - /** - * Resolve a property value, checking both the short key and the "jdbc."-prefixed key. + * Validates catalog properties at CREATE/ALTER time. Building the holder covers what a stored + * catalog must satisfy; checkCreateTimeOnlyRules covers the rest, which has only ever applied to a + * statement (see {@link JdbcCatalogProperties}). The driver_url security rule stays a separate call + * because it is enforced here and in preCreateValidation, and lives with the connector that owns it. */ - private static String resolve(Map properties, String key) { - String value = properties.get(key); - if (value == null) { - value = properties.get(JdbcDorisConnector.JDBC_PROPERTIES_PREFIX + key); - } - return value; - } - - private static int resolveInt(Map properties, String key, int defaultVal) { - String value = resolve(properties, key); - if (value == null || value.isEmpty()) { - return defaultVal; - } - try { - return Integer.parseInt(value.trim()); - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "Property '" + key + "' must be a valid integer, got: " + value); - } + @Override + public void validateProperties(Map properties) { + JdbcCatalogProperties props = JdbcCatalogProperties.of(properties).checkCreateTimeOnlyRules(); + JdbcDorisConnector.checkDriverUrlSecurityRule(props.getDriverUrl()); } } diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcDorisConnector.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcDorisConnector.java index bcc51027dae32c..52f79f8e27809f 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcDorisConnector.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcDorisConnector.java @@ -20,7 +20,6 @@ import org.apache.doris.connector.jdbc.client.JdbcConnectorClient; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorCapability; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; @@ -42,9 +41,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -58,48 +55,24 @@ public class JdbcDorisConnector implements Connector { private static final Logger LOG = LogManager.getLogger(JdbcDorisConnector.class); - private final Map properties; + private final JdbcCatalogProperties props; private final ConnectorContext context; private volatile JdbcConnectorClient client; private volatile JdbcScanPlanProvider scanPlanProvider; private volatile boolean closed; - static final String JDBC_PROPERTIES_PREFIX = "jdbc."; - public JdbcDorisConnector(Map properties, ConnectorContext context) { - Map normalized = new HashMap<>(); - // Strip "jdbc." prefix from property keys for backward compatibility. - // Users may write "jdbc.jdbc_url" or "jdbc.user" in CREATE CATALOG; - // internally we use the short form ("jdbc_url", "user"). - for (Map.Entry entry : properties.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(JDBC_PROPERTIES_PREFIX)) { - String shortKey = key.substring(JDBC_PROPERTIES_PREFIX.length()); - // Only strip if the short key is not already present - if (!properties.containsKey(shortKey)) { - normalized.put(shortKey, entry.getValue()); - continue; - } - } - normalized.put(key, entry.getValue()); - } - String rawUrl = normalized.get(JdbcConnectorProperties.JDBC_URL); - if (rawUrl != null && !rawUrl.isEmpty()) { - JdbcDbType dbType = JdbcDbType.parseFromUrl(rawUrl); - normalized.put(JdbcConnectorProperties.JDBC_URL, - JdbcUrlNormalizer.normalize(rawUrl, dbType, - Boolean.parseBoolean(ConnectorConf.get(context, - JdbcConnectorProperties.CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, - JdbcConnectorProperties.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, - "false")))); - } - this.properties = Collections.unmodifiableMap(normalized); + // The URL normalization needs a per-FE setting, which is why the holder takes it as a function + // rather than reading deployment config itself -- see JdbcCatalogProperties.of. + this.props = JdbcCatalogProperties.of(properties, + url -> JdbcUrlNormalizer.normalize(url, JdbcDbType.parseFromUrl(url), + JdbcConf.forceSqlServerEncryptFalse(context))); this.context = context; } @Override public ConnectorMetadata getMetadata(ConnectorSession session) { - return new JdbcConnectorMetadata(getOrCreateClient(), properties); + return new JdbcConnectorMetadata(getOrCreateClient(), props); } @Override @@ -131,7 +104,7 @@ public ConnectorScanPlanProvider getScanPlanProvider() { // so OceanBase Oracle mode is detected correctly JdbcDbType dbType = getOrCreateClient().getDbType(); scanPlanProvider = new JdbcScanPlanProvider( - dbType, properties, context.getCatalogId()); + dbType, props, context.getCatalogId()); } } } @@ -201,13 +174,13 @@ public ConnectorWritePlanProvider getWritePlanProvider() { // Returning a non-null provider routes jdbc writes through the unified plan-provider sink // path (PhysicalPlanTranslator.visitPhysicalConnectorTableSink). The provider builds the // TJdbcTableSink itself (P6.3-T02 / OQ-1); there is no config-bag path anymore. - return new JdbcWritePlanProvider(getOrCreateClient(), properties); + return new JdbcWritePlanProvider(getOrCreateClient(), props); } @Override public void preCreateValidation(ConnectorValidationContext context) throws Exception { // 1. Validate/resolve JDBC driver — format, whitelist, secure_path, file existence. - String driverUrl = properties.get(JdbcConnectorProperties.DRIVER_URL); + String driverUrl = props.getDriverUrl(); if (driverUrl != null && !driverUrl.isEmpty()) { // Mandatory, non-configurable security rule, enforced on catalog creation only. checkDriverUrlSecurityRule(driverUrl); @@ -215,7 +188,7 @@ public void preCreateValidation(ConnectorValidationContext context) throws Excep // 2. Compute and verify checksum. String computedChecksum = context.computeDriverChecksum(driverUrl); - String providedChecksum = context.getProperty(JdbcConnectorProperties.DRIVER_CHECKSUM); + String providedChecksum = context.getProperty(JdbcCatalogProperties.DRIVER_CHECKSUM); if (providedChecksum != null && !providedChecksum.isEmpty()) { if (!providedChecksum.equals(computedChecksum)) { throw new DorisConnectorException( @@ -224,14 +197,14 @@ public void preCreateValidation(ConnectorValidationContext context) throws Excep + ") for the driver_url."); } } else { - context.storeProperty(JdbcConnectorProperties.DRIVER_CHECKSUM, computedChecksum); + context.storeProperty(JdbcCatalogProperties.DRIVER_CHECKSUM, computedChecksum); } } // 3. Test BE→JDBC connectivity via BRPC (only when test_connection is enabled). // The connector builds the serialized payload; the engine sends it after validation. boolean testConnection = Boolean.parseBoolean( - properties.getOrDefault("test_connection", "true")); + props.getRaw().getOrDefault(JdbcCatalogProperties.TEST_CONNECTION, "true")); if (testConnection) { TTableDescriptor testThrift = buildTestTableDescriptor(context); TOdbcTableType tableType = parseOdbcType(); @@ -272,40 +245,29 @@ private JdbcConnectorClient getOrCreateClient() { } private JdbcConnectorClient createClient() { - String jdbcUrl = properties.get(JdbcConnectorProperties.JDBC_URL); + String jdbcUrl = props.getJdbcUrl(); if (jdbcUrl == null || jdbcUrl.isEmpty()) { - throw new DorisConnectorException("JDBC URL ('" + JdbcConnectorProperties.JDBC_URL + "') is required"); + throw new DorisConnectorException("JDBC URL ('" + JdbcCatalogProperties.JDBC_URL + "') is required"); } JdbcDbType dbType = JdbcDbType.parseFromUrl(jdbcUrl); - String user = properties.getOrDefault(JdbcConnectorProperties.USER, ""); - String password = properties.getOrDefault(JdbcConnectorProperties.PASSWORD, ""); - String driverUrl = resolveDriverUrl(properties.get(JdbcConnectorProperties.DRIVER_URL)); - String driverClass = properties.get(JdbcConnectorProperties.DRIVER_CLASS); - int poolMinSize = JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MIN_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE); - int poolMaxSize = JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE); - int poolMaxWaitTime = JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_WAIT_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME); - int poolMaxLifeTime = JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_LIFE_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME); - boolean onlySpecifiedDatabase = Boolean.parseBoolean( - properties.getOrDefault(JdbcConnectorProperties.ONLY_SPECIFIED_DATABASE, "false")); - boolean enableMappingVarbinary = Boolean.parseBoolean( - properties.getOrDefault(JdbcConnectorProperties.ENABLE_MAPPING_VARBINARY, "false")); - boolean enableMappingTimestampTz = Boolean.parseBoolean( - properties.getOrDefault(JdbcConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + String user = props.getUser(); + String password = props.getPassword(); + String driverUrl = resolveDriverUrl(props.getDriverUrl()); + String driverClass = props.getDriverClass(); + int poolMinSize = props.getConnectionPoolMinSize(); + int poolMaxSize = props.getConnectionPoolMaxSize(); + int poolMaxWaitTime = props.getConnectionPoolMaxWaitTime(); + int poolMaxLifeTime = props.getConnectionPoolMaxLifeTime(); + boolean onlySpecifiedDatabase = props.isOnlySpecifiedDatabase(); + boolean enableMappingVarbinary = props.isEnableMappingVarbinary(); + boolean enableMappingTimestampTz = props.isEnableMappingTimestampTz(); LOG.info("Creating JDBC connector client for dbType={}, url={}", dbType, jdbcUrl); return JdbcConnectorClient.create( dbType, context.getCatalogName(), jdbcUrl, user, password, driverUrl, driverClass, poolMinSize, poolMaxSize, poolMaxWaitTime, poolMaxLifeTime, - onlySpecifiedDatabase, properties, + onlySpecifiedDatabase, props.getRaw(), enableMappingVarbinary, enableMappingTimestampTz, context::sanitizeOutboundUrl); } @@ -339,9 +301,8 @@ private String resolveDriverUrl(String driverUrl) { } // Plain filename — resolve under the configured drivers directory. doris_home is engine-wide // rather than this connector's setting, so it keeps coming from the engine environment. - String driversDir = ConnectorConf.get(context, JdbcConnectorProperties.CONF_DRIVERS_DIR, - JdbcConnectorProperties.ENV_DRIVERS_DIR, null); - String dorisHome = context.getEnvironment().get(JdbcConnectorProperties.ENV_DORIS_HOME); + String driversDir = JdbcConf.driversDir(context); + String dorisHome = JdbcConf.dorisHome(context); if (driversDir != null && !driversDir.isEmpty()) { String newPath = driversDir + "/" + driverUrl; if (new File(newPath).exists()) { @@ -370,33 +331,23 @@ private String resolveDriverUrl(String driverUrl) { private TTableDescriptor buildTestTableDescriptor(ConnectorValidationContext context) { TJdbcTable tJdbcTable = new TJdbcTable(); tJdbcTable.setCatalogId(context.getCatalogId()); - tJdbcTable.setJdbcUrl(properties.getOrDefault(JdbcConnectorProperties.JDBC_URL, "")); - tJdbcTable.setJdbcUser(properties.getOrDefault(JdbcConnectorProperties.USER, "")); - tJdbcTable.setJdbcPassword(properties.getOrDefault(JdbcConnectorProperties.PASSWORD, "")); + tJdbcTable.setJdbcUrl(props.getJdbcUrl()); + tJdbcTable.setJdbcUser(props.getUser()); + tJdbcTable.setJdbcPassword(props.getPassword()); tJdbcTable.setJdbcTableName("test_jdbc_connection"); tJdbcTable.setJdbcDriverClass( - properties.getOrDefault(JdbcConnectorProperties.DRIVER_CLASS, "")); + props.getDriverClass()); tJdbcTable.setJdbcDriverUrl( - properties.getOrDefault(JdbcConnectorProperties.DRIVER_URL, "")); + props.getDriverUrl()); tJdbcTable.setJdbcResourceName(""); // Use the checksum that was computed/verified during driver validation. - String checksum = context.getProperty(JdbcConnectorProperties.DRIVER_CHECKSUM); + String checksum = context.getProperty(JdbcCatalogProperties.DRIVER_CHECKSUM); tJdbcTable.setJdbcDriverChecksum(checksum != null ? checksum : ""); - tJdbcTable.setConnectionPoolMinSize(JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MIN_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE)); - tJdbcTable.setConnectionPoolMaxSize(JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE)); - tJdbcTable.setConnectionPoolMaxWaitTime(JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_WAIT_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME)); - tJdbcTable.setConnectionPoolMaxLifeTime(JdbcConnectorProperties.getInt( - properties, JdbcConnectorProperties.CONNECTION_POOL_MAX_LIFE_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME)); - tJdbcTable.setConnectionPoolKeepAlive(Boolean.parseBoolean( - properties.getOrDefault(JdbcConnectorProperties.CONNECTION_POOL_KEEP_ALIVE, - String.valueOf(JdbcConnectorProperties.DEFAULT_POOL_KEEP_ALIVE)))); + tJdbcTable.setConnectionPoolMinSize(props.getConnectionPoolMinSize()); + tJdbcTable.setConnectionPoolMaxSize(props.getConnectionPoolMaxSize()); + tJdbcTable.setConnectionPoolMaxWaitTime(props.getConnectionPoolMaxWaitTime()); + tJdbcTable.setConnectionPoolMaxLifeTime(props.getConnectionPoolMaxLifeTime()); + tJdbcTable.setConnectionPoolKeepAlive(props.isConnectionPoolKeepAlive()); TTableDescriptor tTableDescriptor = new TTableDescriptor( 0, TTableType.JDBC_TABLE, 0, 0, "test_jdbc_connection", ""); tTableDescriptor.setJdbcTable(tJdbcTable); @@ -404,7 +355,7 @@ private TTableDescriptor buildTestTableDescriptor(ConnectorValidationContext con } private TOdbcTableType parseOdbcType() { - String jdbcUrl = properties.getOrDefault(JdbcConnectorProperties.JDBC_URL, ""); + String jdbcUrl = props.getJdbcUrl(); JdbcDbType dbType = JdbcDbType.parseFromUrl(jdbcUrl); try { return TOdbcTableType.valueOf(dbType.name()); @@ -414,7 +365,7 @@ private TOdbcTableType parseOdbcType() { } private String getTestQuery() { - String jdbcUrl = properties.getOrDefault(JdbcConnectorProperties.JDBC_URL, ""); + String jdbcUrl = props.getJdbcUrl(); JdbcDbType dbType = JdbcDbType.parseFromUrl(jdbcUrl); switch (dbType) { case ORACLE: diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcScanPlanProvider.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcScanPlanProvider.java index f16311cff3a78d..c752c9f10247e5 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcScanPlanProvider.java @@ -49,12 +49,12 @@ public class JdbcScanPlanProvider implements ConnectorScanPlanProvider { private static final Logger LOG = LogManager.getLogger(JdbcScanPlanProvider.class); private final JdbcDbType dbType; - private final Map catalogProperties; + private final JdbcCatalogProperties props; private final long catalogId; - public JdbcScanPlanProvider(JdbcDbType dbType, Map catalogProperties, long catalogId) { + public JdbcScanPlanProvider(JdbcDbType dbType, JdbcCatalogProperties props, long catalogId) { this.dbType = dbType; - this.catalogProperties = catalogProperties; + this.props = props; this.catalogId = catalogId; } @@ -75,8 +75,7 @@ public List planScan(ConnectorSession session, ConnectorScan // Build function pushdown config from catalog properties + session Map sessionProps = session.getSessionProperties(); - String functionRulesJson = catalogProperties.getOrDefault( - JdbcConnectorProperties.FUNCTION_RULES, ""); + String functionRulesJson = props.getFunctionRules(); boolean extFuncPushdown = Boolean.parseBoolean( sessionProps.getOrDefault("enable_ext_func_pred_pushdown", "true")); JdbcFunctionPushdownConfig functionConfig = JdbcFunctionPushdownConfig.create( @@ -91,28 +90,18 @@ public List planScan(ConnectorSession session, ConnectorScan } // Build the scan range with all JDBC connection parameters - String jdbcUrl = getProperty(JdbcConnectorProperties.JDBC_URL, ""); - String user = getProperty(JdbcConnectorProperties.USER, ""); - String password = getProperty(JdbcConnectorProperties.PASSWORD, ""); - String driverClass = getProperty(JdbcConnectorProperties.DRIVER_CLASS, ""); - String driverUrl = getProperty(JdbcConnectorProperties.DRIVER_URL, ""); - String checksum = getProperty("checksum", ""); - - int poolMinSize = JdbcConnectorProperties.getInt( - catalogProperties, JdbcConnectorProperties.CONNECTION_POOL_MIN_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE); - int poolMaxSize = JdbcConnectorProperties.getInt( - catalogProperties, JdbcConnectorProperties.CONNECTION_POOL_MAX_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE); - int poolMaxWaitTime = JdbcConnectorProperties.getInt( - catalogProperties, JdbcConnectorProperties.CONNECTION_POOL_MAX_WAIT_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME); - int poolMaxLifeTime = JdbcConnectorProperties.getInt( - catalogProperties, JdbcConnectorProperties.CONNECTION_POOL_MAX_LIFE_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME); - boolean poolKeepAlive = Boolean.parseBoolean( - getProperty(JdbcConnectorProperties.CONNECTION_POOL_KEEP_ALIVE, - String.valueOf(JdbcConnectorProperties.DEFAULT_POOL_KEEP_ALIVE))); + String jdbcUrl = props.getJdbcUrl(); + String user = props.getUser(); + String password = props.getPassword(); + String driverClass = props.getDriverClass(); + String driverUrl = props.getDriverUrl(); + String checksum = props.getDriverChecksum(); + + int poolMinSize = props.getConnectionPoolMinSize(); + int poolMaxSize = props.getConnectionPoolMaxSize(); + int poolMaxWaitTime = props.getConnectionPoolMaxWaitTime(); + int poolMaxLifeTime = props.getConnectionPoolMaxLifeTime(); + boolean poolKeepAlive = props.isConnectionPoolKeepAlive(); JdbcScanRange scanRange = new JdbcScanRange.Builder() .querySql(querySql) @@ -134,10 +123,6 @@ public List planScan(ConnectorSession session, ConnectorScan return Collections.singletonList(scanRange); } - private String getProperty(String key, String defaultValue) { - return catalogProperties.getOrDefault(key, defaultValue); - } - @Override public Map getScanNodeProperties( ConnectorSession session, @@ -151,8 +136,7 @@ public Map getScanNodeProperties( } else { JdbcTableHandle jdbcHandle = (JdbcTableHandle) handle; Map sessionProps = session.getSessionProperties(); - String functionRulesJson = catalogProperties.getOrDefault( - JdbcConnectorProperties.FUNCTION_RULES, ""); + String functionRulesJson = props.getFunctionRules(); boolean extFuncPushdown = Boolean.parseBoolean( sessionProps.getOrDefault("enable_ext_func_pred_pushdown", "true")); JdbcFunctionPushdownConfig functionConfig = JdbcFunctionPushdownConfig.create( diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java index 1350646829fcbc..32ef3185c11b1b 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/JdbcWritePlanProvider.java @@ -49,17 +49,17 @@ * {@code JdbcConnectorMetadata.getWriteConfig} (property bag) plus * {@code PluginDrivenTableSink.bindJdbcWriteSink} (Thrift assembly) — fused into * this single {@code planWrite} call (P6.3-T02 / RFC OQ-1). The connection-pool - * values are taken from {@link JdbcConnectorProperties#getInt} with the - * {@code DEFAULT_POOL_*} defaults, exactly as {@code getWriteConfig} computed them.

+ * values come off {@link JdbcCatalogProperties}, which holds the same defaults + * {@code getWriteConfig} used to compute them with.

*/ public class JdbcWritePlanProvider implements ConnectorWritePlanProvider { private final JdbcConnectorClient client; - private final Map properties; + private final JdbcCatalogProperties props; - public JdbcWritePlanProvider(JdbcConnectorClient client, Map properties) { + public JdbcWritePlanProvider(JdbcConnectorClient client, JdbcCatalogProperties props) { this.client = client; - this.properties = properties; + this.props = props; } @Override @@ -68,30 +68,21 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl String insertSql = buildInsertSql(session, jdbcHandle, handle.getColumns()); TJdbcTable tJdbcTable = new TJdbcTable(); - tJdbcTable.setJdbcUrl(properties.getOrDefault(JdbcConnectorProperties.JDBC_URL, "")); - tJdbcTable.setJdbcUser(properties.getOrDefault(JdbcConnectorProperties.USER, "")); - tJdbcTable.setJdbcPassword(properties.getOrDefault(JdbcConnectorProperties.PASSWORD, "")); - tJdbcTable.setJdbcDriverUrl(properties.getOrDefault(JdbcConnectorProperties.DRIVER_URL, "")); - tJdbcTable.setJdbcDriverClass(properties.getOrDefault(JdbcConnectorProperties.DRIVER_CLASS, "")); + tJdbcTable.setJdbcUrl(props.getJdbcUrl()); + tJdbcTable.setJdbcUser(props.getUser()); + tJdbcTable.setJdbcPassword(props.getPassword()); + tJdbcTable.setJdbcDriverUrl(props.getDriverUrl()); + tJdbcTable.setJdbcDriverClass(props.getDriverClass()); tJdbcTable.setJdbcDriverChecksum( - properties.getOrDefault(JdbcConnectorProperties.DRIVER_CHECKSUM, "")); + props.getDriverChecksum()); tJdbcTable.setJdbcTableName(jdbcHandle.getRemoteTableName()); tJdbcTable.setJdbcResourceName(""); tJdbcTable.setCatalogId(session.getCatalogId()); - tJdbcTable.setConnectionPoolMinSize(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MIN_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE)); - tJdbcTable.setConnectionPoolMaxSize(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MAX_SIZE, - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE)); - tJdbcTable.setConnectionPoolMaxWaitTime(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MAX_WAIT_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME)); - tJdbcTable.setConnectionPoolMaxLifeTime(JdbcConnectorProperties.getInt(properties, - JdbcConnectorProperties.CONNECTION_POOL_MAX_LIFE_TIME, - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME)); - tJdbcTable.setConnectionPoolKeepAlive(Boolean.parseBoolean(properties.getOrDefault( - JdbcConnectorProperties.CONNECTION_POOL_KEEP_ALIVE, "false"))); + tJdbcTable.setConnectionPoolMinSize(props.getConnectionPoolMinSize()); + tJdbcTable.setConnectionPoolMaxSize(props.getConnectionPoolMaxSize()); + tJdbcTable.setConnectionPoolMaxWaitTime(props.getConnectionPoolMaxWaitTime()); + tJdbcTable.setConnectionPoolMaxLifeTime(props.getConnectionPoolMaxLifeTime()); + tJdbcTable.setConnectionPoolKeepAlive(props.isConnectionPoolKeepAlive()); TJdbcTableSink jdbcSink = new TJdbcTableSink(); jdbcSink.setJdbcTable(tJdbcTable); @@ -139,7 +130,7 @@ private String buildInsertSql(ConnectorSession session, JdbcTableHandle jdbcHand .map(ConnectorColumn::getName) .collect(Collectors.toList()); Map colHandles = - new JdbcConnectorMetadata(client, properties).getColumnHandles(session, jdbcHandle); + new JdbcConnectorMetadata(client, props).getColumnHandles(session, jdbcHandle); Map remoteColumnNames = new HashMap<>(); for (Map.Entry entry : colHandles.entrySet()) { JdbcColumnHandle ch = (JdbcColumnHandle) entry.getValue(); diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcCatalogPropertiesTest.java new file mode 100644 index 00000000000000..2b367d82a11131 --- /dev/null +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcCatalogPropertiesTest.java @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.jdbc; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +/** + * Unit tests for {@link JdbcCatalogProperties}. + * + *

The split this file mostly exists to defend: {@code of()} enforces what a stored catalog + * must satisfy, {@code checkCreateTimeOnlyRules()} enforces what a statement must satisfy. This + * connector validated far more at CREATE than any reader ever did, so moving those rules into of() — + * which runs on every rebuild, including on an FE replaying the edit log — would make catalogs that + * work today unbuildable tomorrow. Each pair of cases below pins one rule on the correct side of that + * line. + */ +class JdbcCatalogPropertiesTest { + + private static Map minimal() { + Map m = new LinkedHashMap<>(); + m.put(JdbcCatalogProperties.JDBC_URL, "jdbc:mysql://host:3306/db"); + m.put(JdbcCatalogProperties.DRIVER_URL, "mysql-connector-j.jar"); + m.put(JdbcCatalogProperties.DRIVER_CLASS, "com.mysql.cj.jdbc.Driver"); + return m; + } + + private static Map with(String key, String value) { + Map m = minimal(); + m.put(key, value); + return m; + } + + @Test + void bindsEveryKeyAndDefaults() { + JdbcCatalogProperties p = JdbcCatalogProperties.of(minimal()); + Assertions.assertEquals("jdbc:mysql://host:3306/db", p.getJdbcUrl()); + Assertions.assertEquals("mysql-connector-j.jar", p.getDriverUrl()); + Assertions.assertEquals("com.mysql.cj.jdbc.Driver", p.getDriverClass()); + Assertions.assertEquals("", p.getUser()); + Assertions.assertEquals("", p.getPassword()); + Assertions.assertEquals("", p.getDriverChecksum()); + Assertions.assertEquals(1, p.getConnectionPoolMinSize()); + Assertions.assertEquals(30, p.getConnectionPoolMaxSize()); + Assertions.assertEquals(5000, p.getConnectionPoolMaxWaitTime()); + Assertions.assertEquals(1800000, p.getConnectionPoolMaxLifeTime()); + Assertions.assertFalse(p.isConnectionPoolKeepAlive()); + Assertions.assertFalse(p.isOnlySpecifiedDatabase()); + Assertions.assertEquals("", p.getIncludeDatabaseList()); + Assertions.assertEquals("", p.getExcludeDatabaseList()); + Assertions.assertFalse(p.isEnableMappingVarbinary()); + Assertions.assertFalse(p.isEnableMappingTimestampTz()); + Assertions.assertEquals("", p.getFunctionRules()); + Assertions.assertFalse(p.isLowerCaseMetaNames()); + Assertions.assertEquals("", p.getMetaNamesMapping()); + + Map m = minimal(); + m.put(JdbcCatalogProperties.USER, "u"); + m.put(JdbcCatalogProperties.PASSWORD, "secret-p"); + m.put(JdbcCatalogProperties.CONNECTION_POOL_MIN_SIZE, "2"); + m.put(JdbcCatalogProperties.CONNECTION_POOL_MAX_SIZE, "40"); + m.put(JdbcCatalogProperties.CONNECTION_POOL_MAX_WAIT_TIME, "6000"); + m.put(JdbcCatalogProperties.CONNECTION_POOL_MAX_LIFE_TIME, "200000"); + m.put(JdbcCatalogProperties.CONNECTION_POOL_KEEP_ALIVE, "true"); + m.put(JdbcCatalogProperties.ONLY_SPECIFIED_DATABASE, "true"); + m.put(JdbcCatalogProperties.LOWER_CASE_META_NAMES, "true"); + JdbcCatalogProperties set = JdbcCatalogProperties.of(m); + Assertions.assertEquals("u", set.getUser()); + Assertions.assertEquals("secret-p", set.getPassword()); + Assertions.assertEquals(2, set.getConnectionPoolMinSize()); + Assertions.assertEquals(40, set.getConnectionPoolMaxSize()); + Assertions.assertEquals(6000, set.getConnectionPoolMaxWaitTime()); + Assertions.assertEquals(200000, set.getConnectionPoolMaxLifeTime()); + Assertions.assertTrue(set.isConnectionPoolKeepAlive()); + Assertions.assertTrue(set.isOnlySpecifiedDatabase()); + Assertions.assertTrue(set.isLowerCaseMetaNames()); + } + + @Test + void missingJdbcUrlFailsNamingTheKey() { + Map m = minimal(); + m.remove(JdbcCatalogProperties.JDBC_URL); + Assertions.assertEquals("Required property '" + JdbcCatalogProperties.JDBC_URL + "' is missing", + Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(m)).getMessage()); + } + + // --- the of() / create-time split --- + + // The one numeric change of this migration: a malformed pool size used to be swallowed by the + // runtime readers, which fell back to the default and left the catalog usable. It is now a bound + // int, so the catalog cannot be built until ALTER CATALOG overwrites the value. CREATE and ALTER + // already refused it, so only a catalog stored by an older version can be affected. + @Test + void malformedPoolSizeIsRefusedWhenBuilding() { + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of( + with(JdbcCatalogProperties.CONNECTION_POOL_MAX_SIZE, "thirty"))); + Assertions.assertTrue(e.getMessage().contains(JdbcCatalogProperties.CONNECTION_POOL_MAX_SIZE), + "the error must name the offending key, got: " + e.getMessage()); + } + + // The pool bounds are the other half of that story: no reader has ever enforced them, so a stored + // catalog outside them runs today and must go on building. Only a statement is held to them. + @Test + void poolBoundsAreCheckedForAStatementButNotForAStoredCatalog() { + Map tooShortLifeTime = + with(JdbcCatalogProperties.CONNECTION_POOL_MAX_LIFE_TIME, "1000"); + Assertions.assertDoesNotThrow(() -> JdbcCatalogProperties.of(tooShortLifeTime), + "a stored catalog outside the bounds must still build"); + Assertions.assertTrue(Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(tooShortLifeTime).checkCreateTimeOnlyRules()) + .getMessage().contains("connection_pool_max_life_time")); + + Map maxBelowMin = minimal(); + maxBelowMin.put(JdbcCatalogProperties.CONNECTION_POOL_MIN_SIZE, "10"); + maxBelowMin.put(JdbcCatalogProperties.CONNECTION_POOL_MAX_SIZE, "5"); + Assertions.assertDoesNotThrow(() -> JdbcCatalogProperties.of(maxBelowMin)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(maxBelowMin).checkCreateTimeOnlyRules()); + } + + // Same shape for the spelled-out booleans: the binder reads anything that is not "true" as false, + // as every reader here always has, so a stored catalog spelling one "yes" keeps working; a + // statement spelling it "yes" is refused, because that is how a typo silently disables an option. + @Test + void booleanSpellingIsCheckedForAStatementButNotForAStoredCatalog() { + Map m = with(JdbcCatalogProperties.ONLY_SPECIFIED_DATABASE, "yes"); + Assertions.assertFalse(JdbcCatalogProperties.of(m).isOnlySpecifiedDatabase()); + Assertions.assertTrue(Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(m).checkCreateTimeOnlyRules()) + .getMessage().contains("must be true or false")); + } + + @Test + void createTimeRulesRejectTheDeprecatedLowerCaseTableNames() { + Map shortForm = with(JdbcCatalogProperties.LOWER_CASE_TABLE_NAMES, "1"); + Assertions.assertTrue(Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(shortForm).checkCreateTimeOnlyRules()) + .getMessage().contains("lower_case_meta_names")); + + // The prefixed spelling was rejected too, and still is: it is stripped on the way in. + Map prefixed = with( + JdbcCatalogProperties.JDBC_PROPERTIES_PREFIX + JdbcCatalogProperties.LOWER_CASE_TABLE_NAMES, "1"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(prefixed).checkCreateTimeOnlyRules()); + } + + @Test + void createTimeRulesRejectADatabaseListWithoutOnlySpecifiedDatabase() { + Map m = with(JdbcCatalogProperties.INCLUDE_DATABASE_LIST, "db1"); + Assertions.assertDoesNotThrow(() -> JdbcCatalogProperties.of(m)); + Assertions.assertTrue(Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(m).checkCreateTimeOnlyRules()) + .getMessage().contains("cannot be set when only_specified_database is false")); + + m.put(JdbcCatalogProperties.ONLY_SPECIFIED_DATABASE, "true"); + Assertions.assertDoesNotThrow( + () -> JdbcCatalogProperties.of(m).checkCreateTimeOnlyRules()); + } + + @Test + void createTimeRulesRequireTheDriverAndRejectAnUnusableNameMapping() { + Map noDriver = minimal(); + noDriver.remove(JdbcCatalogProperties.DRIVER_CLASS); + Assertions.assertDoesNotThrow(() -> JdbcCatalogProperties.of(noDriver)); + Assertions.assertTrue(Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(noDriver).checkCreateTimeOnlyRules()) + .getMessage().contains(JdbcCatalogProperties.DRIVER_CLASS)); + + // An unclosed array is the shape JdbcIdentifierMapper rejects; it is checked for a statement, + // and a catalog already storing one still builds (its own reader raises it at query time). + Map badMapping = with(JdbcCatalogProperties.META_NAMES_MAPPING, + "{\"databases\":[{\"remoteDatabase\":\"a\",\"mapping\":\"b\"}"); + Assertions.assertDoesNotThrow(() -> JdbcCatalogProperties.of(badMapping)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> JdbcCatalogProperties.of(badMapping).checkCreateTimeOnlyRules()); + } + + // --- the jdbc. prefix --- + + @Test + void theJdbcPrefixIsAcceptedAndTheShortSpellingWins() { + Map prefixed = new LinkedHashMap<>(); + prefixed.put(JdbcCatalogProperties.JDBC_PROPERTIES_PREFIX + JdbcCatalogProperties.JDBC_URL, + "jdbc:mysql://prefixed:3306/db"); + prefixed.put(JdbcCatalogProperties.JDBC_PROPERTIES_PREFIX + JdbcCatalogProperties.USER, "u"); + JdbcCatalogProperties p = JdbcCatalogProperties.of(prefixed); + Assertions.assertEquals("jdbc:mysql://prefixed:3306/db", p.getJdbcUrl()); + Assertions.assertEquals("u", p.getUser()); + + Map both = minimal(); + both.put(JdbcCatalogProperties.JDBC_PROPERTIES_PREFIX + JdbcCatalogProperties.JDBC_URL, + "jdbc:mysql://prefixed:3306/db"); + Assertions.assertEquals("jdbc:mysql://host:3306/db", + JdbcCatalogProperties.of(both).getJdbcUrl(), + "the short spelling won in both places that used to resolve these keys"); + } + + // --- URL normalization is the caller's, not this class's --- + + // The normalization depends on a per-FE setting, so a holder of per-catalog properties must not do + // it itself. The validation doors get the URL as written; the connector passes the function. + @Test + void theUrlIsNormalizedOnlyByTheCallerThatSuppliesTheFunction() { + Assertions.assertEquals("jdbc:mysql://host:3306/db", + JdbcCatalogProperties.of(minimal()).getJdbcUrl()); + Assertions.assertEquals("JDBC:MYSQL://HOST:3306/DB", + JdbcCatalogProperties.of(minimal(), url -> url.toUpperCase(Locale.ROOT)).getJdbcUrl()); + } + + // --- the three rules an of() has to obey --- + + // Guards DESIGN D3(2): the map also carries engine keys and storage keys, and ALTER CATALOG merges + // properties -- it can overwrite a key but never remove one, so refusing an unrecognized name would + // leave a catalog that no statement could repair. + @Test + void unknownKeysAreTolerated() { + Map m = minimal(); + m.put("some_future_key", "x"); + m.put(JdbcCatalogProperties.JDBC_PROPERTIES_PREFIX + "some_future_key", "y"); + m.put("type", "jdbc"); + Assertions.assertDoesNotThrow(() -> JdbcCatalogProperties.of(m).checkCreateTimeOnlyRules()); + } + + // Guards DESIGN D3(1): of() runs at CREATE, again on the merged candidate at ALTER, and once more + // every time the connector is rebuilt, so it must be a pure function of its input. + @Test + void ofIsPureAndRepeatable() { + Map m = minimal(); + m.put(JdbcCatalogProperties.JDBC_PROPERTIES_PREFIX + JdbcCatalogProperties.USER, "u"); + Map before = new LinkedHashMap<>(m); + + JdbcCatalogProperties first = JdbcCatalogProperties.of(m); + JdbcCatalogProperties second = JdbcCatalogProperties.of(m); + + Assertions.assertEquals(before, m, "of() must not mutate the caller's map"); + Assertions.assertEquals(first.getJdbcUrl(), second.getJdbcUrl()); + Assertions.assertEquals(first.getUser(), second.getUser()); + Assertions.assertEquals(first.getRaw(), second.getRaw()); + } + + // Guards DESIGN D5: toString() is what a log line renders, and the raw map behind this object holds + // the password. + @Test + void toStringMasksThePassword() { + String rendered = + JdbcCatalogProperties.of(with(JdbcCatalogProperties.PASSWORD, "secret-p")).toString(); + Assertions.assertFalse(rendered.contains("secret-p"), "got: " + rendered); + Assertions.assertTrue(rendered.contains("password=***"), "got: " + rendered); + } +} diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java index f3778acbb69b08..f0c91cdf3a409f 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorMetadataTest.java @@ -47,6 +47,11 @@ */ class JdbcConnectorMetadataTest { + private static JdbcCatalogProperties minimalCatalogProps() { + return JdbcCatalogProperties.of( + Collections.singletonMap(JdbcCatalogProperties.JDBC_URL, "jdbc:mysql://h:3306/d")); + } + private ConnectorSession sessionWithProps(Map props) { return sessionWithScope(props, ConnectorStatementScope.NONE); } @@ -185,14 +190,14 @@ private static List columnSummary(ConnectorTableSchema schema) { @Test void testSupportsCastPredicatePushdown_defaultTrue() { - JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, Collections.emptyMap()); + JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, minimalCatalogProps()); ConnectorSession session = sessionWithProps(Collections.emptyMap()); Assertions.assertTrue(metadata.supportsCastPredicatePushdown(session)); } @Test void testSupportsCastPredicatePushdown_explicitTrue() { - JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, Collections.emptyMap()); + JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, minimalCatalogProps()); Map props = new HashMap<>(); props.put("enable_jdbc_cast_predicate_push_down", "true"); ConnectorSession session = sessionWithProps(props); @@ -201,7 +206,7 @@ void testSupportsCastPredicatePushdown_explicitTrue() { @Test void testSupportsCastPredicatePushdown_explicitFalse() { - JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, Collections.emptyMap()); + JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, minimalCatalogProps()); Map props = new HashMap<>(); props.put("enable_jdbc_cast_predicate_push_down", "false"); ConnectorSession session = sessionWithProps(props); @@ -223,7 +228,7 @@ void columnHandlesAndSchemaShareOneRemoteFetchPerStatement() { // (JdbcConnectorMetadata.COLUMNS_NAMESPACE, keyed by (catalogId, db, table, queryId)) collapses them to // ONE remote fetch. MUTATION: fetching directly on each call (pre-fix) -> counter 3 -> red. CountingJdbcClient client = new CountingJdbcClient(Arrays.asList(field("id"), field("name"))); - JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, Collections.emptyMap()); + JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, minimalCatalogProps()); ConnectorSession session = sessionWithScope(Collections.emptyMap(), liveScope()); JdbcTableHandle handle = new JdbcTableHandle("db", "t"); @@ -244,7 +249,7 @@ void noneScopeFetchesColumnsEveryCall() { // the memo runs the loader on every call, byte-identical to the pre-fix always-fetch behavior. // MUTATION: memoizing under NONE -> counter 1 -> red. CountingJdbcClient client = new CountingJdbcClient(Collections.singletonList(field("id"))); - JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, Collections.emptyMap()); + JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, minimalCatalogProps()); ConnectorSession session = sessionWithProps(Collections.emptyMap()); // default scope = NONE JdbcTableHandle handle = new JdbcTableHandle("db", "t"); @@ -264,7 +269,7 @@ void getTableSchemaStableOverSharedMutatedMemoUnderMutatingTypeConversion() { // reads only names) yield byte-identical columns and reflect allowNull=true for the mutated column "d". // MUTATION: a non-idempotent field mutation, or getColumnHandles depending on the flag -> schemas diverge. MutatingJdbcClient client = new MutatingJdbcClient(Arrays.asList(field("d"), field("id"))); - JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, Collections.emptyMap()); + JdbcConnectorMetadata md = new JdbcConnectorMetadata(client, minimalCatalogProps()); ConnectorSession session = sessionWithScope(Collections.emptyMap(), liveScope()); JdbcTableHandle handle = new JdbcTableHandle("db", "t"); diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorProviderValidateTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorProviderValidateTest.java index 9e0261d4b80686..0f30f725432d31 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorProviderValidateTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcConnectorProviderValidateTest.java @@ -227,7 +227,10 @@ public void testConnectionPoolInvalidInteger() { IllegalArgumentException ex = Assertions.assertThrows( IllegalArgumentException.class, () -> provider.validateProperties(props)); - Assertions.assertTrue(ex.getMessage().contains("valid integer")); + // The binder converts the field now, so the wording comes from it; what still has to hold is + // that the error names the key at fault. + Assertions.assertTrue(ex.getMessage().contains("connection_pool_min_size"), + "got: " + ex.getMessage()); } @Test diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcDorisConnectorTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcDorisConnectorTest.java index a80226e3c5b4e1..8940d142c3f249 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcDorisConnectorTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcDorisConnectorTest.java @@ -64,6 +64,10 @@ private static Map minimalProps() { return props; } + private static JdbcCatalogProperties minimalCatalogProps() { + return JdbcCatalogProperties.of(minimalProps()); + } + @Test void testGetMetadataAfterCloseThrows() throws IOException { JdbcDorisConnector connector = new JdbcDorisConnector(minimalProps(), testContext()); @@ -189,8 +193,8 @@ void testJdbcConnectorSupportsInsertOnly() { // harmless instantiable driver_class (java.lang.Object; never cast to java.sql.Driver here) // so client creation succeeds without a live database or driver jar on the test classpath. Map props = new HashMap<>(); - props.put(JdbcConnectorProperties.JDBC_URL, "jdbc:postgresql://localhost:5432/test"); - props.put(JdbcConnectorProperties.DRIVER_CLASS, "java.lang.Object"); + props.put(JdbcCatalogProperties.JDBC_URL, "jdbc:postgresql://localhost:5432/test"); + props.put(JdbcCatalogProperties.DRIVER_CLASS, "java.lang.Object"); JdbcDorisConnector connector = new JdbcDorisConnector(props, testContext()); ConnectorWritePlanProvider writeProvider = connector.getWritePlanProvider(); Assertions.assertNotNull(writeProvider, "JDBC connector must expose a write plan provider"); @@ -204,7 +208,7 @@ void testJdbcConnectorSupportsInsertOnly() { @Test void testGetWritePlanProviderWithoutDriverClassDoesNotThrow() { - // Regression test: driver_class is optional (JdbcConnectorProperties.DRIVER_CLASS is read + // Regression test: driver_class is optional (JdbcCatalogProperties.DRIVER_CLASS is read // via a plain properties.get(), so it is null when the catalog omits it — see // JdbcDorisConnector#createClient). initializeDataSource() must not pass that null straight // to HikariConfig#setDriverClassName, which NPEs deep inside HikariCP (loadClass(null) -> @@ -214,7 +218,7 @@ void testGetWritePlanProviderWithoutDriverClassDoesNotThrow() { // HikariDataSource is lazy and only resolves the driver from the jdbcUrl at first // getConnection(), so building it here must succeed even without driver_class. Map props = new HashMap<>(); - props.put(JdbcConnectorProperties.JDBC_URL, "jdbc:postgresql://localhost:5432/test"); + props.put(JdbcCatalogProperties.JDBC_URL, "jdbc:postgresql://localhost:5432/test"); JdbcDorisConnector connector = new JdbcDorisConnector(props, testContext()); Assertions.assertDoesNotThrow(() -> { connector.getWritePlanProvider(); @@ -225,7 +229,7 @@ void testGetWritePlanProviderWithoutDriverClassDoesNotThrow() { void testBeginTransactionReturnsNoOpTransaction() { // jdbc writes are auto-committed by BE per row; beginTransaction returns a degenerate no-op // transaction so the engine's write lifecycle is uniform (single ConnectorTransaction model). - JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, minimalProps()); + JdbcConnectorMetadata metadata = new JdbcConnectorMetadata(null, minimalCatalogProps()); ConnectorTransaction txn = metadata.beginTransaction(new FixedIdSession(99L)); Assertions.assertTrue(txn instanceof NoOpConnectorTransaction, diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcScanRangeAndPropertiesTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcScanRangeAndPropertiesTest.java index 015a4330716155..6ec8064324f410 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcScanRangeAndPropertiesTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcScanRangeAndPropertiesTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.HashMap; import java.util.Map; class JdbcScanRangeAndPropertiesTest { @@ -148,60 +147,4 @@ void testConnectionPoolKeepAliveFalse() { Assertions.assertEquals("false", range.getProperties().get("connection_pool_keep_alive")); } - // === JdbcConnectorProperties.getInt === - - @Test - void testGetIntValidValue() { - Map props = new HashMap<>(); - props.put("pool_size", "42"); - Assertions.assertEquals(42, JdbcConnectorProperties.getInt(props, "pool_size", 10)); - } - - @Test - void testGetIntMissingKeyReturnsDefault() { - Map props = new HashMap<>(); - Assertions.assertEquals(10, JdbcConnectorProperties.getInt(props, "missing", 10)); - } - - @Test - void testGetIntNullValueReturnsDefault() { - Map props = new HashMap<>(); - props.put("key", null); - Assertions.assertEquals(5, JdbcConnectorProperties.getInt(props, "key", 5)); - } - - @Test - void testGetIntEmptyStringReturnsDefault() { - Map props = new HashMap<>(); - props.put("key", ""); - Assertions.assertEquals(5, JdbcConnectorProperties.getInt(props, "key", 5)); - } - - @Test - void testGetIntNonNumericReturnsDefault() { - Map props = new HashMap<>(); - props.put("key", "not_a_number"); - Assertions.assertEquals(5, JdbcConnectorProperties.getInt(props, "key", 5)); - } - - @Test - void testGetIntWithWhitespace() { - Map props = new HashMap<>(); - props.put("key", " 100 "); - Assertions.assertEquals(100, JdbcConnectorProperties.getInt(props, "key", 5)); - } - - @Test - void testGetIntNegativeValue() { - Map props = new HashMap<>(); - props.put("key", "-1"); - Assertions.assertEquals(-1, JdbcConnectorProperties.getInt(props, "key", 0)); - } - - @Test - void testGetIntZero() { - Map props = new HashMap<>(); - props.put("key", "0"); - Assertions.assertEquals(0, JdbcConnectorProperties.getInt(props, "key", 99)); - } } diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcUrlNormalizerTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcUrlNormalizerTest.java index a525ad1c88bf90..18ca471cfb32c7 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcUrlNormalizerTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcUrlNormalizerTest.java @@ -17,7 +17,6 @@ package org.apache.doris.connector.jdbc; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.junit.jupiter.api.Assertions; @@ -153,34 +152,26 @@ void theEncryptOverrideIsReadFromThePluginConfFirstThenFeConf() { // trusted: an administrator who turns the override on in jdbc.conf must not be silently // overruled by fe.conf's default, and vice versa an untouched deployment must keep reading // fe.conf exactly as before. - Assertions.assertEquals("true", ConnectorConf.get( - context(Map.of(JdbcConnectorProperties.CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, "true"), - Map.of(JdbcConnectorProperties.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "false")), - JdbcConnectorProperties.CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, - JdbcConnectorProperties.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "false")); + Assertions.assertTrue(JdbcConf.forceSqlServerEncryptFalse( + context(Map.of(JdbcConf.CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, "true"), + Map.of(JdbcConf.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "false")))); - Assertions.assertEquals("true", ConnectorConf.get( - context(Map.of(), Map.of(JdbcConnectorProperties.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "true")), - JdbcConnectorProperties.CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, - JdbcConnectorProperties.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "false")); + Assertions.assertTrue(JdbcConf.forceSqlServerEncryptFalse( + context(Map.of(), Map.of(JdbcConf.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "true")))); - Assertions.assertEquals("false", ConnectorConf.get(context(Map.of(), Map.of()), - JdbcConnectorProperties.CONF_FORCE_SQLSERVER_ENCRYPT_FALSE, - JdbcConnectorProperties.ENV_FORCE_SQLSERVER_ENCRYPT_FALSE, "false")); + Assertions.assertFalse(JdbcConf.forceSqlServerEncryptFalse(context(Map.of(), Map.of()))); } @Test void theDriversDirIsReadFromThePluginConfFirstThenFeConf() { - Assertions.assertEquals("/from/plugin/conf", ConnectorConf.get( - context(Map.of(JdbcConnectorProperties.CONF_DRIVERS_DIR, "/from/plugin/conf"), - Map.of(JdbcConnectorProperties.ENV_DRIVERS_DIR, "/from/fe/conf")), - JdbcConnectorProperties.CONF_DRIVERS_DIR, - JdbcConnectorProperties.ENV_DRIVERS_DIR, null)); - - Assertions.assertEquals("/from/fe/conf", ConnectorConf.get( - context(Map.of(), Map.of(JdbcConnectorProperties.ENV_DRIVERS_DIR, "/from/fe/conf")), - JdbcConnectorProperties.CONF_DRIVERS_DIR, - JdbcConnectorProperties.ENV_DRIVERS_DIR, null)); + // Through JdbcConf, which is what the connector calls: asserting on ConnectorConf.get with + // hand-passed keys would only prove what this test passed it. + Assertions.assertEquals("/from/plugin/conf", JdbcConf.driversDir( + context(Map.of(JdbcConf.CONF_DRIVERS_DIR, "/from/plugin/conf"), + Map.of(JdbcConf.ENV_DRIVERS_DIR, "/from/fe/conf")))); + + Assertions.assertEquals("/from/fe/conf", JdbcConf.driversDir( + context(Map.of(), Map.of(JdbcConf.ENV_DRIVERS_DIR, "/from/fe/conf")))); } @Test diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java index cd6b568ef4fb07..0f97cc95963931 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/JdbcWritePlanProviderTest.java @@ -55,6 +55,11 @@ */ class JdbcWritePlanProviderTest { + // A holder built from the minimum, so the expected pool defaults come from the one place that + // defines them rather than being copied into this file. + private static final JdbcCatalogProperties DEFAULTS = JdbcCatalogProperties.of( + Collections.singletonMap(JdbcCatalogProperties.JDBC_URL, "jdbc:mysql://h:3306/d")); + /** * Test double for {@link JdbcConnectorClient}: the protected constructor only sets * fields (no data source), so we just feed a db type and the remote columns the @@ -203,7 +208,7 @@ void planWriteBuildsJdbcTableSinkWithByteParityFields() { FakeJdbcClient client = new FakeJdbcClient( JdbcDbType.MYSQL, Arrays.asList(field("id"), field("name"))); - JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, JdbcCatalogProperties.of(props)); Map sessionProps = new HashMap<>(); sessionProps.put("enable_odbc_transcation", "true"); @@ -247,7 +252,7 @@ void appendExplainInfoEmitsConnectorWriteDetail() { FakeJdbcClient client = new FakeJdbcClient( JdbcDbType.MYSQL, Arrays.asList(field("id"), field("name"))); - JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, JdbcCatalogProperties.of(props)); Map sessionProps = new HashMap<>(); sessionProps.put("enable_odbc_transcation", "true"); @@ -271,7 +276,7 @@ void planWritePoolAndTxnFieldsFallBackToLegacyDefaultsWhenAbsent() { FakeJdbcClient client = new FakeJdbcClient( JdbcDbType.MYSQL, Collections.singletonList(field("c"))); - JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, JdbcCatalogProperties.of(props)); ConnectorWriteHandle handle = writeHandle( new JdbcTableHandle("test_db", "t1"), Collections.singletonList("c")); @@ -280,17 +285,17 @@ void planWritePoolAndTxnFieldsFallBackToLegacyDefaultsWhenAbsent() { TJdbcTableSink sink = plan.getDataSink().getJdbcTableSink(); TJdbcTable t = sink.getJdbcTable(); - // Pool values come from JdbcConnectorProperties.getInt(..., DEFAULT_*), exactly as the + // Pool values come from JdbcCatalogProperties.getInt(..., DEFAULT_*), exactly as the // legacy getWriteConfig computed them — NOT the bindJdbcWriteSink hard-coded fallbacks // (which were unreachable because the property bag always carried the computed values). Assertions.assertEquals( - JdbcConnectorProperties.DEFAULT_POOL_MIN_SIZE, t.getConnectionPoolMinSize()); + DEFAULTS.getConnectionPoolMinSize(), t.getConnectionPoolMinSize()); Assertions.assertEquals( - JdbcConnectorProperties.DEFAULT_POOL_MAX_SIZE, t.getConnectionPoolMaxSize()); + DEFAULTS.getConnectionPoolMaxSize(), t.getConnectionPoolMaxSize()); Assertions.assertEquals( - JdbcConnectorProperties.DEFAULT_POOL_MAX_WAIT_TIME, t.getConnectionPoolMaxWaitTime()); + DEFAULTS.getConnectionPoolMaxWaitTime(), t.getConnectionPoolMaxWaitTime()); Assertions.assertEquals( - JdbcConnectorProperties.DEFAULT_POOL_MAX_LIFE_TIME, t.getConnectionPoolMaxLifeTime()); + DEFAULTS.getConnectionPoolMaxLifeTime(), t.getConnectionPoolMaxLifeTime()); Assertions.assertFalse(t.isConnectionPoolKeepAlive()); // enable_odbc_transcation absent -> false (note the legacy session-key spelling preserved). Assertions.assertFalse(sink.isUseTransaction()); @@ -306,7 +311,7 @@ void explainInsertSharesOneColumnFetchWithinStatement() { props.put("jdbc_url", "jdbc:mysql://h:3306/test_db"); FakeJdbcClient client = new FakeJdbcClient( JdbcDbType.MYSQL, Arrays.asList(field("id"), field("name"))); - JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, JdbcCatalogProperties.of(props)); ConnectorSession session = session(7L, Collections.emptyMap(), liveScope()); ConnectorWriteHandle handle = writeHandle( new JdbcTableHandle("test_db", "t1"), Arrays.asList("id", "name")); @@ -339,9 +344,9 @@ void scanAndWriteShareOneColumnFetchWithinStatement() { JdbcTableHandle tableHandle = new JdbcTableHandle("test_db", "t1"); // scan path resolves column handles on the funnel metadata - new JdbcConnectorMetadata(client, props).getColumnHandles(session, tableHandle); + new JdbcConnectorMetadata(client, JdbcCatalogProperties.of(props)).getColumnHandles(session, tableHandle); // write path news up its own metadata inside buildInsertSql, yet shares the same scope entry - JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, props); + JdbcWritePlanProvider provider = new JdbcWritePlanProvider(client, JdbcCatalogProperties.of(props)); provider.planWrite(session, writeHandle(tableHandle, Arrays.asList("id", "name"))); Assertions.assertEquals(1, client.columnsFetches.get(), diff --git a/fe/fe-connector/fe-connector-maxcompute/pom.xml b/fe/fe-connector/fe-connector-maxcompute/pom.xml index f0ac826c1f8b83..8ff6cbd0e580be 100644 --- a/fe/fe-connector/fe-connector-maxcompute/pom.xml +++ b/fe/fe-connector/fe-connector-maxcompute/pom.xml @@ -45,6 +45,13 @@ under the License. ${project.version} + + + ${project.groupId} + fe-foundation + ${project.version} + + + + ${project.groupId} + fe-foundation + ${project.version} + + ${project.groupId} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java index be34d6ccf40ad2..7d724d5607e788 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java @@ -17,6 +17,13 @@ package org.apache.doris.connector.paimon; +import org.apache.doris.connector.metastore.MetaStoreProperties; +import org.apache.doris.connector.metastore.paimon.hms.PaimonHmsMetaStoreProperties; +import org.apache.doris.connector.metastore.paimon.jdbc.PaimonJdbcMetaStoreProperties; +import org.apache.doris.connector.metastore.paimon.rest.PaimonRestMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.AbstractMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.MetaStoreProviders; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -27,6 +34,7 @@ import org.apache.paimon.options.Options; import java.io.File; +import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.function.BiConsumer; @@ -58,7 +66,6 @@ public final class PaimonCatalogFactory { private static final String USER_PROPERTY_PREFIX = "paimon."; - private static final String PAIMON_REST_PROPERTY_PREFIX = "paimon.rest."; private static final String JDBC_PREFIX = "jdbc."; /** @@ -76,47 +83,37 @@ public final class PaimonCatalogFactory { private PaimonCatalogFactory() { } - /** Resolves the lower-cased flavor, defaulting to {@code filesystem}. */ - public static String resolveFlavor(Map props) { - return props.getOrDefault( - PaimonConnectorProperties.PAIMON_CATALOG_TYPE, - PaimonConnectorProperties.DEFAULT_CATALOG_TYPE).toLowerCase(Locale.ROOT); - } - - /** - * Returns the first non-blank value among the given keys, or {@code null} if none is set. - * Mirrors the alias-priority semantics of the legacy {@code @ConnectorProperty(names=...)}. - */ - public static String firstNonBlank(Map props, String... keys) { - for (String key : keys) { - String value = props.get(key); - if (StringUtils.isNotBlank(value)) { - return value; - } - } - return null; - } - /** * Builds the Paimon catalog {@link Options} for the resolved flavor. PURE: depends only on - * {@code props}. Ports {@code AbstractPaimonProperties.appendCatalogOptions()} (common) plus - * each flavor's {@code appendCustomCatalogOptions()}. + * {@code catalogProperties}. Ports {@code AbstractPaimonProperties.appendCatalogOptions()} (common) + * plus each flavor's {@code appendCustomCatalogOptions()}. */ - public static Options buildCatalogOptions(Map props) { + public static Options buildCatalogOptions(PaimonCatalogProperties catalogProperties) { Options options = new Options(); - String flavor = resolveFlavor(props); + Map props = catalogProperties.getRaw(); + String flavor = catalogProperties.getFlavor(); + // Resolve the metastore identifier BEFORE binding: an unknown flavor must keep failing with + // this factory's own "Unknown paimon.catalog.type value" message rather than the metastore + // dispatcher's "no provider supports these properties". + String metastore = metastoreIdentifier(flavor); + // Bind the flavor's typed facts and assemble from THOSE. The alias-resolved values were + // already being computed here for validation; re-scanning the raw map for assembly is what let + // a catalog validate one value and connect with another. Storage plays no part in the option + // keys, so an empty storage map is enough (the connector binds its own storage-carrying + // instance for the HiveConf). + MetaStoreProperties bound = MetaStoreProviders.bind(props, Collections.emptyMap()); - appendCommonOptions(props, options, flavor); + appendCommonOptions(props, options, metastore, ((AbstractMetaStoreProperties) bound).getWarehouse()); switch (flavor) { - case PaimonConnectorProperties.HMS: - appendHmsOptions(props, options); + case PaimonCatalogProperties.HMS: + ((PaimonHmsMetaStoreProperties) bound).toCatalogOptions().forEach(options::set); break; - case PaimonConnectorProperties.REST: - appendRestOptions(props, options); + case PaimonCatalogProperties.REST: + ((PaimonRestMetaStoreProperties) bound).toRestOptions().forEach(options::set); break; - case PaimonConnectorProperties.JDBC: - appendJdbcOptions(props, options); + case PaimonCatalogProperties.JDBC: + appendJdbcOptions(props, options, (PaimonJdbcMetaStoreProperties) bound); break; default: // filesystem: nothing custom. @@ -125,12 +122,12 @@ public static Options buildCatalogOptions(Map props) { return options; } - private static void appendCommonOptions(Map props, Options options, String flavor) { - String warehouse = props.get(PaimonConnectorProperties.WAREHOUSE); + private static void appendCommonOptions(Map props, Options options, String metastore, + String warehouse) { if (StringUtils.isNotBlank(warehouse)) { options.set(CatalogOptions.WAREHOUSE.key(), warehouse); } - options.set(CatalogOptions.METASTORE.key(), metastoreIdentifier(flavor)); + options.set(CatalogOptions.METASTORE.key(), metastore); // FIXME(cmy): Rethink these custom properties (ported from AbstractPaimonProperties). // Re-key generic paimon.* props by stripping the prefix, excluding storage prefixes which @@ -153,13 +150,13 @@ private static void appendCommonOptions(Map props, Options optio private static String metastoreIdentifier(String flavor) { switch (flavor) { - case PaimonConnectorProperties.FILESYSTEM: + case PaimonCatalogProperties.FILESYSTEM: return FileSystemCatalogFactory.IDENTIFIER; - case PaimonConnectorProperties.JDBC: + case PaimonCatalogProperties.JDBC: return JdbcCatalogFactory.IDENTIFIER; - case PaimonConnectorProperties.REST: + case PaimonCatalogProperties.REST: return "rest"; - case PaimonConnectorProperties.HMS: + case PaimonCatalogProperties.HMS: // = org.apache.paimon.hive.HiveCatalogOptions.IDENTIFIER; kept as a literal to // mirror the existing rest/jdbc style (this is a pure option string, not a type ref). return "hive"; @@ -177,38 +174,12 @@ private static boolean isStoragePrefixed(String key) { return false; } - private static void appendHmsOptions(Map props, Options options) { - String pool = props.getOrDefault( - PaimonConnectorProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, - PaimonConnectorProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS_DEFAULT); - String location = props.getOrDefault( - PaimonConnectorProperties.LOCATION_IN_PROPERTIES, - PaimonConnectorProperties.LOCATION_IN_PROPERTIES_DEFAULT); - options.set(PaimonConnectorProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, pool); - options.set(PaimonConnectorProperties.LOCATION_IN_PROPERTIES, location); - options.set("uri", firstNonBlank(props, PaimonConnectorProperties.HMS_URI)); - } - - private static void appendRestOptions(Map props, Options options) { - options.set("uri", firstNonBlank(props, PaimonConnectorProperties.REST_URI)); - props.forEach((k, v) -> { - if (k.startsWith(PAIMON_REST_PROPERTY_PREFIX)) { - options.set(k.substring(PAIMON_REST_PROPERTY_PREFIX.length()), v); - } - }); - } - - private static void appendJdbcOptions(Map props, Options options) { - options.set(CatalogOptions.URI.key(), firstNonBlank(props, PaimonConnectorProperties.JDBC_URI)); - String user = firstNonBlank(props, PaimonConnectorProperties.JDBC_USER); - if (StringUtils.isNotBlank(user)) { - options.set("jdbc.user", user); - } - String password = firstNonBlank(props, PaimonConnectorProperties.JDBC_PASSWORD); - if (StringUtils.isNotBlank(password)) { - options.set("jdbc.password", password); - } - // Pass through any raw jdbc.* key not already set (legacy appendRawJdbcCatalogOptions). + private static void appendJdbcOptions(Map props, Options options, + PaimonJdbcMetaStoreProperties jdbc) { + jdbc.toCatalogOptions().forEach(options::set); + // Pass through any raw jdbc.* key not already set (legacy appendRawJdbcCatalogOptions). This is a + // wildcard forward of keys the holder does not model, so it stays a raw read -- and it runs after + // the bound keys so an alias-resolved user/password still wins. props.forEach((k, v) -> { if (k != null && k.startsWith(JDBC_PREFIX) && !options.keySet().contains(k)) { options.set(k, v); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java new file mode 100644 index 00000000000000..2a16f1e81f67d0 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.metastore.spi.MetaStoreProviders; +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Everything a user writes in {@code CREATE CATALOG} for a paimon catalog that is not specific to one + * metastore backend (the "A class" of the connector property convention; deployment-level settings + * live in {@link PaimonConf}). The per-flavor keys — uri, credentials, warehouse — belong to the + * bound {@code *MetaStoreProperties} instead, which is why only three keys are declared here. + * + *

The two entry points are not interchangeable. {@link #of(Map)} runs at CREATE, at ALTER + * validation, AND on every connector build — including the lazy rebuild after an FE restart, which + * happens at least twice for an interactively created catalog. So it binds and derives, and it does + * not enforce anything the connector can run without: a rule added after a catalog was created must + * not stop that catalog from coming back. {@link #checkCreateTimeOnlyRules()} carries the rules that + * only ever ran against a statement, and only the provider calls it. + * + *

The flavor is bound as a plain String, not an enum. Backends are discovered through + * {@code ServiceLoader}, so the set is open by design and a central enum would have to be edited + * every time one is added. + */ +public final class PaimonCatalogProperties { + + private static final Logger LOG = LogManager.getLogger(PaimonCatalogProperties.class); + + /** Paimon catalog backend type: filesystem, hms, rest, jdbc. */ + public static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type"; + + /** + * Whether to map Paimon BINARY/VARBINARY to Doris VARBINARY instead of STRING. + * + *

Canonical (dotted) CREATE-CATALOG key, mirroring fe-core + * {@code CatalogProperty.ENABLE_MAPPING_VARBINARY} and the legacy paimon path. The connector + * receives the raw catalog property map ({@code catalogProperty.getProperties()}), which only + * ever carries this dotted key (fe-core {@code setDefaultPropsIfMissing} writes only it), so the + * read MUST use the dotted spelling — an underscore variant is never present and would read false. + */ + public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; + + /** + * Whether to map Paimon TIMESTAMP_WITH_LOCAL_TIME_ZONE to TIMESTAMPTZ. + * + *

Canonical (dotted) CREATE-CATALOG key, mirroring fe-core + * {@code CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ} and the legacy paimon path. + */ + public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; + + // ---- Flavor literals (the accepted paimon.catalog.type values) ---- + public static final String FILESYSTEM = "filesystem"; + public static final String HMS = "hms"; + public static final String REST = "rest"; + public static final String JDBC = "jdbc"; + + /** Default catalog type when not specified. */ + private static final String DEFAULT_CATALOG_TYPE = FILESYSTEM; + + // Legacy PaimonExternalCatalog.checkProperties validated the table-handle cache knobs + // (meta.cache.paimon.table.{enable,ttl-second,capacity}) via CacheSpec. FIX-4 restores ttl-second: it + // now sizes the connector latest-snapshot cache (data) AND the generic schema cache (via + // schemaCacheTtlSecondOverride). enable/capacity remain not-wired on the plugin path, so they are + // still reported as ignored (R2) -- ttl-second is intentionally excluded from this set since it again + // takes effect. + private static final String DEAD_TABLE_CACHE_PREFIX = "meta.cache.paimon.table."; + + @ConnectorProperty(names = {PAIMON_CATALOG_TYPE}, required = false, + description = "The metastore backend: filesystem, hms, rest or jdbc.") + private String catalogType = DEFAULT_CATALOG_TYPE; + + @ConnectorProperty(names = {ENABLE_MAPPING_VARBINARY}, required = false, + description = "Map paimon BINARY/VARBINARY to Doris VARBINARY instead of STRING.") + private boolean enableMappingVarbinary; + + @ConnectorProperty(names = {ENABLE_MAPPING_TIMESTAMP_TZ}, required = false, + description = "Map paimon TIMESTAMP_WITH_LOCAL_TIME_ZONE to Doris TIMESTAMPTZ.") + private boolean enableMappingTimestampTz; + + private final Map raw; + private String flavor; + + private PaimonCatalogProperties(Map properties) { + this.raw = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + /** + * Binds and derives. Pure, idempotent, and free of I/O — it runs on every connector build. + * + *

Unknown keys are never rejected: the same map carries engine keys ({@code type}, + * {@code meta.cache.*}, ...) and storage keys ({@code s3.*}, ...), and ALTER CATALOG can only + * overwrite a key, never remove one, so a rejected unknown key could not be repaired. + */ + public static PaimonCatalogProperties of(Map properties) { + PaimonCatalogProperties p = new PaimonCatalogProperties(properties); + ConnectorPropertiesUtils.bindConnectorProperties(p, properties); + p.flavor = p.catalogType.toLowerCase(Locale.ROOT); + return p; + } + + /** + * The checks that only ever ran against a CREATE/ALTER CATALOG statement, kept out of + * {@link #of(Map)} so they cannot break the rebuild of a catalog that predates them. + * + *

Order is load-bearing and matches the provider method this replaces: an invalid meta-cache + * value is rejected outright, while a valid-but-unwired enable/capacity is still reported as + * ignored; then the table options; then the backend's own rules. + * + * @return this, so the provider can call it in one expression + */ + public PaimonCatalogProperties checkCreateTimeOnlyRules() { + checkMetaCacheProperties(raw); + warnIgnoredDeadTableCacheKeys(raw); + // #65955: an unknown or unparseable paimon.table-option.* must fail the CREATE/ALTER CATALOG. + // Upstream got this from AbstractPaimonProperties.initNormalizeAndCheckProps(), which the SPI + // path no longer runs; this is that path's fail-fast hook. + PaimonTableOptions.extract(raw); + // Selects the backend by paimon.catalog.type and enforces its fail-fast rules (warehouse, uri, + // HMS kerberos forbidIf/requireIf, JDBC driver_class-when-driver_url, REST dlf-token AK/SK). + // Dispatched on the RAW map rather than the derived flavor: the filesystem provider claims an + // ABSENT catalog-type token, and resolving the default here first would change which backend an + // odd value lands on. Storage plays no part in validation, so an empty storage map is passed. + // The bound instance is deliberately not kept -- it was built without storage and must not be + // mistaken for the one the HMS catalog build needs. + MetaStoreProviders.bind(raw, Collections.emptyMap()).validate(); + return this; + } + + /** + * Byte-for-byte parity with the (deleted) legacy {@code PaimonExternalCatalog.checkProperties}: + * {@code table.enable} must be boolean, {@code table.ttl-second} must be a long ≥ -1, + * {@code table.capacity} must be a long ≥ 0. Absent keys are skipped. + */ + private static void checkMetaCacheProperties(Map properties) { + CacheSpec.checkBooleanProperty(properties.get(PaimonConnector.TABLE_CACHE_ENABLE), + PaimonConnector.TABLE_CACHE_ENABLE); + CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_TTL_SECOND), + -1L, PaimonConnector.TABLE_CACHE_TTL_SECOND); + CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_CAPACITY), + 0L, PaimonConnector.TABLE_CACHE_CAPACITY); + } + + // R2: warn (do not reject, do not strip) when a CREATE/ALTER CATALOG carries the now-dead paimon + // table-cache knobs, so the operator learns their cache tuning no longer takes effect on the plugin + // path. Deliberately NOT in of(): that runs on every lazy rebuild, which would turn a one-off notice + // into a recurring log line. + private static void warnIgnoredDeadTableCacheKeys(Map properties) { + List dead = properties.keySet().stream() + .filter(k -> k.startsWith(DEAD_TABLE_CACHE_PREFIX)) + // ttl-second is restored (FIX-4): it sizes the snapshot cache + schema cache TTL, so it is NOT dead. + .filter(k -> !k.equals(PaimonConnector.TABLE_CACHE_TTL_SECOND)) + .sorted() + .collect(Collectors.toList()); + if (!dead.isEmpty()) { + LOG.warn("Paimon catalog cache property/properties {} no longer take effect on the plugin path " + + "(the table metadata cache configuration is obsolete) and are ignored.", dead); + } + } + + /** The metastore backend, lower-cased; {@code filesystem} when the catalog does not name one. */ + public String getFlavor() { + return flavor; + } + + public boolean isEnableMappingVarbinary() { + return enableMappingVarbinary; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + + /** + * The raw catalog properties, unmodifiable. Kept because several consumers forward whole key + * NAMESPACES the connector does not model — {@code paimon.*} into the catalog options, + * {@code fs./dfs./hadoop.} into the Hadoop config, {@code jdbc.*} and {@code paimon.jni.*} into the + * BE-bound scan options. Those are copy-all passthroughs, not properties this class owns. + */ + public Map getRaw() { + return raw; + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConf.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConf.java new file mode 100644 index 00000000000000..5ddde68b8fcba6 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConf.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.spi.ConnectorConf; +import org.apache.doris.connector.spi.ConnectorContext; + +/** + * This connector's deployment-level settings — the "B class" of the connector property convention, + * as opposed to the per-catalog properties in {@link PaimonCatalogProperties}. These are read from + * the plugin's own {@code paimon.conf} (named after {@code ConnectorProvider.name()}), each falling + * back to the {@code ENV_} name below it, which is the fe.conf key it used to live under and still + * works. + * + *

Both conf keys are shared with other connectors at the fe.conf end — one + * {@code jdbc_drivers_dir} and one {@code hive_metastore_client_timeout_second} serve jdbc, iceberg + * and paimon. A plugin conf cannot express that, so a deployment that moves to these files sets the + * value in each plugin's own conf. That is the accepted cost of a per-plugin file; the fe.conf keys + * stay as the shared fallback. + */ +public final class PaimonConf { + + public static final String CONF_DRIVERS_DIR = "drivers_dir"; + public static final String CONF_METASTORE_CLIENT_TIMEOUT_SECOND = "metastore_client_timeout_second"; + + /** The fe.conf name of {@link #CONF_DRIVERS_DIR}, forwarded through the engine environment. */ + public static final String ENV_JDBC_DRIVERS_DIR = "jdbc_drivers_dir"; + /** The fe.conf name of {@link #CONF_METASTORE_CLIENT_TIMEOUT_SECOND}. */ + public static final String ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND = + "hive_metastore_client_timeout_second"; + /** Engine-wide, not this connector's: the FE install root. Stays in the engine environment. */ + public static final String ENV_DORIS_HOME = "doris_home"; + /** Legacy default when neither channel names a metastore client timeout. */ + public static final String DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND = "10"; + + private PaimonConf() { + } + + /** + * The directory a bare driver jar name resolves under, from this plugin's own {@code paimon.conf} + * or fe.conf's {@code jdbc_drivers_dir}. Null when neither names one — {@code JdbcDriverSupport} + * then falls back to {@code /plugins/jdbc_drivers}, as before. + * + *

Shared by the two call sites (FE driver registration in {@code PaimonConnector} and the + * BE-bound scan options in {@code PaimonScanPlanProvider}) so they cannot resolve a given + * {@code driver_url} differently — which is the same reason both delegate to + * {@code JdbcDriverSupport}. A null context is a direct-construction unit test, which has neither + * channel. + */ + public static String driversDir(ConnectorContext context) { + return context == null ? null + : ConnectorConf.get(context, CONF_DRIVERS_DIR, ENV_JDBC_DRIVERS_DIR, null); + } + + /** The FE install root. Engine-wide rather than this connector's, so it stays in the environment. */ + public static String dorisHome(ConnectorContext context) { + return context == null ? null : context.getEnvironment().get(ENV_DORIS_HOME); + } + + /** + * The hive metastore client socket timeout, in seconds, handed to the shared HMS parser so it can + * apply it when the catalog does not set {@code hive.metastore.client.socket.timeout} itself. + * Falls back to the legacy literal so a deployment with neither file behaves as it did. + */ + public static String metastoreClientTimeoutSecond(ConnectorContext context) { + return ConnectorConf.get(context, CONF_METASTORE_CLIENT_TIMEOUT_SECOND, + ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 2515f63c796d55..5a620f1a222448 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -19,11 +19,12 @@ import org.apache.doris.connector.cache.ConnectorMetadataCache; import org.apache.doris.connector.metastore.HmsMetaStoreProperties; +import org.apache.doris.connector.metastore.paimon.jdbc.PaimonJdbcMetaStoreProperties; +import org.apache.doris.connector.metastore.spi.AbstractHmsMetaStoreProperties; import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.metastore.spi.MetaStoreProviders; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorCapability; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorPartitionInfo; @@ -109,7 +110,7 @@ public class PaimonConnector implements Connector { // Catalog property key gating the plugin-side Kerberos authenticator (value matches AuthType.KERBEROS). private static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; - private final Map properties; + private final PaimonCatalogProperties catalogProps; private final ConnectorContext context; private volatile Catalog catalog; @@ -150,7 +151,10 @@ public class PaimonConnector implements Connector { private final Map tableOptions; public PaimonConnector(Map properties, ConnectorContext context) { - this.properties = properties; + // Construct-time BIND, not validation: of() carries only what the connector cannot run without, + // so a catalog created before a rule existed still comes back after an FE restart. The + // CREATE/ALTER-only rules live behind checkCreateTimeOnlyRules(), which only the provider calls. + this.catalogProps = PaimonCatalogProperties.of(properties); this.tableOptions = PaimonTableOptions.extractCompatible(properties); // Wrap the FE-injected context so every executeAuthenticated pins the TCCL to the plugin loader (the // paimon plugin bundles paimon-core + hadoop child-first) and, for a Kerberos catalog, runs the op @@ -177,7 +181,7 @@ private HadoopAuthenticator pluginAuthenticator() { if (!pluginAuthComputed) { synchronized (this) { if (!pluginAuthComputed) { - pluginAuth = buildPluginAuthenticator(properties, buildStorageHadoopConfig()); + pluginAuth = buildPluginAuthenticator(catalogProps.getRaw(), buildStorageHadoopConfig()); pluginAuthComputed = true; } } @@ -213,7 +217,7 @@ static HadoopAuthenticator buildPluginAuthenticator(Map properti return HadoopAuthenticator.getHadoopAuthenticator( PaimonCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig)); } - if (PaimonConnectorProperties.HMS.equals(PaimonCatalogFactory.resolveFlavor(properties))) { + if (PaimonCatalogProperties.HMS.equals(PaimonCatalogProperties.of(properties).getFlavor())) { HmsMetaStoreProperties hms = (HmsMetaStoreProperties) MetaStoreProviders.bind(properties, storageHadoopConfig); Optional spec = hms.kerberos(); @@ -252,7 +256,7 @@ private static long resolveTableCacheTtlSecond(Map properties) { public ConnectorMetadata getMetadata(ConnectorSession session) { return new PaimonConnectorMetadata( new PaimonCatalogOps.CatalogBackedPaimonCatalogOps(ensureCatalog(), tableOptions), - properties, context, schemaAtMemo, latestSnapshotCache, partitionViewCache); + catalogProps, context, schemaAtMemo, latestSnapshotCache, partitionViewCache); } /** @@ -316,7 +320,7 @@ public OptionalLong schemaCacheTtlSecondOverride() { // Restore the legacy single-knob semantics: meta.cache.paimon.table.ttl-second also governs the schema // cache (the SPI routes paimon schema to the generic schema cache keyed by schema.cache.ttl-second). So // the no-cache catalog (ttl-second=0) serves FRESH schema. Absent -> no override (engine default TTL). - String raw = properties.get(TABLE_CACHE_TTL_SECOND); + String raw = catalogProps.getRaw().get(TABLE_CACHE_TTL_SECOND); if (raw == null || raw.trim().isEmpty()) { return OptionalLong.empty(); } @@ -331,7 +335,7 @@ public OptionalLong schemaCacheTtlSecondOverride() { public ConnectorScanPlanProvider getScanPlanProvider() { // FIX-B-R2-be: inject the SAME per-catalog schemaAtMemo getMetadata uses, so the schema-evolution // dict's per-schema-id reads are memoized across scans (and shared with the B-MC2 time-travel path). - return new PaimonScanPlanProvider(properties, + return new PaimonScanPlanProvider(catalogProps, new PaimonCatalogOps.CatalogBackedPaimonCatalogOps(ensureCatalog(), tableOptions), context, schemaAtMemo); } @@ -390,8 +394,8 @@ private Catalog ensureCatalog() { } private Catalog createCatalog() { - Options options = PaimonCatalogFactory.buildCatalogOptions(properties); - String flavor = PaimonCatalogFactory.resolveFlavor(properties); + Options options = PaimonCatalogFactory.buildCatalogOptions(catalogProps); + String flavor = catalogProps.getFlavor(); // Canonical storage config from the FE-bound fe-filesystem StorageProperties (P1-T03), replacing // the legacy buildObjectStorageHadoopConfig path: object stores contribute their fs.s3a.*/fs.oss.* // /fs.cosn.*/fs.obs.* translation, and an HDFS-backed catalog contributes its hadoop.config.resources @@ -401,24 +405,26 @@ private Catalog createCatalog() { Map storageHadoopConfig = buildStorageHadoopConfig(); switch (flavor) { - case PaimonConnectorProperties.FILESYSTEM: { + case PaimonCatalogProperties.FILESYSTEM: { // filesystem carries a Hadoop Configuration for HDFS/S3 storage. - Configuration conf = PaimonCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig); + Configuration conf = PaimonCatalogFactory.buildHadoopConfiguration( + catalogProps.getRaw(), storageHadoopConfig); return createCatalogFromContext(CatalogContext.create(options, conf), flavor, "Failed to create Paimon catalog with filesystem metastore"); } - case PaimonConnectorProperties.REST: { + case PaimonCatalogProperties.REST: { // rest is Options-only (no storage Configuration; the REST server owns storage). return createCatalogFromContext(CatalogContext.create(options), flavor, "Failed to create Paimon catalog with REST metastore"); } - case PaimonConnectorProperties.JDBC: { + case PaimonCatalogProperties.JDBC: { maybeRegisterJdbcDriver(); - Configuration conf = PaimonCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig); + Configuration conf = PaimonCatalogFactory.buildHadoopConfiguration( + catalogProps.getRaw(), storageHadoopConfig); return createCatalogFromContext(CatalogContext.create(options, conf), flavor, "Failed to create Paimon catalog with JDBC metastore"); } - case PaimonConnectorProperties.HMS: { + case PaimonCatalogProperties.HMS: { // NOTE (B1/cutover-blocker P5-B7): the live metastore=hive path needs the Thrift // metastore client (org.apache.hadoop.hive.metastore.IMetaStoreClient / // HiveMetaStoreClient), which is NOT provided by this connector's compile deps @@ -435,14 +441,11 @@ private Catalog createCatalog() { // so connection-critical settings present only in that file reach the live metastore client. // Shared parser produces the neutral HiveConf overrides (P2-T03); the connector seeds the // external hive-site.xml as the BASE first, then overlays the overrides (F2 ordering). - HmsMetaStoreProperties hms = (HmsMetaStoreProperties) - MetaStoreProviders.bind(properties, storageHadoopConfig); + AbstractHmsMetaStoreProperties hms = (AbstractHmsMetaStoreProperties) + MetaStoreProviders.bind(catalogProps.getRaw(), storageHadoopConfig); HiveConf hc = PaimonCatalogFactory.assembleHiveConf( - PaimonCatalogFactory.firstNonBlank(properties, "hive.conf.resources"), - hms.toHiveConfOverrides(ConnectorConf.get(context, - PaimonConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND))); + hms.getConfResources(), + hms.toHiveConfOverrides(PaimonConf.metastoreClientTimeoutSecond(context))); return createCatalogFromContext(CatalogContext.create(options, hc), flavor, "Failed to create Paimon catalog with HMS metastore"); } @@ -506,11 +509,10 @@ private Catalog createCatalogFromContext(CatalogContext catalogContext, String f */ @Override public void preCreateValidation(ConnectorValidationContext validationContext) throws Exception { - if (!PaimonConnectorProperties.JDBC.equals(PaimonCatalogFactory.resolveFlavor(properties))) { + if (!PaimonCatalogProperties.JDBC.equals(catalogProps.getFlavor())) { return; } - String driverUrl = PaimonCatalogFactory.firstNonBlank( - properties, PaimonConnectorProperties.JDBC_DRIVER_URL); + String driverUrl = PaimonJdbcMetaStoreProperties.of(catalogProps.getRaw()).getDriverUrl(); if (StringUtils.isNotBlank(driverUrl)) { validationContext.validateAndResolveDriverPath(driverUrl); } @@ -525,14 +527,12 @@ public void preCreateValidation(ConnectorValidationContext validationContext) th * against {@code ConnectorContext.getEnvironment()}. */ private void maybeRegisterJdbcDriver() { - String driverUrl = PaimonCatalogFactory.firstNonBlank( - properties, PaimonConnectorProperties.JDBC_DRIVER_URL); + PaimonJdbcMetaStoreProperties jdbc = PaimonJdbcMetaStoreProperties.of(catalogProps.getRaw()); + String driverUrl = jdbc.getDriverUrl(); if (StringUtils.isBlank(driverUrl)) { return; } - String driverClass = PaimonCatalogFactory.firstNonBlank( - properties, PaimonConnectorProperties.JDBC_DRIVER_CLASS); - registerJdbcDriver(driverUrl, driverClass); + registerJdbcDriver(driverUrl, jdbc.getDriverClass()); LOG.info("Using dynamic JDBC driver for Paimon JDBC catalog from: {}", driverUrl); } @@ -552,8 +552,8 @@ private void maybeRegisterJdbcDriver() { */ private String resolveFullDriverUrl(String driverUrl) { return JdbcDriverSupport.resolveDriverUrl(driverUrl, - PaimonConnectorProperties.configuredDriversDir(context), - PaimonConnectorProperties.configuredDorisHome(context)); + PaimonConf.driversDir(context), + PaimonConf.dorisHome(context)); } private void registerJdbcDriver(String driverUrl, String driverClassName) { diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 6a1ad0bc4d0040..7f124db18c55fb 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -84,7 +84,7 @@ public class PaimonConnectorMetadata implements ConnectorMetadata { // for the HMS-only-props gate in createDatabase. This is the same data as // session.getCatalogProperties() (the FE injects both from one source), but using the // directly-injected map avoids depending on the session being populated and is simpler. - private final Map catalogProperties; + private final PaimonCatalogProperties catalogProperties; // FIX-B-MC2: time-travel schema-at-snapshot memo. Injected by PaimonConnector (the per-catalog, // long-lived owner) so the at-snapshot resolve hits across queries. The public 3-arg ctor gives each @@ -108,18 +108,18 @@ public class PaimonConnectorMetadata implements ConnectorMetadata { // listPartitions for its LIST/timestamp partition view. private final ConnectorMetadataCache> partitionViewCache; - public PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map properties, + public PaimonConnectorMetadata(PaimonCatalogOps catalogOps, PaimonCatalogProperties properties, ConnectorContext context) { this(catalogOps, properties, context, new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE)); } - PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map properties, + PaimonConnectorMetadata(PaimonCatalogOps catalogOps, PaimonCatalogProperties properties, ConnectorContext context, PaimonSchemaAtMemo schemaAtMemo) { this(catalogOps, properties, context, schemaAtMemo, new PaimonLatestSnapshotCache(0L, 1)); } /** Convenience ctor without the PERF-06 derived partition-view cache (null -> listPartitions always live). */ - PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map properties, + PaimonConnectorMetadata(PaimonCatalogOps catalogOps, PaimonCatalogProperties properties, ConnectorContext context, PaimonSchemaAtMemo schemaAtMemo, PaimonLatestSnapshotCache latestSnapshotCache) { this(catalogOps, properties, context, schemaAtMemo, latestSnapshotCache, null); @@ -131,7 +131,7 @@ public PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map * {@code List}, keyed by {@code (db, table, snapshotId, schemaId)}. {@code null} * for the convenience/test ctors (no cross-query derived layer -> compute directly every call). */ - PaimonConnectorMetadata(PaimonCatalogOps catalogOps, Map properties, + PaimonConnectorMetadata(PaimonCatalogOps catalogOps, PaimonCatalogProperties properties, ConnectorContext context, PaimonSchemaAtMemo schemaAtMemo, PaimonLatestSnapshotCache latestSnapshotCache, ConnectorMetadataCache> partitionViewCache) { @@ -1075,8 +1075,8 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle handle) { @Override public void createDatabase(ConnectorSession session, String dbName, Map properties) { - String flavor = PaimonCatalogFactory.resolveFlavor(catalogProperties); - if (!properties.isEmpty() && !PaimonConnectorProperties.HMS.equals(flavor)) { + String flavor = catalogProperties.getFlavor(); + if (!properties.isEmpty() && !PaimonCatalogProperties.HMS.equals(flavor)) { throw new DorisConnectorException( "Not supported: create database with properties for paimon catalog type: " + flavor); } @@ -1611,15 +1611,8 @@ private List mapFields(List fields, List pri return columns; } - private static PaimonTypeMapping.Options buildTypeMappingOptions(Map props) { - boolean binaryAsVarbinary = Boolean.parseBoolean( - props.getOrDefault( - PaimonConnectorProperties.ENABLE_MAPPING_VARBINARY, - "false")); - boolean timestampTz = Boolean.parseBoolean( - props.getOrDefault( - PaimonConnectorProperties.ENABLE_MAPPING_TIMESTAMP_TZ, - "false")); - return new PaimonTypeMapping.Options(binaryAsVarbinary, timestampTz); + private static PaimonTypeMapping.Options buildTypeMappingOptions(PaimonCatalogProperties props) { + return new PaimonTypeMapping.Options( + props.isEnableMappingVarbinary(), props.isEnableMappingTimestampTz()); } } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProperties.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProperties.java deleted file mode 100644 index e8ac1b4257b502..00000000000000 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProperties.java +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.paimon; - -import org.apache.doris.connector.spi.ConnectorConf; -import org.apache.doris.connector.spi.ConnectorContext; - -/** - * Property key constants for Paimon connector configuration, plus the accessors that read this - * connector's deployment-level settings (same shape as {@code HiveConnectorProperties.getInt} / - * {@code JdbcConnectorProperties.getInt}). Where a Doris-facing property accepts multiple - * aliases (matching the legacy fe-core {@code @ConnectorProperty(names = {...})} - * declarations), the aliases are exposed as a {@code String[]} in alias-priority - * order so {@link PaimonCatalogFactory} can resolve them with - * {@code firstNonBlank}. - */ -public final class PaimonConnectorProperties { - - // -- Deployment-level settings, read from this plugin's own paimon.conf (named after - // ConnectorProvider.name()). Each falls back to the ENV_ name below it, which is the fe.conf key it - // used to live under and still works. - // - // Both are shared with other connectors at the fe.conf end -- one jdbc_drivers_dir and one - // hive_metastore_client_timeout_second serve jdbc, iceberg and paimon. A plugin conf cannot express - // that, so a deployment that moves to these files sets the value in each plugin's own conf. That is - // the accepted cost of a per-plugin file; the fe.conf keys stay as the shared fallback. -- - public static final String CONF_DRIVERS_DIR = "drivers_dir"; - public static final String CONF_METASTORE_CLIENT_TIMEOUT_SECOND = "metastore_client_timeout_second"; - - /** The fe.conf name of {@link #CONF_DRIVERS_DIR}, forwarded through the engine environment. */ - public static final String ENV_JDBC_DRIVERS_DIR = "jdbc_drivers_dir"; - /** The fe.conf name of {@link #CONF_METASTORE_CLIENT_TIMEOUT_SECOND}. */ - public static final String ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND = - "hive_metastore_client_timeout_second"; - /** Engine-wide, not this connector's: the FE install root. Stays in the engine environment. */ - public static final String ENV_DORIS_HOME = "doris_home"; - /** Legacy default when neither channel names a metastore client timeout. */ - public static final String DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND = "10"; - - /** - * The directory a bare driver jar name resolves under, from this plugin's own {@code paimon.conf} - * or fe.conf's {@code jdbc_drivers_dir}. Null when neither names one — {@code JdbcDriverSupport} - * then falls back to {@code /plugins/jdbc_drivers}, as before. - * - *

Shared by the two call sites (FE driver registration in {@code PaimonConnector} and the - * BE-bound scan options in {@code PaimonScanPlanProvider}) so they cannot resolve a given - * {@code driver_url} differently — which is the same reason both delegate to - * {@code JdbcDriverSupport}. A null context is a direct-construction unit test, which has neither - * channel. - */ - public static String configuredDriversDir(ConnectorContext context) { - return context == null ? null - : ConnectorConf.get(context, CONF_DRIVERS_DIR, ENV_JDBC_DRIVERS_DIR, null); - } - - /** The FE install root. Engine-wide rather than this connector's, so it stays in the environment. */ - public static String configuredDorisHome(ConnectorContext context) { - return context == null ? null : context.getEnvironment().get(ENV_DORIS_HOME); - } - - /** Paimon catalog backend type: filesystem, hms, rest, jdbc. */ - public static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type"; - - /** Warehouse location for the Paimon catalog. */ - public static final String WAREHOUSE = "warehouse"; - - /** - * Whether to map Paimon BINARY/VARBINARY to Doris VARBINARY instead of STRING. - * - *

Canonical (dotted) CREATE-CATALOG key, mirroring fe-core - * {@code CatalogProperty.ENABLE_MAPPING_VARBINARY} and the legacy paimon path. The connector - * receives the raw catalog property map ({@code catalogProperty.getProperties()}), which only - * ever carries this dotted key (fe-core {@code setDefaultPropsIfMissing} writes only it), so the - * read MUST use the dotted spelling — an underscore variant is never present and would read false. - */ - public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; - - /** - * Whether to map Paimon TIMESTAMP_WITH_LOCAL_TIME_ZONE to TIMESTAMPTZ. - * - *

Canonical (dotted) CREATE-CATALOG key, mirroring fe-core - * {@code CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ} and the legacy paimon path. - */ - public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; - - /** Default catalog type when not specified. */ - public static final String DEFAULT_CATALOG_TYPE = "filesystem"; - - // ---- Flavor literals (the accepted paimon.catalog.type values) ---- - public static final String FILESYSTEM = "filesystem"; - public static final String HMS = "hms"; - public static final String REST = "rest"; - public static final String JDBC = "jdbc"; - - // ---- HMS flavor keys ---- - /** Hive metastore uri; primary key + the {@code "uri"} alias (legacy HMSBaseProperties). */ - public static final String[] HMS_URI = {"hive.metastore.uris", "uri"}; - public static final String CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS = "client-pool-cache.eviction-interval-ms"; - /** Default client-pool-cache eviction interval (ms) = 5 minutes (legacy default). */ - public static final String CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS_DEFAULT = "300000"; - public static final String LOCATION_IN_PROPERTIES = "location-in-properties"; - public static final String LOCATION_IN_PROPERTIES_DEFAULT = "false"; - - // ---- REST flavor keys ---- - public static final String[] REST_URI = {"paimon.rest.uri", "uri"}; - // REST_TOKEN_PROVIDER / REST_DLF_ACCESS_KEY_ID / REST_DLF_ACCESS_KEY_SECRET removed (P2-T03): the - // REST dlf-token requireIf is now owned by RestMetaStoreProperties (the @ConnectorProperty aliases - // live in fe-connector-metastore-spi); the connector no longer hand-checks them. - - // ---- JDBC flavor keys ---- - public static final String[] JDBC_URI = {"uri", "paimon.jdbc.uri"}; - public static final String[] JDBC_USER = {"paimon.jdbc.user", "jdbc.user"}; - public static final String[] JDBC_PASSWORD = {"paimon.jdbc.password", "jdbc.password"}; - public static final String[] JDBC_DRIVER_URL = {"paimon.jdbc.driver_url", "jdbc.driver_url"}; - public static final String[] JDBC_DRIVER_CLASS = {"paimon.jdbc.driver_class", "jdbc.driver_class"}; - - - private PaimonConnectorProperties() { - } -} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProvider.java index b34102b070d02d..7bdd8753eef9c0 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorProvider.java @@ -17,21 +17,14 @@ package org.apache.doris.connector.paimon; -import org.apache.doris.connector.cache.CacheSpec; -import org.apache.doris.connector.metastore.spi.MetaStoreProviders; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorProvider; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; /** * SPI entry point for the Paimon connector. @@ -41,15 +34,6 @@ */ public class PaimonConnectorProvider implements ConnectorProvider { - private static final Logger LOG = LogManager.getLogger(PaimonConnectorProvider.class); - - // Legacy PaimonExternalCatalog.checkProperties validated the table-handle cache knobs - // (meta.cache.paimon.table.{enable,ttl-second,capacity}) via CacheSpec. FIX-4 restores ttl-second: it now - // sizes the connector latest-snapshot cache (data) AND the generic schema cache (via - // schemaCacheTtlSecondOverride). enable/capacity remain not-wired on the plugin path, so they are still - // reported as ignored (R2) — ttl-second is intentionally excluded from this set since it again takes effect. - private static final String DEAD_TABLE_CACHE_PREFIX = "meta.cache.paimon.table."; - @Override public String getType() { return "paimon"; @@ -70,31 +54,18 @@ public Set acceptedCreateTableEngineNames() { } /** - * Validates catalog properties at CREATE CATALOG time via the shared metastore parsers (P2-T03): - * {@link MetaStoreProviders#bind} selects the backend by {@code paimon.catalog.type} and the bound - * {@code MetaStoreProperties.validate()} enforces the per-flavor fail-fast rules (warehouse, uri, - * HMS kerberos forbidIf/requireIf, DLF AK/SK + endpoint-or-region + OSS storage, JDBC - * driver_class-when-driver_url, REST dlf-token AK/SK). These restore the true-legacy - * {@code HMSBaseProperties}/{@code AliyunDLFBaseProperties}/{@code ParamRules} rules. Storage is not - * needed for validation, so an empty storage map is passed; an unknown {@code paimon.catalog.type} - * makes {@code bind} throw (no provider supports it). Throws {@link IllegalArgumentException}, which - * the caller ({@code PluginDrivenExternalCatalog.checkProperties}) wraps into a DdlException. + * Binds and validates through the typed holder. {@code of(...)} carries what the connector cannot + * run without; {@code checkCreateTimeOnlyRules()} carries the rules that only ever applied to a + * statement -- the meta-cache knobs, the dead-knob warning, the paimon table options, and the + * backend's own fail-fast rules -- which is why none of them run on the connector build path. * - *

The meta-cache knobs are validated first (restoring the legacy - * {@code PaimonExternalCatalog.checkProperties} fail-fast dropped at the SPI cutover), so a bad - * {@code meta.cache.paimon.table.*} value is rejected at CREATE/ALTER. This runs before the - * dead-knob warning: an invalid value is rejected outright, while a valid-but-unwired enable/capacity - * is still reported as ignored. + *

This also serves ALTER through the SPI default {@code validatePropertiesForUpdate}, which + * merges and calls back here. Throws {@link IllegalArgumentException}, which the caller + * ({@code PluginDrivenExternalCatalog.checkProperties}) wraps into a DdlException. */ @Override public void validateProperties(Map properties) { - checkMetaCacheProperties(properties); - warnIgnoredDeadTableCacheKeys(properties); - // #65955: an unknown or unparseable paimon.table-option.* must fail the CREATE/ALTER CATALOG. - // Upstream got this from AbstractPaimonProperties.initNormalizeAndCheckProps(), which the SPI - // path no longer runs; validateProperties is this path's fail-fast hook. - PaimonTableOptions.extract(properties); - MetaStoreProviders.bind(properties, Collections.emptyMap()).validate(); + PaimonCatalogProperties.of(properties).checkCreateTimeOnlyRules(); } @Override @@ -119,32 +90,4 @@ public void validatePropertiesForUpdate( validateProperties(candidate); } - /** - * Byte-for-byte parity with the (deleted) legacy {@code PaimonExternalCatalog.checkProperties}: - * {@code table.enable} must be boolean, {@code table.ttl-second} must be a long ≥ -1, {@code - * table.capacity} must be a long ≥ 0. Absent keys are skipped. - */ - private static void checkMetaCacheProperties(Map properties) { - CacheSpec.checkBooleanProperty(properties.get(PaimonConnector.TABLE_CACHE_ENABLE), - PaimonConnector.TABLE_CACHE_ENABLE); - CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_TTL_SECOND), - -1L, PaimonConnector.TABLE_CACHE_TTL_SECOND); - CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_CAPACITY), - 0L, PaimonConnector.TABLE_CACHE_CAPACITY); - } - - // R2: warn (do not reject, do not strip) when a CREATE/ALTER CATALOG carries the now-dead paimon - // table-cache knobs, so the operator learns their cache tuning no longer takes effect on the plugin path. - private static void warnIgnoredDeadTableCacheKeys(Map properties) { - List dead = properties.keySet().stream() - .filter(k -> k.startsWith(DEAD_TABLE_CACHE_PREFIX)) - // ttl-second is restored (FIX-4): it sizes the snapshot cache + schema cache TTL, so it is NOT dead. - .filter(k -> !k.equals(PaimonConnector.TABLE_CACHE_TTL_SECOND)) - .sorted() - .collect(Collectors.toList()); - if (!dead.isEmpty()) { - LOG.warn("Paimon catalog cache property/properties {} no longer take effect on the plugin path " - + "(the table metadata cache configuration is obsolete) and are ignored.", dead); - } - } } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java index 2e9c3f86232566..2b801ebbb7a4e2 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.paimon; +import org.apache.doris.connector.metastore.paimon.jdbc.PaimonJdbcMetaStoreProperties; import org.apache.doris.connector.metastore.spi.JdbcDriverSupport; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorSession; @@ -47,6 +48,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.paimon.CoreOptions; @@ -214,7 +216,7 @@ public class PaimonScanPlanProvider implements ConnectorScanPlanProvider { private static final String DORIS_SERIALIZED_SYSTEM_SOURCE = "doris.serialized-system-source"; private static final String DORIS_SYSTEM_TABLE_TYPE = "doris.system-table-type"; - private final Map properties; + private final PaimonCatalogProperties catalogProps; private final PaimonCatalogOps catalogOps; private final ConnectorContext context; // FIX-B-R2-be: connector-level (per-catalog, long-lived) memo of the per-committed-schema-id field @@ -230,18 +232,18 @@ public class PaimonScanPlanProvider implements ConnectorScanPlanProvider { // path (paimon never streams), so the CopyOnWriteArrayList value is only ever appended single-threaded. private final ConcurrentHashMap> scanProfileStash = new ConcurrentHashMap<>(); - public PaimonScanPlanProvider(Map properties, PaimonCatalogOps catalogOps) { - this(properties, catalogOps, null); + public PaimonScanPlanProvider(PaimonCatalogProperties catalogProps, PaimonCatalogOps catalogOps) { + this(catalogProps, catalogOps, null); } - public PaimonScanPlanProvider(Map properties, PaimonCatalogOps catalogOps, + public PaimonScanPlanProvider(PaimonCatalogProperties catalogProps, PaimonCatalogOps catalogOps, ConnectorContext context) { - this(properties, catalogOps, context, new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE)); + this(catalogProps, catalogOps, context, new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE)); } - PaimonScanPlanProvider(Map properties, PaimonCatalogOps catalogOps, + PaimonScanPlanProvider(PaimonCatalogProperties catalogProps, PaimonCatalogOps catalogOps, ConnectorContext context, PaimonSchemaAtMemo schemaAtMemo) { - this.properties = properties; + this.catalogProps = catalogProps; this.catalogOps = catalogOps; this.context = context; this.schemaAtMemo = schemaAtMemo; @@ -1708,6 +1710,9 @@ static String serializePartitionValue(DataType type, Object value, String timeZo // Package-private for direct unit testing (PaimonScanPlanProviderTest). Map getBackendPaimonOptions() { Map options = new HashMap<>(); + // Two wildcard namespaces the holder does not model, forwarded verbatim: the paimon.jni.* knobs + // below and the jdbc.* / warehouse / uri / metastore / catalog-key set further down. + Map properties = catalogProps.getRaw(); // #65332: forward the JNI IOManager options for ALL metastore flavors (mirrors upstream // PaimonScanNode.getBackendPaimonOptions returning them before the jdbc-only branch), so // non-jdbc catalogs are no longer silently stripped of the enable flag. @@ -1717,8 +1722,7 @@ Map getBackendPaimonOptions() { options.put(option, properties.get(prefixed)); } } - String metastoreType = properties.get("paimon.catalog.type"); - if (!"jdbc".equalsIgnoreCase(metastoreType)) { + if (!PaimonCatalogProperties.JDBC.equals(catalogProps.getFlavor())) { return options; } // Forward relevant JDBC catalog properties for BE's paimon-cpp reader @@ -1737,16 +1741,13 @@ Map getBackendPaimonOptions() { // BE reader accepts (PaimonJdbcDriverUtils reads both aliases): honor either alias and resolve // a bare jar name to a full file:// URL. Mirrors legacy // PaimonJdbcMetaStoreProperties.getBackendPaimonOptions (getFullDriverUrl + driver_class). - String driverUrl = PaimonCatalogFactory.firstNonBlank( - properties, PaimonConnectorProperties.JDBC_DRIVER_URL); - if (driverUrl != null) { - options.put("jdbc.driver_url", JdbcDriverSupport.resolveDriverUrl(driverUrl, - PaimonConnectorProperties.configuredDriversDir(context), - PaimonConnectorProperties.configuredDorisHome(context))); - String driverClass = PaimonCatalogFactory.firstNonBlank( - properties, PaimonConnectorProperties.JDBC_DRIVER_CLASS); - if (driverClass != null) { - options.put("jdbc.driver_class", driverClass); + PaimonJdbcMetaStoreProperties jdbc = PaimonJdbcMetaStoreProperties.of(catalogProps.getRaw()); + if (StringUtils.isNotBlank(jdbc.getDriverUrl())) { + options.put("jdbc.driver_url", JdbcDriverSupport.resolveDriverUrl(jdbc.getDriverUrl(), + PaimonConf.driversDir(context), + PaimonConf.dorisHome(context))); + if (StringUtils.isNotBlank(jdbc.getDriverClass())) { + options.put("jdbc.driver_class", jdbc.getDriverClass()); } } return options; diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java index 60effd7f40e799..30592f07c0b96a 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java @@ -79,11 +79,11 @@ public class PaimonBackendBoundTableTest { private static final DataField C2 = new DataField(1, "c2", DataTypes.INT()); private static PaimonScanPlanProvider provider() { - return new PaimonScanPlanProvider(Collections.emptyMap(), null); + return new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), null); } private static PaimonConnectorMetadata metadata(RecordingPaimonCatalogOps ops) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } // ==================== the loader itself ==================== diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBuildTableDescriptorTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBuildTableDescriptorTest.java index 453d6e54f51ce9..039de739eef717 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBuildTableDescriptorTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBuildTableDescriptorTest.java @@ -47,7 +47,7 @@ public class PaimonBuildTableDescriptorTest { @Test public void buildsHiveTableDescriptorWithAddressing() { PaimonConnectorMetadata metadata = new PaimonConnectorMetadata( - null, Collections.emptyMap(), new RecordingConnectorContext()); + null, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); long tableId = 42L; String tableName = "local_table"; diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java index 2d8e44ce26d54b..37bef131fc36ba 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java @@ -81,7 +81,7 @@ private static Map storage(String... kv) { @Test public void filesystemSetsMetastoreFilesystemAndWarehouse() { Options opts = PaimonCatalogFactory.buildCatalogOptions( - props("paimon.catalog.type", "filesystem", "warehouse", "/wh")); + PaimonCatalogProperties.of(props("paimon.catalog.type", "filesystem", "warehouse", "/wh"))); // WHY: filesystem is the default flavor; its metastore identifier selects // FileSystemCatalogFactory and the warehouse is the on-disk root. Both are load-bearing @@ -92,10 +92,10 @@ public void filesystemSetsMetastoreFilesystemAndWarehouse() { @Test public void hmsSetsHiveMetastoreUriPoolAndLocation() { - Options opts = PaimonCatalogFactory.buildCatalogOptions(props( + Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "hms", "warehouse", "/wh", - "hive.metastore.uris", "thrift://nn:9083")); + "hive.metastore.uris", "thrift://nn:9083"))); // WHY: hms maps to paimon's "hive" metastore; the legacy HMS flavor always emits the // metastore uri plus the pool-eviction + location-in-properties defaults. Dropping any @@ -109,12 +109,12 @@ public void hmsSetsHiveMetastoreUriPoolAndLocation() { @Test public void hmsAcceptsUriAliasAndOverrides() { - Options opts = PaimonCatalogFactory.buildCatalogOptions(props( + Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "hms", "warehouse", "/wh", "uri", "thrift://alias:9083", "client-pool-cache.eviction-interval-ms", "60000", - "location-in-properties", "true")); + "location-in-properties", "true"))); // WHY: the legacy HMS flavor accepts the bare "uri" alias for the metastore URI and lets // the user override the pool/location defaults. MUTATION: ignoring the alias or hardcoding @@ -126,10 +126,10 @@ public void hmsAcceptsUriAliasAndOverrides() { @Test public void restSetsMetastoreRestUriAndStripsRestPrefix() { - Options opts = PaimonCatalogFactory.buildCatalogOptions(props( + Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "rest", "paimon.rest.uri", "http://rest:8080", - "paimon.rest.token.provider", "bear")); + "paimon.rest.token.provider", "bear"))); // WHY: rest maps to the "rest" metastore; the legacy rest flavor sets "uri" from // paimon.rest.uri and re-keys every paimon.rest.* prop by stripping the prefix (so @@ -142,13 +142,13 @@ public void restSetsMetastoreRestUriAndStripsRestPrefix() { @Test public void jdbcSetsMetastoreUriUserAndRawJdbcKeys() { - Options opts = PaimonCatalogFactory.buildCatalogOptions(props( + Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "jdbc", "warehouse", "/wh", "uri", "jdbc:mysql://db:3306/meta", "paimon.jdbc.user", "alice", "jdbc.password", "secret", - "jdbc.foo", "bar")); + "jdbc.foo", "bar"))); // WHY: jdbc maps to JdbcCatalogFactory; the legacy jdbc flavor sets the CatalogOptions URI, // the jdbc.user/jdbc.password (read from either alias), and passes through any raw jdbc.* @@ -168,19 +168,19 @@ public void removedDlfCatalogTypeNoLongerBuildsOptions() { // than emit a metastore.client.class naming a class that no longer ships. MUTATION: re-adding the dlf // arm -> red. IllegalArgumentException ex = Assertions.assertThrows(IllegalArgumentException.class, - () -> PaimonCatalogFactory.buildCatalogOptions(props( + () -> PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "dlf", - "warehouse", "/wh"))); + "warehouse", "/wh")))); Assertions.assertTrue(ex.getMessage().contains("dlf"), ex.getMessage()); } @Test public void paimonPrefixPassthroughExcludesStoragePrefixes() { - Options opts = PaimonCatalogFactory.buildCatalogOptions(props( + Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "filesystem", "warehouse", "/wh", "paimon.read.batch-size", "4096", - "paimon.s3.access-key", "should-not-leak")); + "paimon.s3.access-key", "should-not-leak"))); // WHY: the legacy appendCatalogOptions re-keys generic paimon.* props by stripping the // prefix, BUT deliberately excludes storage prefixes (paimon.s3./s3a./fs.s3./fs.oss.) @@ -194,9 +194,9 @@ public void paimonPrefixPassthroughExcludesStoragePrefixes() { @Test public void restBuildOptionsOmitsBlankWarehouse() { - Options opts = PaimonCatalogFactory.buildCatalogOptions(props( + Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( "paimon.catalog.type", "rest", - "paimon.rest.uri", "http://rest:8080")); + "paimon.rest.uri", "http://rest:8080"))); // WHY: this pins option ASSEMBLY only (independent of validate, which now requires a // warehouse for rest too): the common appender sets the warehouse option only when the diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogOptionsSnapshotTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogOptionsSnapshotTest.java new file mode 100644 index 00000000000000..027b37e9e9e659 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogOptionsSnapshotTest.java @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.paimon.options.Options; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * WHOLE-MAP snapshots of {@link PaimonCatalogFactory#buildCatalogOptions}, one per flavor. + * + *

Why a snapshot and not more per-key assertions. {@link PaimonCatalogFactoryTest} asserts + * that the keys it names are right; nothing there notices a key that appears, disappears, or is spelled + * differently. The catalog Options ARE the connection: paimon silently ignores an option it does not + * recognize, so a dropped or misspelled key does not throw — it produces a catalog that connects with + * different settings than the operator asked for. These tests assert the ENTIRE map, so any drift shows + * up as a diff. + * + *

They exist because the per-flavor assembly is being folded onto the bound + * {@code *MetaStoreProperties} holders, retiring the parallel raw-map scan. Once the old path is gone + * there is nothing left to compare against, so the reference is captured HERE, before the change, and + * kept afterwards as the permanent guard that holder and assembly stay in agreement. + * + *

Each input map deliberately carries the awkward keys as well as the ordinary ones: an alias form, + * a generic {@code paimon.*} passthrough, and the three namespaces the passthrough must exclude + * ({@code paimon.s3.*} storage, {@code paimon.table-option.*} per-table, {@code paimon.jni.*} BE knob). + * + *

A leak these snapshots caught and deliberately keep. {@code paimon.catalog.type} is a + * Doris-side key, but it matches the generic {@code paimon.} passthrough like any other, so every + * catalog emits a {@code catalog.type} option paimon does not define. It is inert (paimon's + * {@code Options} is an open bag) and dropping it would change what a live catalog is built with, so it + * is pinned here rather than quietly removed while the assembly is being reworked. Cleaning it up is a + * separate, deliberate change. + */ +public class PaimonCatalogOptionsSnapshotTest { + + private static Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + /** Sorted so an assertion failure renders as a readable diff of the whole option map. */ + private static void assertOptions(Map expected, Map input) { + Options actual = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(input)); + Assertions.assertEquals(new TreeMap<>(expected), new TreeMap<>(actual.toMap())); + } + + /** + * The three excluded namespaces, appended to every input map. None of them may reach the catalog + * Options: storage keys belong in the Hadoop Configuration, table options are applied per table on + * load, and the jni namespace is a BE scanner knob forwarded by the scan-plan provider. + */ + private static Map withExcludedNamespaces(Map base) { + base.put("paimon.s3.access-key", "must-not-leak"); + base.put("paimon.table-option.scan.snapshot-id", "7"); + base.put("paimon.jni.batch-size", "2048"); + return base; + } + + @Test + public void filesystemSnapshot() { + assertOptions( + props("metastore", "filesystem", + "catalog.type", "filesystem", + "warehouse", "/wh", + "read.batch-size", "4096"), + withExcludedNamespaces(props( + "paimon.catalog.type", "filesystem", + "warehouse", "/wh", + "paimon.read.batch-size", "4096", + // Hadoop-recognized keys ride the Configuration, never the catalog Options. + "fs.defaultFS", "hdfs://nn:8020"))); + } + + /** No flavor key at all: the default flavor is filesystem. */ + @Test + public void defaultFlavorSnapshotIsFilesystem() { + assertOptions( + props("metastore", "filesystem", + "warehouse", "/wh"), + props("warehouse", "/wh")); + } + + @Test + public void hmsSnapshotWithDefaults() { + assertOptions( + props("metastore", "hive", + "catalog.type", "hms", + "warehouse", "/wh", + "uri", "thrift://nn:9083", + // Both are emitted unconditionally, at their legacy defaults when unset. + "client-pool-cache.eviction-interval-ms", "300000", + "location-in-properties", "false", + "read.batch-size", "4096"), + withExcludedNamespaces(props( + "paimon.catalog.type", "hms", + "warehouse", "/wh", + "hive.metastore.uris", "thrift://nn:9083", + "paimon.read.batch-size", "4096", + // hive.* keys feed the HiveConf, not the catalog Options. + "hive.conf.resources", "hive-site.xml"))); + } + + /** The bare {@code uri} alias plus explicit overrides of both hms defaults. */ + @Test + public void hmsSnapshotWithUriAliasAndOverrides() { + assertOptions( + props("metastore", "hive", + "catalog.type", "hms", + "warehouse", "/wh", + "uri", "thrift://alias:9083", + "client-pool-cache.eviction-interval-ms", "60000", + "location-in-properties", "true"), + props("paimon.catalog.type", "hms", + "warehouse", "/wh", + "uri", "thrift://alias:9083", + "client-pool-cache.eviction-interval-ms", "60000", + "location-in-properties", "true")); + } + + /** + * A padded metastore uri reaches the catalog Options TRIMMED, because the option now comes from the + * bound hms properties instead of a second scan of the raw map. + * + *

This is a deliberate behaviour change, and it removes an inconsistency rather than introducing + * one: the HiveConf this same catalog connects with has always been built from the bound value, so + * before this change one catalog would talk to the metastore as "thrift://nn:9083" while its paimon + * Options claimed "thrift://nn:9083 ". + */ + @Test + public void hmsSnapshotTrimsThePaddedUriLikeTheHiveConfAlwaysDid() { + assertOptions( + props("metastore", "hive", + "catalog.type", "hms", + "warehouse", "/wh", + "uri", "thrift://nn:9083", + "client-pool-cache.eviction-interval-ms", "300000", + "location-in-properties", "false"), + props("paimon.catalog.type", "hms", + "warehouse", " /wh ", + "hive.metastore.uris", " thrift://nn:9083 ")); + } + + /** + * REST emits every {@code paimon.rest.*} key TWICE: once as {@code rest.} (the generic + * {@code paimon.} passthrough in the common appender) and once as {@code } (the rest appender's + * own prefix strip). Pinned because it is surprising, not because it is desirable — the rest + * appender's strip is the one paimon actually reads. + */ + @Test + public void restSnapshotEmitsBothStrippedForms() { + assertOptions( + props("metastore", "rest", + "catalog.type", "rest", + "warehouse", "/wh", + "uri", "http://rest:8080", + "token.provider", "dlf", + "dlf.access-key-id", "ak", + "rest.uri", "http://rest:8080", + "rest.token.provider", "dlf", + "rest.dlf.access-key-id", "ak", + "read.batch-size", "4096"), + withExcludedNamespaces(props( + "paimon.catalog.type", "rest", + "warehouse", "/wh", + "paimon.rest.uri", "http://rest:8080", + "paimon.rest.token.provider", "dlf", + "paimon.rest.dlf.access-key-id", "ak", + "paimon.read.batch-size", "4096"))); + } + + /** The rest flavor also accepts the bare {@code uri} alias, which carries no prefix to strip. */ + @Test + public void restSnapshotWithUriAlias() { + assertOptions( + props("metastore", "rest", + "catalog.type", "rest", + "warehouse", "/wh", + "uri", "http://rest:8080"), + props("paimon.catalog.type", "rest", + "warehouse", "/wh", + "uri", "http://rest:8080")); + } + + /** + * The rest flavor's {@code uri} comes from the BOUND value, so it wins over the same key arriving + * through the {@code paimon.rest.} prefix strip. + * + *

Both write to {@code uri}: {@code paimon.rest.uri} strips down onto it verbatim, and the + * alias-resolved value is written last. Without that ordering a padded {@code paimon.rest.uri} would + * connect with a value {@code validate()} never saw. The generic passthrough's own {@code rest.uri} + * copy stays verbatim -- it is a wildcard forward of keys this connector does not interpret. + */ + @Test + public void restSnapshotUriComesFromTheBoundValue() { + assertOptions( + props("metastore", "rest", + "catalog.type", "rest", + "warehouse", "/wh", + "uri", "http://rest:8080", + "rest.uri", " http://rest:8080 "), + props("paimon.catalog.type", "rest", + "warehouse", " /wh ", + "paimon.rest.uri", " http://rest:8080 ")); + } + + /** + * JDBC mixes all three sources: the {@code paimon.} passthrough, the alias-resolved + * user/password, and the raw {@code jdbc.*} passthrough that fills in whatever the first two left + * unset. Both alias spellings are exercised (user via {@code paimon.jdbc.}, password via bare + * {@code jdbc.}). + */ + @Test + public void jdbcSnapshot() { + assertOptions( + props("metastore", "jdbc", + "catalog.type", "jdbc", + "warehouse", "/wh", + "uri", "jdbc:mysql://db:3306/meta", + "jdbc.user", "alice", + "jdbc.password", "secret", + "jdbc.driver_url", "mysql.jar", + "jdbc.driver_class", "com.mysql.cj.jdbc.Driver", + "jdbc.foo", "bar", + "read.batch-size", "4096"), + withExcludedNamespaces(props( + "paimon.catalog.type", "jdbc", + "warehouse", "/wh", + "uri", "jdbc:mysql://db:3306/meta", + "paimon.jdbc.user", "alice", + "jdbc.password", "secret", + "paimon.jdbc.driver_url", "mysql.jar", + "paimon.jdbc.driver_class", "com.mysql.cj.jdbc.Driver", + "jdbc.foo", "bar", + "paimon.read.batch-size", "4096"))); + } + + /** JDBC with only the {@code paimon.jdbc.uri} alias and no optional key at all. */ + @Test + public void jdbcSnapshotMinimal() { + assertOptions( + props("metastore", "jdbc", + "catalog.type", "jdbc", + "warehouse", "/wh", + "uri", "jdbc:mysql://db:3306/meta", + "jdbc.uri", "jdbc:mysql://db:3306/meta"), + props("paimon.catalog.type", "jdbc", + "warehouse", "/wh", + "paimon.jdbc.uri", "jdbc:mysql://db:3306/meta")); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogPropertiesTest.java new file mode 100644 index 00000000000000..f4712a9b585a1d --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogPropertiesTest.java @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * The connector-level catalog properties: the backend flavor and the two type-mapping switches. + * + *

The split between {@link PaimonCatalogProperties#of(Map)} and + * {@link PaimonCatalogProperties#checkCreateTimeOnlyRules()} is the thing to hold on to. {@code of()} + * runs on EVERY connector build, including the lazy rebuild after an FE restart, so anything that + * throws there turns a catalog that works today into one that cannot be reconstructed tomorrow. The + * rules that only ever ran at CREATE/ALTER stay behind the second call, which only the provider makes. + */ +public class PaimonCatalogPropertiesTest { + + private static Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + @Test + public void bindsFlavorAndBothMappingSwitches() { + PaimonCatalogProperties p = PaimonCatalogProperties.of(props( + PaimonCatalogProperties.PAIMON_CATALOG_TYPE, "hms", + PaimonCatalogProperties.ENABLE_MAPPING_VARBINARY, "true", + PaimonCatalogProperties.ENABLE_MAPPING_TIMESTAMP_TZ, "true")); + + Assertions.assertEquals(PaimonCatalogProperties.HMS, p.getFlavor()); + Assertions.assertTrue(p.isEnableMappingVarbinary()); + Assertions.assertTrue(p.isEnableMappingTimestampTz()); + } + + @Test + public void defaultsToFilesystemWithBothMappingsOff() { + PaimonCatalogProperties p = PaimonCatalogProperties.of(props("warehouse", "/wh")); + + Assertions.assertEquals(PaimonCatalogProperties.FILESYSTEM, p.getFlavor()); + Assertions.assertFalse(p.isEnableMappingVarbinary()); + Assertions.assertFalse(p.isEnableMappingTimestampTz()); + } + + /** The flavor drives a switch in four classes, so it is lower-cased once, here. */ + @Test + public void flavorIsLowerCased() { + Assertions.assertEquals(PaimonCatalogProperties.JDBC, + PaimonCatalogProperties.of(props(PaimonCatalogProperties.PAIMON_CATALOG_TYPE, "JDBC")).getFlavor()); + } + + /** + * WHY: fe-core writes only the dotted spelling of the mapping keys, and an underscore variant would + * read false and silently map the column the other way. Pinned so a "helpful" alias cannot be added + * without someone deciding what a catalog carrying both should do. + */ + @Test + public void mappingSwitchesReadOnlyTheDottedSpelling() { + PaimonCatalogProperties p = PaimonCatalogProperties.of(props( + "enable_mapping_varbinary", "true", + "enable_mapping_timestamp_tz", "true")); + + Assertions.assertFalse(p.isEnableMappingVarbinary()); + Assertions.assertFalse(p.isEnableMappingTimestampTz()); + } + + /** + * WHY (convention rule D3.2): ALTER CATALOG merges properties -- it can overwrite a key but never + * remove one -- and the same map carries engine keys and storage keys. Rejecting an unknown key + * would build a catalog nobody can repair. + */ + @Test + public void unknownKeysAreTolerated() { + Map m = props( + "type", "paimon", + "warehouse", "/wh", + "s3.endpoint", "http://minio:9000", + "some_future_key", "x"); + + Assertions.assertDoesNotThrow(() -> PaimonCatalogProperties.of(m)); + } + + /** + * WHY: of() runs on every connector build. A catalog created before a validation rule existed still + * has to come back after an FE restart, so of() must not enforce anything the connector can run + * without. Every rule that fails below is reachable only through checkCreateTimeOnlyRules(). + */ + @Test + public void ofAcceptsWhatOnlyTheCreateGateRejects() { + // No warehouse (every flavor's validate() requires one), a garbage meta-cache ttl, and an + // unparseable table option -- three separate CREATE-time rules. + Map m = props( + PaimonCatalogProperties.PAIMON_CATALOG_TYPE, "filesystem", + "meta.cache.paimon.table.ttl-second", "not-a-number", + "paimon.table-option.", "orphan"); + + PaimonCatalogProperties p = Assertions.assertDoesNotThrow(() -> PaimonCatalogProperties.of(m)); + Assertions.assertThrows(IllegalArgumentException.class, p::checkCreateTimeOnlyRules); + } + + @Test + public void createGateRejectsAnUnknownFlavor() { + PaimonCatalogProperties p = PaimonCatalogProperties.of(props( + PaimonCatalogProperties.PAIMON_CATALOG_TYPE, "dlf", "warehouse", "/wh")); + + Assertions.assertThrows(IllegalArgumentException.class, p::checkCreateTimeOnlyRules); + } + + @Test + public void createGateAcceptsAValidCatalog() { + PaimonCatalogProperties p = PaimonCatalogProperties.of(props( + PaimonCatalogProperties.PAIMON_CATALOG_TYPE, "hms", + "warehouse", "/wh", + "hive.metastore.uris", "thrift://nn:9083")); + + Assertions.assertDoesNotThrow(p::checkCreateTimeOnlyRules); + } + + /** + * WHY (convention rule D3.1): of() runs at CREATE, at ALTER validation and on every rebuild, so it + * has to be pure and idempotent. Re-binding the same map must give the same answer. + */ + @Test + public void ofIsIdempotent() { + Map m = props( + PaimonCatalogProperties.PAIMON_CATALOG_TYPE, "rest", + PaimonCatalogProperties.ENABLE_MAPPING_VARBINARY, "true"); + + Assertions.assertEquals(PaimonCatalogProperties.of(m).getFlavor(), + PaimonCatalogProperties.of(m).getFlavor()); + Assertions.assertEquals(PaimonCatalogProperties.of(m).isEnableMappingVarbinary(), + PaimonCatalogProperties.of(m).isEnableMappingVarbinary()); + Assertions.assertEquals(m, PaimonCatalogProperties.of(m).getRaw()); + } + + /** The raw map stays available for the wildcard passthroughs, but must not be writable through it. */ + @Test + public void rawMapIsNotWritable() { + PaimonCatalogProperties p = PaimonCatalogProperties.of(props("warehouse", "/wh")); + + Assertions.assertThrows(UnsupportedOperationException.class, () -> p.getRaw().put("k", "v")); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorConfTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorConfTest.java index 01af5258223bc6..c313c3a5b4b06d 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorConfTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorConfTest.java @@ -17,7 +17,6 @@ package org.apache.doris.connector.paimon; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.junit.jupiter.api.Assertions; @@ -40,18 +39,18 @@ public class PaimonConnectorConfTest { @Test public void driversDirPrefersThePluginConfThenFeConf() { - Assertions.assertEquals("/from/plugin/conf", PaimonConnectorProperties.configuredDriversDir( - context(Collections.singletonMap(PaimonConnectorProperties.CONF_DRIVERS_DIR, + Assertions.assertEquals("/from/plugin/conf", PaimonConf.driversDir( + context(Collections.singletonMap(PaimonConf.CONF_DRIVERS_DIR, "/from/plugin/conf"), - Collections.singletonMap(PaimonConnectorProperties.ENV_JDBC_DRIVERS_DIR, + Collections.singletonMap(PaimonConf.ENV_JDBC_DRIVERS_DIR, "/from/fe/conf")))); - Assertions.assertEquals("/from/fe/conf", PaimonConnectorProperties.configuredDriversDir( + Assertions.assertEquals("/from/fe/conf", PaimonConf.driversDir( context(Collections.emptyMap(), - Collections.singletonMap(PaimonConnectorProperties.ENV_JDBC_DRIVERS_DIR, + Collections.singletonMap(PaimonConf.ENV_JDBC_DRIVERS_DIR, "/from/fe/conf")))); - Assertions.assertNull(PaimonConnectorProperties.configuredDriversDir( + Assertions.assertNull(PaimonConf.driversDir( context(Collections.emptyMap(), Collections.emptyMap()))); } @@ -59,35 +58,29 @@ public void driversDirPrefersThePluginConfThenFeConf() { public void nullContextResolvesToNothingRatherThanThrowing() { // Both accessors are reached from direct-construction unit tests that pass no context at all // (PaimonConnector.resolveFullDriverUrl / PaimonScanPlanProvider both null-check today). - Assertions.assertNull(PaimonConnectorProperties.configuredDriversDir(null)); - Assertions.assertNull(PaimonConnectorProperties.configuredDorisHome(null)); + Assertions.assertNull(PaimonConf.driversDir(null)); + Assertions.assertNull(PaimonConf.dorisHome(null)); } @Test public void metastoreTimeoutPrefersThePluginConfThenFeConfThenTen() { - Assertions.assertEquals("30", ConnectorConf.get( + // Through the production reader, not a re-spelling of it: the call site used to inline the + // three-argument ConnectorConf.get, so a test that inlined it too would still pass if the + // connector started reading a different key. + Assertions.assertEquals("30", PaimonConf.metastoreClientTimeoutSecond( context(Collections.singletonMap( - PaimonConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, "30"), + PaimonConf.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, "30"), Collections.singletonMap( - PaimonConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")), - PaimonConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND)); + PaimonConf.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")))); - Assertions.assertEquals("20", ConnectorConf.get( + Assertions.assertEquals("20", PaimonConf.metastoreClientTimeoutSecond( context(Collections.emptyMap(), Collections.singletonMap( - PaimonConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")), - PaimonConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND)); + PaimonConf.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, "20")))); // The literal the call site used before this channel existed; keeping it is what makes a // deployment with neither file behave exactly as it did. - Assertions.assertEquals("10", ConnectorConf.get( - context(Collections.emptyMap(), Collections.emptyMap()), - PaimonConnectorProperties.CONF_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.ENV_HIVE_METASTORE_CLIENT_TIMEOUT_SECOND, - PaimonConnectorProperties.DEFAULT_METASTORE_CLIENT_TIMEOUT_SECOND)); + Assertions.assertEquals("10", PaimonConf.metastoreClientTimeoutSecond( + context(Collections.emptyMap(), Collections.emptyMap()))); } @Test diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDbDdlTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDbDdlTest.java index a3577dedd3ae9d..e0790ea8d58c56 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDbDdlTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDbDdlTest.java @@ -44,15 +44,15 @@ public class PaimonConnectorMetadataDbDdlTest { /** Metadata with default (filesystem) flavor: catalogProperties has no paimon.catalog.type. */ private static PaimonConnectorMetadata filesystemMetadata(RecordingPaimonCatalogOps ops, RecordingConnectorContext ctx) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), ctx); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), ctx); } /** Metadata with HMS flavor: catalogProperties carries paimon.catalog.type=hms. */ private static PaimonConnectorMetadata hmsMetadata(RecordingPaimonCatalogOps ops, RecordingConnectorContext ctx) { Map catalogProps = new HashMap<>(); - catalogProps.put(PaimonConnectorProperties.PAIMON_CATALOG_TYPE, PaimonConnectorProperties.HMS); - return new PaimonConnectorMetadata(ops, catalogProps, ctx); + catalogProps.put(PaimonCatalogProperties.PAIMON_CATALOG_TYPE, PaimonCatalogProperties.HMS); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(catalogProps), ctx); } private static Map dbProps() { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDdlTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDdlTest.java index 87af1137c2ee84..e3c545f550a347 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDdlTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataDdlTest.java @@ -48,7 +48,7 @@ public class PaimonConnectorMetadataDdlTest { private static PaimonConnectorMetadata metadata(RecordingPaimonCatalogOps ops, RecordingConnectorContext ctx) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), ctx); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), ctx); } /** Builds a CREATE TABLE request: db1.t1, columns (id INT, name STRING), partitioned by id, ifNotExists. */ diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java index f702c51c0e77ce..6c27125af2b26a 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataMvccTest.java @@ -58,7 +58,7 @@ public class PaimonConnectorMetadataMvccTest { private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } // Builds a metadata sharing an EXTERNAL PaimonSchemaAtMemo, modelling two queries (each a fresh @@ -66,7 +66,7 @@ private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps op private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops, PaimonSchemaAtMemo memo) { return new PaimonConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext(), memo); + ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext(), memo); } private static RowType rowType(String... columnNames) { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataOptionsTagPinTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataOptionsTagPinTest.java index 579798580b77e2..c97fb21a7d291e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataOptionsTagPinTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataOptionsTagPinTest.java @@ -161,7 +161,7 @@ private static List readSnapshotFileSeamCalls(RecordingPaimonCatalogOps } private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } /** A normal (non-sys) handle whose transient paimon Table is the REAL local table. */ diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java index 6bc5cc0485bbb2..56d0456f8747de 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionTest.java @@ -65,7 +65,7 @@ public class PaimonConnectorMetadataPartitionTest { private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops) { // Read-path tests ignore the context; a default RecordingConnectorContext is a no-op wrapper. - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } /** Two-key partitioned table: dt (DATE) + region (STRING). */ @@ -444,7 +444,7 @@ public void catalogReaderPolicyIsPreservedDuringPartitionEnumeration(@TempDir Pa PaimonCatalogOps ops = new PaimonCatalogOps.CatalogBackedPaimonCatalogOps(catalog, Collections.singletonMap("scan.manifest.parallelism", "1")); PaimonConnectorMetadata metadata = new PaimonConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext()); + ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.singletonList("pt"), Collections.emptyList()); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java index aff106ae4375c9..831a5d59601bc1 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java @@ -59,7 +59,7 @@ public class PaimonConnectorMetadataPartitionViewCacheTest { private static PaimonConnectorMetadata metadataWithCache(RecordingPaimonCatalogOps ops, ConnectorMetadataCache> cache) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext(), + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext(), new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE), new PaimonLatestSnapshotCache(0L, 1), cache); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java index 4eec483f1957e1..8d30fa4364a5f3 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataReadAuthTest.java @@ -50,7 +50,7 @@ public class PaimonConnectorMetadataReadAuthTest { private static PaimonConnectorMetadata metadata(RecordingPaimonCatalogOps ops, RecordingConnectorContext ctx) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), ctx); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), ctx); } private static PaimonTableHandle baseHandle() { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataStatisticsTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataStatisticsTest.java index f4cf4ac6d5e7eb..410cb35ff3ee14 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataStatisticsTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataStatisticsTest.java @@ -50,12 +50,12 @@ public class PaimonConnectorMetadataStatisticsTest { private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } private static PaimonConnectorMetadata metadataWith( RecordingPaimonCatalogOps ops, RecordingConnectorContext context) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), context); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), context); } private static RowType rowType(String... columnNames) { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataSysTableTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataSysTableTest.java index 755be8875720cb..8c8cf08b0c583d 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataSysTableTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataSysTableTest.java @@ -57,7 +57,7 @@ public class PaimonConnectorMetadataSysTableTest { private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops) { - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } private static RowType rowType(String... columnNames) { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java index a1907b44254c6e..1ca8e18400d251 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java @@ -48,7 +48,7 @@ public class PaimonConnectorMetadataTest { private static PaimonConnectorMetadata metadataWith(RecordingPaimonCatalogOps ops) { // Read-path tests ignore the context; a default RecordingConnectorContext is a no-op wrapper. - return new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + return new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); } private static RowType rowType(String... columnNames) { @@ -218,7 +218,7 @@ public void getColumnHandlesUsesTransientTableWithoutReload() { @Test public void disablesCastPredicatePushdown() { PaimonConnectorMetadata metadata = - new PaimonConnectorMetadata(null, Collections.emptyMap(), new RecordingConnectorContext()); + new PaimonConnectorMetadata(null, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); // WHY: the shared converter unwraps CAST shells, so if this returned true (the SPI // default), a predicate like CAST(str_col AS INT)=5 would be pushed to Paimon as @@ -467,7 +467,7 @@ public void getTableSchemaHonorsDottedMappingKeys() { props.put("enable.mapping.varbinary", "true"); props.put("enable.mapping.timestamp_tz", "true"); PaimonConnectorMetadata metadata = - new PaimonConnectorMetadata(ops, props, new RecordingConnectorContext()); + new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(props), new RecordingConnectorContext()); ConnectorTableHandle handle = metadata.getTableHandle(null, "db1", "t1").get(); ConnectorTableSchema schema = metadata.getTableSchema(null, handle); @@ -493,7 +493,7 @@ public void getTableSchemaDefaultsMappingFlagsOff() { // No mapping keys set — the default (legacy-compatible) behavior. PaimonConnectorMetadata metadata = - new PaimonConnectorMetadata(ops, Collections.emptyMap(), new RecordingConnectorContext()); + new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()); ConnectorTableHandle handle = metadata.getTableHandle(null, "db1", "t1").get(); ConnectorTableSchema schema = metadata.getTableSchema(null, handle); @@ -528,7 +528,7 @@ public void getTableSchemaMarksLtzColumnsWithTimeZoneRegardlessOfMapping() { ops.table = new FakePaimonTable( "t1", binaryAndLtzRowType(), Collections.emptyList(), Collections.emptyList()); PaimonConnectorMetadata metadata = - new PaimonConnectorMetadata(ops, props, new RecordingConnectorContext()); + new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(props), new RecordingConnectorContext()); ConnectorTableHandle handle = metadata.getTableHandle(null, "db1", "t1").get(); ConnectorTableSchema schema = metadata.getTableSchema(null, handle); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCreateTableValidationTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCreateTableValidationTest.java index 469bf8303e374f..4bf6f014dce90e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCreateTableValidationTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCreateTableValidationTest.java @@ -44,7 +44,7 @@ public class PaimonCreateTableValidationTest { private PaimonConnectorMetadata metadata() { // Non-null (empty) properties: the ctor derives type-mapping options from them; catalog/context stay null // (rejectDistribution touches only the request). - return new PaimonConnectorMetadata(null, Collections.emptyMap(), null); + return new PaimonConnectorMetadata(null, PaimonCatalogProperties.of(Collections.emptyMap()), null); } @Test diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLiveConnectivityTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLiveConnectivityTest.java index 0c330a297ceaa5..359d0ea429e702 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLiveConnectivityTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonLiveConnectivityTest.java @@ -72,9 +72,9 @@ public void liveMetadataRoundTrip() { String catalogType = System.getenv("PAIMON_CATALOG_TYPE"); Map props = new HashMap<>(); - props.put(PaimonConnectorProperties.WAREHOUSE, warehouse); + props.put("warehouse", warehouse); if (catalogType != null && !catalogType.isEmpty()) { - props.put(PaimonConnectorProperties.PAIMON_CATALOG_TYPE, catalogType); + props.put(PaimonCatalogProperties.PAIMON_CATALOG_TYPE, catalogType); } // Exercise the full production path: PaimonConnector lazily builds a real Catalog and diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanExplainTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanExplainTest.java index 2c6821b6d2e2d3..3501b0eb407361 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanExplainTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanExplainTest.java @@ -48,7 +48,7 @@ public class PaimonScanExplainTest { private static PaimonScanPlanProvider provider() { - return new PaimonScanPlanProvider(new HashMap<>(), null); + return new PaimonScanPlanProvider(PaimonCatalogProperties.of(new HashMap<>()), null); } // ==================== appendExplainInfo: paimonNativeReadSplits ==================== diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderCapabilityTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderCapabilityTest.java index cfc3035743c63d..cdd8018f57dabc 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderCapabilityTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderCapabilityTest.java @@ -42,14 +42,14 @@ public class PaimonScanPlanProviderCapabilityTest { @Test public void paimonOptsOutOfPruneToZeroShortCircuit() { - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), null); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), null); Assertions.assertTrue(provider.ignorePartitionPruneShortCircuit(), "paimon is predicate-driven and must opt out of the prune-to-zero short-circuit"); } @Test public void systemTableScanParamCapabilitiesArePublishedPerTable() { - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), null); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), null); // WHY: the fe-core sys-table guard (PluginDrivenScanNode.checkSysTableScanConstraints) and its // pin resolver both ask THIS provider whether a given metadata view honors @incr / @options. A @@ -108,7 +108,7 @@ public void scannedPartitionCountReturnsDistinctPartitions() { // distinct dataSplit.partition() (== distinct rendered getPartitionValues() maps). Three ranges // over TWO partitions (two ranges of dt=1 + one of dt=2) must count 2, not 3. A mutation that // drops the override (default OptionalLong.empty()) makes this red. - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), null); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), null); List ranges = Arrays.asList( rangeWithPartition("/t/dt=1/a.parquet", part("dt", "1")), rangeWithPartition("/t/dt=1/b.parquet", part("dt", "1")), @@ -120,7 +120,7 @@ public void scannedPartitionCountReturnsDistinctPartitions() { public void scannedPartitionCountEmptyForUnpartitionedTable() { // Every range's partition map is empty (unpartitioned table) -> report nothing so the engine keeps // its own count. (Same value as the un-overridden default; documents the fall-through, not RED-able.) - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), null); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), null); List ranges = Arrays.asList( rangeWithPartition("/t/a.parquet", Collections.emptyMap()), rangeWithPartition("/t/b.parquet", Collections.emptyMap())); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java index bfec168b6b32a0..ce8c337a65de43 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanPlanProviderTest.java @@ -119,7 +119,7 @@ public void resolveTableReloadsWhenTransientTableNull() { "db1", "t1", Collections.emptyList(), Collections.emptyList()); Assertions.assertNull(handle.getPaimonTable(), "precondition: transient table is null"); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table table = provider.resolveTable(handle); // WHY: this is the serde-survival safety net. With a null transient Table, the scan path's @@ -150,7 +150,7 @@ public void resolveTableForSysHandleReloadsViaFourArgSysIdentifier() { "db1", "t1", "snapshots", false); Assertions.assertNull(sysHandle.getPaimonTable(), "precondition: transient table is null"); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table resolved = provider.resolveTable(sysHandle); // WHY: BLOCKER fix — the scan path's own resolveTable used to ALWAYS reload via the 2-arg @@ -179,7 +179,7 @@ public void resolveTableRunsInsideAuthenticatorWhenContextPresent() { "t1", rowType("id"), Collections.emptyList(), Collections.emptyList()); RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuth = true; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops, ctx); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, ctx); PaimonTableHandle handle = new PaimonTableHandle( "db1", "t1", Collections.emptyList(), Collections.emptyList()); @@ -195,7 +195,7 @@ public void resolveTableEntersAuthenticatorOnHappyPath() { ops.table = new FakePaimonTable( "t1", rowType("id"), Collections.emptyList(), Collections.emptyList()); RecordingConnectorContext ctx = new RecordingConnectorContext(); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops, ctx); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, ctx); PaimonTableHandle handle = new PaimonTableHandle( "db1", "t1", Collections.emptyList(), Collections.emptyList()); @@ -213,7 +213,7 @@ public void transientFreeSystemSourceReloadRunsInsideAuthenticator() { "t1$snapshots", rowType("snapshot_id"), Collections.emptyList(), Collections.emptyList()); RecordingConnectorContext ctx = new RecordingConnectorContext(); ctx.failAuthOnInvocation = 2; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops, ctx); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, ctx); PaimonTableHandle handle = PaimonTableHandle.forSystemTable( "db1", "t1", "snapshots", false); @@ -255,7 +255,7 @@ public void planScanEnumeratesSplitsInsideAuthScope(@TempDir Path warehouse) thr RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; RecordingConnectorContext ctx = new RecordingConnectorContext(); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops, ctx); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, ctx); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); @@ -345,7 +345,7 @@ public void forceJniScannerRoutesNativeEligibleSplitToJni() { public void nativeRangeNormalizesBothDataAndDeletionVectorPaths() { RecordingConnectorContext ctx = new RecordingConnectorContext(); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), ctx); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), ctx); RawFile file = parquetRawFile("oss://bkt/warehouse/db/t/part-0.parquet"); DeletionFile dv = new DeletionFile( "oss://bkt/warehouse/db/t/index/dv-0.index", 8L, 16L, 4L); @@ -372,7 +372,7 @@ public void nativeRangeNormalizesBothDataAndDeletionVectorPaths() { public void nativeRangeWithoutDeletionVectorNormalizesOnlyDataPath() { RecordingConnectorContext ctx = new RecordingConnectorContext(); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), ctx); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), ctx); PaimonScanRange range = provider.buildNativeRange( parquetRawFile("oss://bkt/a/part-0.parquet"), null, "parquet", @@ -392,7 +392,7 @@ public void nativeRangeWithoutContextPreservesRawPath() { // available, so the raw path is preserved without NPE. The real oss://->s3:// rewrite is // covered by DefaultConnectorContextNormalizeUriTest (fe-core). PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps()); PaimonScanRange range = provider.buildNativeRange( parquetRawFile("oss://bkt/a/part-0.parquet"), null, "parquet", @@ -412,7 +412,7 @@ public void buildNativeRangeThreadsVendedTokenToBothPaths() { // whatever token the scan computes is threaded VERBATIM to each normalize call. RecordingConnectorContext ctx = new RecordingConnectorContext(); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), ctx); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), ctx); Map vendedToken = new HashMap<>(); vendedToken.put("fs.oss.accessKeyId", "STS.ak"); vendedToken.put("fs.oss.accessKeySecret", "sk"); @@ -455,7 +455,7 @@ public void resolveScanTableAppliesSnapshotPinViaCopy() { PaimonTableHandle pinnedHandle = handle.withScanOptions( Collections.singletonMap("scan.snapshot-id", "5")); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table scanTable = provider.resolveScanTable(pinnedHandle); // WHY: a snapshot-pinned handle must read at the pinned version on BOTH the planned-splits @@ -481,7 +481,7 @@ public void resolveScanTableWithoutScanOptionsDoesNotCopy() { "db1", "t1", Collections.emptyList(), Collections.emptyList()); handle.setPaimonTable(base); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table scanTable = provider.resolveScanTable(handle); // WHY: a normal read must NOT call Table.copy at all — copying with empty options is wasted @@ -504,7 +504,7 @@ public void resolveScanTableRejectsUnsafePhysicalOptionsAtFinalBoundary() { handle.setPaimonTable(base); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - Collections.emptyMap(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingPaimonCatalogOps()); IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> provider.resolveScanTable(handle)); @@ -527,7 +527,7 @@ public void resolveScanTableAllowsSafeRelationOptionToOverrideUnsafePhysicalOpti Collections.singletonMap("read.batch-size", "4096"))); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - Collections.emptyMap(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingPaimonCatalogOps()); Assertions.assertSame(copied, provider.resolveScanTable(optionsHandle)); } @@ -546,7 +546,7 @@ public void resolveSystemScanTableValidatesHiddenDataTable() { "db1", "t1", "partitions", false); handle.setPaimonTable(systemTable); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); // The connector boundary must keep one stable exception type while preserving the // actionable validation detail in the user-facing message and the original cause. @@ -573,11 +573,11 @@ public void resolveSystemScanTableValidatesPinnedSourceGeneration() { "db1", "t1", Collections.emptyList(), Collections.emptyList()); base.setPaimonTable(pinnedSource); PaimonTableHandle system = (PaimonTableHandle) new PaimonConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext()) + ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()) .getSysTableHandle(null, base, "partitions").orElseThrow(AssertionError::new); ops.table = reloadedSource; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); DorisConnectorException e = Assertions.assertThrows(DorisConnectorException.class, () -> provider.resolveScanTable(system)); @@ -609,7 +609,7 @@ public void resolveSystemScanTableAppliesSafeOverrideToHiddenDataTable() { PaimonTableHandle optionsHandle = handle.withScanOptions(PaimonScanParams.markAsOptions( Collections.singletonMap("scan.manifest.parallelism", "1"))); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Assertions.assertSame(safeSystemTable, provider.resolveScanTable(optionsHandle)); Assertions.assertEquals("1", dataTable.lastCopyOptions.get("scan.manifest.parallelism")); @@ -646,7 +646,7 @@ public void resolveSystemScanTableDoesNotRecapReturnedWrapper() { PaimonTableHandle optionsHandle = handle.withScanOptions(PaimonScanParams.markAsOptions( Collections.singletonMap("read.batch-size", "4096"))); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table resolved = provider.resolveScanTable(optionsHandle); // The hidden source is normalized independently. Re-copying the outer wrapper with one cap @@ -687,7 +687,7 @@ public void resolveScanTableResetsStalePinForIncrementalRead(@TempDir Path wareh PaimonTableHandle incrHandle = handle.withScanOptions( Collections.singletonMap("incremental-between", "3,5")); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table scanTable = provider.resolveScanTable(incrHandle); // WHY (FIX-INCR-SCAN-RESET): an @incr read over a base table that persists a stale @@ -732,7 +732,7 @@ public void getScanNodePropertiesAlwaysEmitsPredicateForNoFilterScan(@TempDir Pa "db", "t", Collections.emptyList(), Collections.emptyList()); handle.setPaimonTable(base); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Map props = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -776,7 +776,7 @@ public void getScanNodePropertiesEmitsPathPartitionKeysForPartitionedTable(@Temp "db", "t", Collections.emptyList(), Collections.emptyList()); handle.setPaimonTable(base); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Map props = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -826,7 +826,7 @@ public void getScanNodePropertiesEmitsSchemaEvolutionForReadOptimizedSysTable(@T // JNI), transient Table lost so resolveScanTable reloads the ReadOptimizedTable. PaimonTableHandle handle = PaimonTableHandle.forSystemTable("db", "t", "ro", false); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Map props = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -872,11 +872,11 @@ public void readOptimizedSchemaDictionaryUsesPinnedSourceGeneration(@TempDir Pat "db", "t", Collections.emptyList(), Collections.emptyList()); base.setPaimonTable(pinned); PaimonTableHandle ro = (PaimonTableHandle) new PaimonConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext()) + ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()) .getSysTableHandle(null, base, "ro").orElseThrow(AssertionError::new); ops.table = reloaded; - Map props = new PaimonScanPlanProvider(Collections.emptyMap(), ops) + Map props = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops) .getScanNodeProperties(null, ro, Collections.emptyList(), Optional.empty()); TFileScanRangeParams params = new TFileScanRangeParams(); PaimonScanPlanProvider.applySchemaEvolutionParam( @@ -921,12 +921,12 @@ public void readOptimizedSchemaDictionaryUsesOptionSelectedSnapshot(@TempDir Pat "db", "t", Collections.emptyList(), Collections.emptyList()); base.setPaimonTable(latestGeneration); PaimonTableHandle ro = (PaimonTableHandle) new PaimonConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext()) + ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()) .getSysTableHandle(null, base, "ro").orElseThrow(AssertionError::new); ro = ro.withScanOptions(PaimonScanParams.markAsOptions(Collections.singletonMap( CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(oldSnapshotId)))); - Map props = new PaimonScanPlanProvider(Collections.emptyMap(), ops) + Map props = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops) .getScanNodeProperties(null, ro, Collections.singletonList(new PaimonColumnHandle("old_name", 1)), Optional.empty()); @@ -953,7 +953,7 @@ public void resolveTableUsesTransientWithoutReload() { "db1", "t1", Collections.emptyList(), Collections.emptyList()); handle.setPaimonTable(table); - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); Table resolved = provider.resolveTable(handle); // WHY: the fast path — when the transient Table is already present, resolveTable must use it @@ -1125,7 +1125,7 @@ public void countPushdownCollapsesMultipleSplitsToOneRangeBearingSummedTotal( RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); ConnectorSession session = sessionWithProps(Collections.emptyMap()); @@ -1213,7 +1213,7 @@ public void jniAndCountRangesCarryRealFileFormatNotJni(@TempDir Path warehouse) RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1297,7 +1297,7 @@ public void jniAndCountRangesUseFileSuffixNotAlteredTableDefault(@TempDir Path w RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = altered; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1366,7 +1366,7 @@ public void ignoreJniDropsForcedJniSplit(@TempDir Path warehouse) throws Excepti RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1417,7 +1417,7 @@ public void ignoreNativeDropsNativeSplit(@TempDir Path warehouse) throws Excepti RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1472,7 +1472,7 @@ public void ignorePaimonCppIsNoOpParity(@TempDir Path warehouse) throws Exceptio RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1524,7 +1524,7 @@ public void cppReaderSessionFlagNoLongerChangesThePlan(@TempDir Path warehouse) RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1679,7 +1679,7 @@ public void nativeFileIsSubSplitWhenFileSplitSizeForcesIt(@TempDir Path warehous RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(Collections.emptyMap(), ops); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); PaimonTableHandle handle = new PaimonTableHandle( "db", "t", Collections.emptyList(), Collections.emptyList()); List noColumns = Collections.emptyList(); @@ -1728,7 +1728,7 @@ public void nativeFileIsSubSplitWhenFileSplitSizeForcesIt(@TempDir Path warehous public void buildNativeRangesAttachesSameDeletionVectorToEverySubRange() { RecordingConnectorContext ctx = new RecordingConnectorContext(); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), ctx); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), ctx); RawFile file = parquetRawFile("oss://bkt/a/part-0.parquet"); DeletionFile dv = new DeletionFile("oss://bkt/a/index/dv-0.index", 8L, 16L, 4L); long target = Math.max(1L, file.length() / 3); // force the file to sub-split into >=2 ranges @@ -1758,7 +1758,7 @@ public void buildNativeRangesKeepsFileWholeWhenTargetNonPositive() { // siphoned to the count arm (no precomputed merged count) is kept WHOLE — legacy parity // (splittable=!applyCountPushdown). MUTATION: sub-splitting under count pushdown -> >1 range -> red. PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps()); RawFile file = parquetRawFile("oss://bkt/a/part-0.parquet"); List ranges = provider.buildNativeRanges( @@ -1969,7 +1969,7 @@ public void getScanNodePropertiesNormalizesStaticCreds() { backendStatic.put("AWS_ENDPOINT", "ep"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), scanContext(backendStatic, Collections.emptyMap())); Map scanProps = provider.getScanNodeProperties( @@ -2009,7 +2009,7 @@ public void getScanNodePropertiesOverlaysVendedCreds() { vended.put("AWS_ENDPOINT", "vended-ep"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), scanContext(backendStatic, vended)); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), scanContext(backendStatic, vended)); Map scanProps = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -2037,7 +2037,7 @@ public void getScanNodePropertiesNoContextNoStorageProps() { Map props = new HashMap<>(); props.put("s3.access_key", "raw-ak"); // 2-arg ctor -> context == null (the offline harness path). - PaimonScanPlanProvider provider = new PaimonScanPlanProvider(props, new RecordingPaimonCatalogOps()); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider(PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps()); Map scanProps = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -2070,7 +2070,7 @@ public void getScanNodePropertiesSkipsStoragePropsWithoutBackendMappingAndMerges Arrays.asList(fakeStorageWithoutBackend(), fakeBackendStorage(beMap)); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), scanContextWithStorage(storage)); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), scanContextWithStorage(storage)); Map scanProps = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -2169,7 +2169,7 @@ public void backendOptionsResolveBareDriverUrl() { props.put("jdbc.driver_url", "mysql.jar"); props.put("jdbc.driver_class", "com.mysql.cj.jdbc.Driver"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.singletonMap("jdbc_drivers_dir", "/opt/drivers"))); Map opts = provider.getBackendPaimonOptions(); @@ -2188,7 +2188,7 @@ public void backendOptionsHonorPaimonJdbcAlias() { props.put("paimon.jdbc.driver_url", "mysql.jar"); props.put("paimon.jdbc.driver_class", "com.mysql.cj.jdbc.Driver"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.singletonMap("jdbc_drivers_dir", "/opt/drivers"))); Map opts = provider.getBackendPaimonOptions(); @@ -2207,14 +2207,15 @@ props, new RecordingPaimonCatalogOps(), public void backendOptionsResolveWhenBothAliasesSet() { Map props = new HashMap<>(); props.put("paimon.catalog.type", "jdbc"); - // Both alias forms present with DIFFERENT values. firstNonBlank(JDBC_DRIVER_URL) order is - // {paimon.jdbc.driver_url, jdbc.driver_url} -> the paimon.jdbc.* value wins (legacy priority). + // Both alias forms present with DIFFERENT values. PaimonJdbcMetaStoreProperties declares the + // driver_url aliases as {paimon.jdbc.driver_url, jdbc.driver_url}, and declaration order is + // priority order -> the paimon.jdbc.* value wins (legacy priority). props.put("paimon.jdbc.driver_url", "postgres.jar"); props.put("jdbc.driver_url", "mysql.jar"); props.put("paimon.jdbc.driver_class", "org.postgresql.Driver"); props.put("jdbc.driver_class", "com.mysql.cj.jdbc.Driver"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.singletonMap("jdbc_drivers_dir", "/opt/drivers"))); Map opts = provider.getBackendPaimonOptions(); @@ -2229,6 +2230,27 @@ props, new RecordingPaimonCatalogOps(), "the raw lower-priority alias value must not survive in the BE options"); } + @Test + public void backendOptionsResolveAPaddedDriverUrl() { + Map props = new HashMap<>(); + props.put("paimon.catalog.type", "jdbc"); + props.put("paimon.jdbc.driver_url", " mysql.jar "); + props.put("paimon.jdbc.driver_class", " com.mysql.cj.jdbc.Driver "); + PaimonScanPlanProvider provider = new PaimonScanPlanProvider( + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), + envContext(Collections.singletonMap("jdbc_drivers_dir", "/opt/drivers"))); + + Map opts = provider.getBackendPaimonOptions(); + + // WHY: these two keys are what the BE turns into a real URL and a real Class.forName. Reading + // them from the bound jdbc properties (rather than re-scanning the raw map) means a padded value + // is trimmed before it is resolved -- otherwise BE gets "file:///opt/drivers/ mysql.jar " and + // fails at load time with an error naming neither the property nor the padding. + // MUTATION: forwarding the raw value -> the resolved url keeps the spaces -> red. + Assertions.assertEquals("file:///opt/drivers/mysql.jar", opts.get("jdbc.driver_url")); + Assertions.assertEquals("com.mysql.cj.jdbc.Driver", opts.get("jdbc.driver_class")); + } + @Test public void backendOptionsPreserveSchemeBearingDriverUrl() { Map props = new HashMap<>(); @@ -2236,7 +2258,7 @@ public void backendOptionsPreserveSchemeBearingDriverUrl() { props.put("jdbc.driver_url", "file:///custom/path/mysql.jar"); props.put("jdbc.driver_class", "com.mysql.cj.jdbc.Driver"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); // A value already carrying a scheme is shipped unchanged (no double-prefixing). Assertions.assertEquals("file:///custom/path/mysql.jar", @@ -2249,7 +2271,7 @@ public void backendOptionsEmptyForNonJdbcFlavor() { props.put("paimon.catalog.type", "filesystem"); props.put("jdbc.driver_url", "mysql.jar"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); // Regression guard: the driver_url logic must not leak into non-JDBC flavors. Assertions.assertTrue(provider.getBackendPaimonOptions().isEmpty()); @@ -2265,7 +2287,7 @@ public void backendOptionsForwardJniIoManagerRegardlessOfMetastore() { props.put("paimon.jni.io_manager.tmp_dir", "/tmp/doris-paimon"); props.put("paimon.jni.io_manager.impl_class", "org.example.CustomIOManager"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); Map opts = provider.getBackendPaimonOptions(); @@ -2291,7 +2313,7 @@ public void backendOptionsDropRetiredFileReaderAsyncOptOut() { props.put("paimon.catalog.type", "filesystem"); props.put("paimon.jni.enable_file_reader_async", "false"); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - props, new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); + PaimonCatalogProperties.of(props), new RecordingPaimonCatalogOps(), envContext(Collections.emptyMap())); Map opts = provider.getBackendPaimonOptions(); @@ -2345,7 +2367,7 @@ public void buildSchemaInfoCarriesTimestampSemantics() { @Test public void doesNotUseHiveParquetInt96TimeZone() { Assertions.assertFalse( - new PaimonScanPlanProvider(Collections.emptyMap(), null) + new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), null) .usesHiveParquetInt96TimeZone()); } @@ -2435,7 +2457,7 @@ public void serializedTableReachesScanLevelParams() { "db1", "t1", Collections.emptyList(), Collections.emptyList()); handle.setPaimonTable(table); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps(), + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps(), scanContext(Collections.emptyMap(), Collections.emptyMap())); Map scanProps = provider.getScanNodeProperties( @@ -2563,7 +2585,7 @@ public void getScanNodePropertiesSkipsSchemaEvolutionForNonFileStoreTable() { "db1", "t1", Collections.emptyList(), Collections.emptyList()); handle.setPaimonTable(table); PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps()); Map scanProps = provider.getScanNodeProperties( null, handle, Collections.emptyList(), Optional.empty()); @@ -2595,7 +2617,7 @@ public void buildNativeRangeSetsProportionalWeightFromLengthAndDv() { // when a DV is attached (PaimonSplit:72,112). The native FE weight reproduces that and carries the // scan-level denominator. MUTATION: dropping the native .selfSplitWeight(...) -> weight 0 -> red. PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps()); RawFile file = parquetRawFile("/data/part-0.parquet"); DeletionFile dv = new DeletionFile("/data/dv-0.index", 8L, 16L, 4L); @@ -2618,7 +2640,7 @@ public void buildNativeRangesThreadsDenominatorDistinctFromFileSplitTarget() { // params. Splitting must follow the FILE-SPLIT target while every sub-range carries the DENOMINATOR. // MUTATION: swapping the two args -> wrong split count AND wrong targetSplitSize -> red. PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps()); RawFile file = parquetRawFile("/data/part-0.parquet"); // length 100 long fileSplitTarget = Math.max(1L, file.length() / 3); // 33 -> >=2 sub-ranges long denominator = 64 * MB; // numerically distinct from 33 @@ -2644,7 +2666,7 @@ public void buildNativeRangesCarriesDenominatorEvenWhenFileSplitSizeZero() { // computed independently, so the single range still gets a positive denominator (non-standard // weight) and the whole-file length as its weight. PaimonScanPlanProvider provider = new PaimonScanPlanProvider( - new HashMap<>(), new RecordingPaimonCatalogOps()); + PaimonCatalogProperties.of(new HashMap<>()), new RecordingPaimonCatalogOps()); RawFile file = parquetRawFile("/data/part-0.parquet"); List ranges = provider.buildNativeRanges( @@ -2707,7 +2729,7 @@ public void schemaEvolutionDictPopulatesSharedMemo(@TempDir Path warehouse) thro PaimonTableHandle handle = plainHandle(); PaimonSchemaAtMemo memo = new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); PaimonScanPlanProvider provider = - new PaimonScanPlanProvider(Collections.emptyMap(), ops, null, memo); + new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, null, memo); provider.getScanNodeProperties(null, handle, Collections.emptyList(), Optional.empty()); @@ -2732,7 +2754,7 @@ public void schemaEvolutionDictReadsFromMemoOnHit(@TempDir Path warehouse) throw // The real (unseeded) dict. RecordingPaimonCatalogOps opsReal = new RecordingPaimonCatalogOps(); opsReal.table = base; - String encodedReal = new PaimonScanPlanProvider(Collections.emptyMap(), opsReal, null, + String encodedReal = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), opsReal, null, new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE)) .getScanNodeProperties(null, handle, Collections.emptyList(), Optional.empty()) .get("paimon.schema_evolution"); @@ -2745,7 +2767,7 @@ public void schemaEvolutionDictReadsFromMemoOnHit(@TempDir Path warehouse) throw Collections.emptyList(), Collections.emptyList())); RecordingPaimonCatalogOps opsSeeded = new RecordingPaimonCatalogOps(); opsSeeded.table = base; - String encodedSeeded = new PaimonScanPlanProvider(Collections.emptyMap(), opsSeeded, null, seeded) + String encodedSeeded = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), opsSeeded, null, seeded) .getScanNodeProperties(null, handle, Collections.emptyList(), Optional.empty()) .get("paimon.schema_evolution"); @@ -2769,14 +2791,14 @@ public void schemaEvolutionDictByteIdenticalWithMemo(@TempDir Path warehouse) th RecordingPaimonCatalogOps opsA = new RecordingPaimonCatalogOps(); opsA.table = base; // 2-arg ctor: fresh per-instance memo => first build is a direct read => pre-fix behavior. - String encodedA = new PaimonScanPlanProvider(Collections.emptyMap(), opsA) + String encodedA = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), opsA) .getScanNodeProperties(null, handle, Collections.emptyList(), Optional.empty()) .get("paimon.schema_evolution"); RecordingPaimonCatalogOps opsB = new RecordingPaimonCatalogOps(); opsB.table = base; // 4-arg ctor with a shared memo (first build is also a direct read, then cached). - String encodedB = new PaimonScanPlanProvider(Collections.emptyMap(), opsB, null, + String encodedB = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), opsB, null, new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE)) .getScanNodeProperties(null, handle, Collections.emptyList(), Optional.empty()) .get("paimon.schema_evolution"); @@ -2802,7 +2824,7 @@ public void schemaEvolutionDictSkippedUnderForceJniLeavesMemoEmpty(@TempDir Path ops.table = base; PaimonTableHandle binlog = PaimonTableHandle.forSystemTable("db", "t", "binlog", true); PaimonSchemaAtMemo memo = new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); - Map props = new PaimonScanPlanProvider(Collections.emptyMap(), ops, null, memo) + Map props = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops, null, memo) .getScanNodeProperties(null, binlog, Collections.emptyList(), Optional.empty()); Assertions.assertFalse(props.containsKey("paimon.schema_evolution"), "a force-jni handle skips the schema dict"); @@ -2813,7 +2835,7 @@ public void schemaEvolutionDictSkippedUnderForceJniLeavesMemoEmpty(@TempDir Path RecordingPaimonCatalogOps ops2 = new RecordingPaimonCatalogOps(); ops2.table = base; PaimonSchemaAtMemo memo2 = new PaimonSchemaAtMemo(PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); - Map props2 = new PaimonScanPlanProvider(Collections.emptyMap(), ops2, null, memo2) + Map props2 = new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops2, null, memo2) .getScanNodeProperties( sessionWithProps(Collections.singletonMap("force_jni_scanner", "true")), plainHandle(), Collections.emptyList(), Optional.empty()); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java index 121474d7c2eceb..db6f6b5e4a1290 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonScanRangeBucketTest.java @@ -119,7 +119,7 @@ private static Set rangeBuckets(List ranges) { private static PaimonScanPlanProvider providerFor(Table table) { RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); ops.table = table; - return new PaimonScanPlanProvider(Collections.emptyMap(), ops); + return new PaimonScanPlanProvider(PaimonCatalogProperties.of(Collections.emptyMap()), ops); } private static PaimonTableHandle handleFor(String tableName) { diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableHandleScanOptionsTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableHandleScanOptionsTest.java index dda4607b97dcf8..eb81b8e430e41b 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableHandleScanOptionsTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableHandleScanOptionsTest.java @@ -307,7 +307,7 @@ public void getTableSchemaEmitsPartitionColumnsKeyForPartitionedHandle() { handle.setPaimonTable(table); ConnectorTableSchema schema = new PaimonConnectorMetadata( - ops, Collections.emptyMap(), new RecordingConnectorContext()) + ops, PaimonCatalogProperties.of(Collections.emptyMap()), new RecordingConnectorContext()) .getTableSchema(null, handle); Map props = schema.getProperties(); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableOptionsTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableOptionsTest.java index 22cf5a88fe1130..f291ea74fc63e2 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableOptionsTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonTableOptionsTest.java @@ -70,7 +70,7 @@ public void tableOptionAndJniNamespacesAreExcludedFromCatalogOptions() { props.put("paimon.jni.enable_jni_io_manager", "true"); props.put("paimon.client-pool-size", "7"); - Options options = PaimonCatalogFactory.buildCatalogOptions(props); + Options options = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props)); // WHY (#65955): both namespaces are re-keyed by the generic "paimon.*" passthrough unless // excluded, which would push a per-TABLE option and a BE scanner knob into the Paimon CATALOG diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/package-info.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/package-info.java index 8f10a7db723fe2..208b5d4c92fe77 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/package-info.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/package-info.java @@ -188,7 +188,7 @@ * and defaults stay entirely inside the connector — prefix the key with the connector's type name * so two connectors cannot collide. Shipped precedent: {@code hive.ignore_absent_partitions}, * {@code hive.enable_hms_events_incremental_sync} and {@code hive.hms_events_batch_size_per_rpc} are - * declared in {@code HiveConnectorProperties} and read in {@code HiveScanPlanProvider} / + * declared in {@code HiveCatalogProperties} and read in {@code HiveScanPlanProvider} / * {@code HiveConnector}; those key strings appear nowhere in {@code fe-core}. *

  • Per FE process (one deployment-level value for every catalog, e.g. a driver directory) * → a key in the plugin's own {@code .conf}, read with {@code ConnectorConf.get}. The engine diff --git a/fe/fe-connector/fe-connector-trino/pom.xml b/fe/fe-connector/fe-connector-trino/pom.xml index 792f024e551ec1..89c19b1a465889 100644 --- a/fe/fe-connector/fe-connector-trino/pom.xml +++ b/fe/fe-connector/fe-connector-trino/pom.xml @@ -46,6 +46,13 @@ under the License. ${project.version} + + + ${project.groupId} + fe-foundation + ${project.version} + + ${project.groupId} diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoBootstrap.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoBootstrap.java index 42b0f171969409..2663d4afefc50a 100644 --- a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoBootstrap.java +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoBootstrap.java @@ -334,15 +334,15 @@ private static void configureJulLogging() { * That compatibility path was dropped deliberately — a deployment whose plugins still sit in a * legacy dir must move them or point the config at them. * - * @param properties catalog properties (unstripped, may carry {@code trino.plugin.dir}) + * @param catalogOverride the catalog's own {@code trino.plugin.dir}, already bound by + * {@link TrinoCatalogProperties}; empty when the catalog names none * @param configuredDir the deployment-level setting, already resolved by the caller from * {@code plugin_dir} in the plugin's own conf or {@code trino_connector_plugin_dir} * in fe.conf; null or empty means the engine delivered neither */ - public static String resolvePluginDir(Map properties, String configuredDir) { - String explicitDir = properties.get("trino.plugin.dir"); - if (explicitDir != null && !explicitDir.isEmpty()) { - return explicitDir; + public static String resolvePluginDir(String catalogOverride, String configuredDir) { + if (catalogOverride != null && !catalogOverride.isEmpty()) { + return catalogOverride; } if (configuredDir == null || configuredDir.isEmpty()) { @@ -350,7 +350,7 @@ public static String resolvePluginDir(Map properties, String con // the engine failed to deliver it. Guessing a dir here would surface as "catalog creates fine // but every query fails", so fail where the cause is still visible. throw new IllegalStateException( - "neither '" + TrinoConnectorProvider.CONF_PLUGIN_DIR + "' in " + "neither '" + TrinoConf.CONF_PLUGIN_DIR + "' in " + TrinoConnectorProvider.TYPE + ".conf nor trino_connector_plugin_dir in " + "fe.conf was delivered; cannot resolve the Trino plugin dir"); } diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoCatalogProperties.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoCatalogProperties.java new file mode 100644 index 00000000000000..7de1a1d9ae1fb0 --- /dev/null +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoCatalogProperties.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.trino; + +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; +import org.apache.doris.foundation.property.ParamRules; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Everything a user writes in {@code CREATE CATALOG} for a trino-connector catalog, bound and + * validated in one step. Deployment-level settings — one per FE rather than one per catalog — are the + * other half and live in {@link TrinoConf}. + * + *

    {@link #of(Map)} binds, derives and validates together: if an instance exists, its properties are + * usable. It does no I/O and is idempotent, because it runs at CREATE, at ALTER validation (on the + * merged candidate) and on every connector rebuild — including on an FE replaying the edit log. The + * heavyweight part (loading Trino plugins, resolving the connector factory) stays behind + * {@code TrinoDorisConnector}'s lazy init, which {@code preCreateValidation} pulls forward to CREATE. + * + *

    Unknown keys are accepted, always. The same map carries engine keys ({@code type}, + * {@code meta.cache.*}, ...) and storage keys, and {@code ALTER CATALOG} merges properties — it can + * overwrite a key but never remove one, so a key refused here would leave a catalog no statement could + * repair. Bad values are refused; unrecognized names are not. + */ +public final class TrinoCatalogProperties { + + private static final Logger LOG = LogManager.getLogger(TrinoCatalogProperties.class); + + /** + * The prefix on every property destined for the Trino connector itself. Unlike the adbc + * connector's driver options, the prefix is stripped: what the Trino connector — and the + * BE-side scanner — see is the bare Trino property name. + */ + public static final String TRINO_PROPERTIES_PREFIX = "trino."; + + /** The Trino connector to bridge to, e.g. {@code postgresql}. */ + public static final String CONNECTOR_NAME = TRINO_PROPERTIES_PREFIX + "connector.name"; + + /** + * Per-catalog override of the deployment-level plugin directory ({@link TrinoConf#CONF_PLUGIN_DIR}), + * which it wins over. The combination is resolved by {@code TrinoBootstrap.resolvePluginDir}: an + * A-class key overriding a B-class setting belongs to neither class alone. + */ + public static final String PLUGIN_DIR = TRINO_PROPERTIES_PREFIX + "plugin.dir"; + + /** The bare key {@link #CONNECTOR_NAME} carries once the prefix is stripped. */ + private static final String STRIPPED_CONNECTOR_NAME = + CONNECTOR_NAME.substring(TRINO_PROPERTIES_PREFIX.length()); + + @ConnectorProperty(names = {CONNECTOR_NAME}, + description = "the Trino connector to bridge to, e.g. postgresql") + private String connectorName; + + @ConnectorProperty(names = {PLUGIN_DIR}, required = false, + description = "per-catalog override of the deployment-level Trino plugin directory") + private String pluginDirOverride = ""; + + private final Map raw; + private Map trinoProperties; + + private TrinoCatalogProperties(Map properties) { + this.raw = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); + } + + public static TrinoCatalogProperties of(Map properties) { + TrinoCatalogProperties p = new TrinoCatalogProperties(properties); + ConnectorPropertiesUtils.bindConnectorProperties(p, properties); + new ParamRules() + .require(p.connectorName, "Required property '" + CONNECTOR_NAME + "' is missing") + .validate(); + p.connectorName = correctDeprecatedSpelling(p.connectorName); + p.trinoProperties = deriveTrinoProperties(p.raw, p.connectorName); + return p; + } + + /** + * Trino renamed its dashed connector names to underscores; its {@code ConnectorName} constructor + * now rejects anything outside {@code [a-z][a-z0-9_]*} outright, so the old spellings have to be + * translated rather than passed through. + */ + private static String correctDeprecatedSpelling(String name) { + if (name.indexOf('-') < 0) { + return name; + } + String corrected = name.replace('-', '_'); + LOG.warn("Using deprecated connector name '{}', corrected to '{}'", name, corrected); + return corrected; + } + + /** + * The {@code trino.*} properties with the prefix stripped — what both the Trino connector and the + * BE-side scanner consume. {@code connector.name} is kept in the map (BE reads it back out of this + * very payload; {@code TrinoBootstrap} removes it before handing the rest to Trino) and carries the + * corrected spelling: it used to carry the raw one, so a catalog written with a dashed name + * worked for metadata on FE and failed every scan on BE, where the same name goes straight into + * Trino's {@code ConnectorName}. + */ + private static Map deriveTrinoProperties(Map raw, String connectorName) { + Map stripped = new LinkedHashMap<>(); + for (Map.Entry entry : raw.entrySet()) { + if (entry.getKey().startsWith(TRINO_PROPERTIES_PREFIX)) { + stripped.put(entry.getKey().substring(TRINO_PROPERTIES_PREFIX.length()), entry.getValue()); + } + } + stripped.put(STRIPPED_CONNECTOR_NAME, connectorName); + return Collections.unmodifiableMap(stripped); + } + + /** The Trino connector name, with any deprecated dashed spelling already corrected. */ + public String getConnectorName() { + return connectorName; + } + + /** The per-catalog plugin directory, or {@code ""} when this catalog names none. */ + public String getPluginDirOverride() { + return pluginDirOverride; + } + + public Map getTrinoProperties() { + return trinoProperties; + } + + public Map getRaw() { + return raw; + } + + @Override + public String toString() { + return ConnectorPropertiesUtils.toMaskedString(this); + } +} diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConf.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConf.java new file mode 100644 index 00000000000000..a3980cdc5c90fc --- /dev/null +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConf.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.trino; + +import org.apache.doris.connector.spi.ConnectorConf; +import org.apache.doris.connector.spi.ConnectorContext; + +/** + * The deployment-level settings of this plugin: one per FE, not one per catalog. Per-catalog settings + * are the other half and live in {@link TrinoCatalogProperties}. + * + *

    They are read from the plugin's own conf file, which is named after + * {@link TrinoConnectorProvider#TYPE} ({@code trino-connector.conf}) rather than after the plugin + * directory, each falling back to the {@code fe.conf} key it used to live under. + */ +public final class TrinoConf { + + /** + * Directory holding the Trino plugins this connector loads, in {@code trino-connector.conf}. + * A catalog may override it with {@link TrinoCatalogProperties#PLUGIN_DIR}, which wins over this. + */ + public static final String CONF_PLUGIN_DIR = "plugin_dir"; + + /** The fe.conf name of {@link #CONF_PLUGIN_DIR}, forwarded through the engine environment. */ + public static final String ENV_PLUGIN_DIR = "trino_connector_plugin_dir"; + + private TrinoConf() { + } + + /** + * The configured Trino plugin directory, or null when the engine delivered neither the plugin conf + * key nor the fe.conf one. There is no built-in default: guessing a directory would surface as + * "catalog creates fine but every query fails", so the caller fails where the cause is visible. + */ + public static String pluginDir(ConnectorContext context) { + return ConnectorConf.get(context, CONF_PLUGIN_DIR, ENV_PLUGIN_DIR, null); + } +} diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorProvider.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorProvider.java index 6a317548e72da3..5faeb174c28536 100644 --- a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorProvider.java +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoConnectorProvider.java @@ -29,8 +29,6 @@ */ public class TrinoConnectorProvider implements ConnectorProvider { - static final String TRINO_CONNECTOR_NAME = "trino.connector.name"; - /** * This connector's type, and therefore its {@code name()} — which is what the engine names its * conf file after, so the plugin must ship {@code trino-connector.conf.template}. Note that this is @@ -39,15 +37,6 @@ public class TrinoConnectorProvider implements ConnectorProvider { */ public static final String TYPE = "trino-connector"; - /** - * Directory holding the Trino plugins this connector loads, in {@code trino-connector.conf}. - * Falls back to fe.conf's {@code trino_connector_plugin_dir}, which is where it used to live. - */ - public static final String CONF_PLUGIN_DIR = "plugin_dir"; - - /** The fe.conf name of {@link #CONF_PLUGIN_DIR}, forwarded through the engine environment. */ - public static final String ENV_PLUGIN_DIR = "trino_connector_plugin_dir"; - @Override public String getType() { return TYPE; @@ -58,12 +47,12 @@ public Connector create(Map properties, ConnectorContext context return new TrinoDorisConnector(properties, context); } + /** + * Binds and validates through the typed holder; the ALTER door reaches this same method through the + * SPI default {@code validatePropertiesForUpdate}, which validates the merged candidate. + */ @Override public void validateProperties(Map properties) { - String connectorName = properties.get(TRINO_CONNECTOR_NAME); - if (connectorName == null || connectorName.isEmpty()) { - throw new IllegalArgumentException( - "Required property '" + TRINO_CONNECTOR_NAME + "' is missing"); - } + TrinoCatalogProperties.of(properties); } } diff --git a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoDorisConnector.java b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoDorisConnector.java index 17449d3948dcf4..826f69a6f5fe88 100644 --- a/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoDorisConnector.java +++ b/fe/fe-connector/fe-connector-trino/src/main/java/org/apache/doris/connector/trino/TrinoDorisConnector.java @@ -18,14 +18,12 @@ package org.apache.doris.connector.trino; import org.apache.doris.connector.spi.Connector; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.ConnectorValidationContext; import org.apache.doris.connector.spi.scan.ConnectorScanPlanProvider; -import com.google.common.collect.ImmutableMap; import io.trino.Session; import io.trino.connector.ConnectorName; import io.trino.spi.classloader.ThreadContextClassLoader; @@ -35,7 +33,6 @@ import java.io.IOException; import java.util.Map; -import java.util.stream.Collectors; /** * Doris Connector SPI implementation that bridges to a Trino Connector. @@ -46,19 +43,19 @@ public class TrinoDorisConnector implements Connector { private static final Logger LOG = LogManager.getLogger(TrinoDorisConnector.class); - private static final String TRINO_PROPERTIES_PREFIX = "trino."; - private final Map properties; + // Bound + validated once here: an existing TrinoDorisConnector always has usable catalog + // properties. The heavyweight half (plugin loading, connector factory) stays in doInitialize(). + private final TrinoCatalogProperties props; private final ConnectorContext context; private volatile io.trino.spi.connector.Connector trinoConnector; private volatile Session trinoSession; private volatile CatalogHandle trinoCatalogHandle; private volatile ConnectorName trinoConnectorName; - private volatile ImmutableMap trinoProperties; public TrinoDorisConnector(Map properties, ConnectorContext context) { - this.properties = properties; + this.props = TrinoCatalogProperties.of(properties); this.context = context; } @@ -103,12 +100,11 @@ public void close() throws IOException { } /** - * Returns all Trino-specific properties (with "trino." prefix stripped). - * Used by fe-core for backward compatibility (e.g., BE needs create_time key). + * All Trino-specific properties, with the {@code trino.} prefix stripped. Consumed by + * {@link TrinoScanPlanProvider}, which serializes them into the BE scan payload. */ public Map getTrinoProperties() { - ensureInitialized(); - return trinoProperties; + return props.getTrinoProperties(); } public io.trino.spi.connector.Connector getTrinoConnector() { @@ -142,40 +138,22 @@ private void ensureInitialized() { } private void doInitialize() { - // 1. Extract Trino properties (strip "trino." prefix) - trinoProperties = ImmutableMap.copyOf( - properties.entrySet().stream() - .filter(e -> e.getKey().startsWith(TRINO_PROPERTIES_PREFIX)) - .collect(Collectors.toMap( - e -> e.getKey().substring(TRINO_PROPERTIES_PREFIX.length()), - Map.Entry::getValue))); - - String connectorNameStr = trinoProperties.get("connector.name"); - if (connectorNameStr == null || connectorNameStr.isEmpty()) { - throw new RuntimeException( - "Cannot find trino.connector.name property. " - + "Please specify a connector name in catalog properties."); - } - - if (connectorNameStr.indexOf('-') >= 0) { - String deprecated = connectorNameStr; - connectorNameStr = connectorNameStr.replace('-', '_'); - LOG.warn("Using deprecated connector name '{}', corrected to '{}'", - deprecated, connectorNameStr); - } + // The stripped trino.* map, the connector name and its deprecated-spelling correction are all + // the holder's, computed once when this connector was built. + String connectorNameStr = props.getConnectorName(); - // 2. Initialize Trino plugin infrastructure (singleton). + // 1. Initialize Trino plugin infrastructure (singleton). // The plugin dir is a deployment-level setting: this plugin's classloader cannot see FE Config // directly, so it arrives either in this plugin's own trino-connector.conf or, for a deployment - // that has not moved to that file, from fe.conf through the engine environment. - String pluginDir = TrinoBootstrap.resolvePluginDir(properties, - ConnectorConf.get(context, TrinoConnectorProvider.CONF_PLUGIN_DIR, - TrinoConnectorProvider.ENV_PLUGIN_DIR, null)); + // that has not moved to that file, from fe.conf through the engine environment. A catalog may + // override it for itself. + String pluginDir = TrinoBootstrap.resolvePluginDir(props.getPluginDirOverride(), + TrinoConf.pluginDir(context)); TrinoBootstrap bootstrap = TrinoBootstrap.getInstance(pluginDir); - // 3. Create Trino Connector + Session for this catalog + // 2. Create Trino Connector + Session for this catalog TrinoBootstrap.TrinoConnectionResult result = bootstrap.createConnection( - context.getCatalogName(), connectorNameStr, trinoProperties); + context.getCatalogName(), connectorNameStr, props.getTrinoProperties()); // Publish the guard field (trinoConnector) LAST. ensureInitialized() and the other readers use // `trinoConnector != null` as the initialized flag and then read trinoSession/trinoCatalogHandle. diff --git a/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoBootstrapTest.java b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoBootstrapTest.java index 5c56aff30460ea..d5e9c969edda33 100644 --- a/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoBootstrapTest.java +++ b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoBootstrapTest.java @@ -17,7 +17,6 @@ package org.apache.doris.connector.trino; -import org.apache.doris.connector.spi.ConnectorConf; import org.apache.doris.connector.spi.ConnectorContext; import com.google.common.collect.ImmutableMap; @@ -28,7 +27,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; import java.util.Map; /** @@ -55,7 +53,10 @@ private static void installPluginIn(Path dorisHome, String subdir) throws IOExce @Test public void perCatalogPropertyTakesPrecedence() { String resolved = TrinoBootstrap.resolvePluginDir( - ImmutableMap.of("trino.plugin.dir", "/custom/catalog/dir"), "/should/be/ignored"); + TrinoCatalogProperties.of(ImmutableMap.of( + TrinoCatalogProperties.CONNECTOR_NAME, "postgresql", + TrinoCatalogProperties.PLUGIN_DIR, "/custom/catalog/dir")).getPluginDirOverride(), + "/should/be/ignored"); Assertions.assertEquals("/custom/catalog/dir", resolved); } @@ -64,7 +65,7 @@ public void theDeploymentLevelSettingIsHonored() { // Exactly what the regression environment configures, delivered by the engine because the // plugin classloader cannot read FE Config. String resolved = TrinoBootstrap.resolvePluginDir( - Collections.emptyMap(), "/tmp/trino_connector/connectors"); + "", "/tmp/trino_connector/connectors"); Assertions.assertEquals("/tmp/trino_connector/connectors", resolved); } @@ -79,7 +80,7 @@ public void legacyPluginDirsAreNotConsultedEvenWhenTheyHoldPlugins(@TempDir Path installPluginIn(dorisHome, "plugins/connectors"); String resolved = TrinoBootstrap.resolvePluginDir( - Collections.emptyMap(), dorisHome + "/plugins/trino_plugins"); + "", dorisHome + "/plugins/trino_plugins"); Assertions.assertEquals(dorisHome + "/plugins/trino_plugins", resolved); } @@ -90,11 +91,11 @@ public void missingSettingFailsLoudNamingBothPlacesItCanBeSet() { // but every query fails"; throwing keeps the cause at the point of breakage. The message names // both channels because after the migration either one could be the missing half. IllegalStateException e = Assertions.assertThrows(IllegalStateException.class, - () -> TrinoBootstrap.resolvePluginDir(Collections.emptyMap(), null)); + () -> TrinoBootstrap.resolvePluginDir("", null)); Assertions.assertTrue(e.getMessage().contains("trino-connector.conf"), e.getMessage()); Assertions.assertTrue(e.getMessage().contains("trino_connector_plugin_dir"), e.getMessage()); Assertions.assertThrows(IllegalStateException.class, - () -> TrinoBootstrap.resolvePluginDir(Collections.emptyMap(), "")); + () -> TrinoBootstrap.resolvePluginDir("", "")); } @Test @@ -102,18 +103,15 @@ public void pluginConfBeatsFeConfAndBothBeatNothing() { // The resolution the connector performs before calling resolvePluginDir. Asserted here rather // than trusted, because getting it backwards would make an administrator's edit to // trino-connector.conf silently do nothing while fe.conf still holds the old value. - Assertions.assertEquals("/from/plugin/conf", ConnectorConf.get( - context(ImmutableMap.of(TrinoConnectorProvider.CONF_PLUGIN_DIR, "/from/plugin/conf"), - ImmutableMap.of(TrinoConnectorProvider.ENV_PLUGIN_DIR, "/from/fe/conf")), - TrinoConnectorProvider.CONF_PLUGIN_DIR, TrinoConnectorProvider.ENV_PLUGIN_DIR, null)); + Assertions.assertEquals("/from/plugin/conf", TrinoConf.pluginDir( + context(ImmutableMap.of(TrinoConf.CONF_PLUGIN_DIR, "/from/plugin/conf"), + ImmutableMap.of(TrinoConf.ENV_PLUGIN_DIR, "/from/fe/conf")))); - Assertions.assertEquals("/from/fe/conf", ConnectorConf.get( + Assertions.assertEquals("/from/fe/conf", TrinoConf.pluginDir( context(ImmutableMap.of(), - ImmutableMap.of(TrinoConnectorProvider.ENV_PLUGIN_DIR, "/from/fe/conf")), - TrinoConnectorProvider.CONF_PLUGIN_DIR, TrinoConnectorProvider.ENV_PLUGIN_DIR, null)); + ImmutableMap.of(TrinoConf.ENV_PLUGIN_DIR, "/from/fe/conf")))); - Assertions.assertNull(ConnectorConf.get(context(ImmutableMap.of(), ImmutableMap.of()), - TrinoConnectorProvider.CONF_PLUGIN_DIR, TrinoConnectorProvider.ENV_PLUGIN_DIR, null)); + Assertions.assertNull(TrinoConf.pluginDir(context(ImmutableMap.of(), ImmutableMap.of()))); } @Test diff --git a/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoCatalogPropertiesTest.java new file mode 100644 index 00000000000000..7569e797e51337 --- /dev/null +++ b/fe/fe-connector/fe-connector-trino/src/test/java/org/apache/doris/connector/trino/TrinoCatalogPropertiesTest.java @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.trino; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Tests {@link TrinoCatalogProperties} — the typed holder for everything a user writes in + * {@code CREATE CATALOG} for a trino-connector catalog. + */ +public class TrinoCatalogPropertiesTest { + + private static Map minimal() { + Map m = new LinkedHashMap<>(); + m.put(TrinoCatalogProperties.CONNECTOR_NAME, "postgresql"); + return m; + } + + @Test + public void bindsBothKeysAndDefaults() { + Map m = minimal(); + m.put(TrinoCatalogProperties.PLUGIN_DIR, "/custom/catalog/dir"); + TrinoCatalogProperties p = TrinoCatalogProperties.of(m); + Assertions.assertEquals("postgresql", p.getConnectorName()); + Assertions.assertEquals("/custom/catalog/dir", p.getPluginDirOverride()); + } + + @Test + public void pluginDirOverrideDefaultsToEmpty() { + Assertions.assertEquals("", TrinoCatalogProperties.of(minimal()).getPluginDirOverride()); + } + + // ===== required-ness ===== + + @Test + public void missingConnectorNameFailsNamingTheKey() { + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> TrinoCatalogProperties.of(Collections.emptyMap())); + Assertions.assertTrue(e.getMessage().contains(TrinoCatalogProperties.CONNECTOR_NAME), e.getMessage()); + } + + @Test + public void emptyConnectorNameCountsAsMissing() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> TrinoCatalogProperties.of(ImmutableMap.of(TrinoCatalogProperties.CONNECTOR_NAME, ""))); + } + + /** + * The binder treats a blank value as unset, where the hand-written check only tested + * {@code isEmpty()}. A whitespace-only name could never have worked; it now fails at CREATE. + */ + @Test + public void blankConnectorNameCountsAsMissing() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> TrinoCatalogProperties.of(ImmutableMap.of(TrinoCatalogProperties.CONNECTOR_NAME, " "))); + } + + // ===== the trino.* passthrough map ===== + + @Test + public void trinoPropertiesAreStrippedOfThePrefix() { + Map m = minimal(); + m.put("trino.hive.metastore.uri", "thrift://host:9083"); + m.put("trino.case-insensitive-name-matching", "true"); + Map trino = TrinoCatalogProperties.of(m).getTrinoProperties(); + Assertions.assertEquals("thrift://host:9083", trino.get("hive.metastore.uri")); + Assertions.assertEquals("true", trino.get("case-insensitive-name-matching")); + // connector.name stays in the map: BE reads it back out of this very payload. + Assertions.assertEquals("postgresql", trino.get("connector.name")); + } + + @Test + public void nonTrinoKeysDoNotEnterTheTrinoProperties() { + Map m = minimal(); + m.put("type", "trino-connector"); + m.put("s3.endpoint", "http://minio:9000"); + m.put("meta.cache.ttl-second", "60"); + Map trino = TrinoCatalogProperties.of(m).getTrinoProperties(); + Assertions.assertEquals(Collections.singleton("connector.name"), trino.keySet()); + } + + @Test + public void trinoPropertiesAreUnmodifiable() { + Assertions.assertThrows(UnsupportedOperationException.class, + () -> TrinoCatalogProperties.of(minimal()).getTrinoProperties().put("x", "y")); + } + + // ===== the deprecated dashed spelling ===== + + @Test + public void deprecatedDashedNameIsCorrected() { + Assertions.assertEquals("delta_lake", + TrinoCatalogProperties.of(ImmutableMap.of( + TrinoCatalogProperties.CONNECTOR_NAME, "delta-lake")).getConnectorName()); + } + + /** + * Guards the fix for a real defect: the correction used to live in a local variable, so the map + * serialized into the BE scan payload kept the dashed spelling. BE feeds that value straight into + * Trino's {@code ConnectorName}, whose constructor rejects anything outside + * {@code [a-z][a-z0-9_]*} — so every SELECT on such a catalog failed on BE with + * "Invalid connector name", while FE metadata worked fine. + */ + @Test + public void theCorrectionAlsoReachesTheTrinoPropertiesSentToBackends() { + Map trino = TrinoCatalogProperties.of(ImmutableMap.of( + TrinoCatalogProperties.CONNECTOR_NAME, "delta-lake")).getTrinoProperties(); + Assertions.assertEquals("delta_lake", trino.get("connector.name")); + } + + @Test + public void anUndashedNameIsLeftAlone() { + Assertions.assertEquals("delta_lake", + TrinoCatalogProperties.of(ImmutableMap.of( + TrinoCatalogProperties.CONNECTOR_NAME, "delta_lake")).getConnectorName()); + } + + // ===== the three rules of of() ===== + + /** + * Guards DESIGN D3(2): ALTER CATALOG merges properties — it can overwrite a key but never remove + * one — and the same map carries engine keys and storage keys. A key refused here would leave a + * catalog no statement could repair. + */ + @Test + public void unknownKeysAreTolerated() { + Map m = minimal(); + m.put("some_future_key", "x"); + m.put("s3.endpoint", "http://minio:9000"); + Assertions.assertDoesNotThrow(() -> TrinoCatalogProperties.of(m)); + } + + /** Guards DESIGN D3(1): of() runs on every connector rebuild, so it must be pure and idempotent. */ + @Test + public void ofIsIdempotent() { + Map m = minimal(); + m.put("trino.hive.metastore.uri", "thrift://host:9083"); + TrinoCatalogProperties first = TrinoCatalogProperties.of(m); + TrinoCatalogProperties second = TrinoCatalogProperties.of(m); + Assertions.assertEquals(first.getConnectorName(), second.getConnectorName()); + Assertions.assertEquals(first.getTrinoProperties(), second.getTrinoProperties()); + } + + /** + * The provider's door must stay wired to the holder: {@code validateProperties} is one statement, + * so nothing else would fail if someone emptied it. + */ + @Test + public void providerDoorRunsTheHolder() { + TrinoConnectorProvider provider = new TrinoConnectorProvider(); + Assertions.assertThrows(IllegalArgumentException.class, + () -> provider.validateProperties(Collections.emptyMap())); + Assertions.assertDoesNotThrow(() -> provider.validateProperties(minimal())); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java index 7f842b6b3f3b1f..d5a70b233e9ea1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java @@ -600,7 +600,7 @@ private static Map buildEnvironment() { // Hive CREATE TABLE defaults (P7.1): the fe-connector-hive plugin cannot read FE Config, so the two // FE-global CREATE TABLE toggles are threaded through the environment (not persisted into the catalog // property map) and applied by HiveConnectorMetadata.createTable when the user did not override them. - // Keys must stay byte-identical to the reads in HiveConnectorProperties. + // Keys must stay byte-identical to the reads in the hive plugin (HmsConf / HiveConnectorMetadata). env.put("hive_default_file_format", Config.hive_default_file_format); env.put("enable_create_hive_bucket_table", String.valueOf(Config.enable_create_hive_bucket_table)); // Build version stamped into a created Hive table's doris.version parameter (legacy diff --git a/fe/fe-foundation/src/test/java/org/apache/doris/foundation/property/ConnectorPropertiesUtilsTest.java b/fe/fe-foundation/src/test/java/org/apache/doris/foundation/property/ConnectorPropertiesUtilsTest.java index 15da4a8b531e55..6f6ad22cb77865 100644 --- a/fe/fe-foundation/src/test/java/org/apache/doris/foundation/property/ConnectorPropertiesUtilsTest.java +++ b/fe/fe-foundation/src/test/java/org/apache/doris/foundation/property/ConnectorPropertiesUtilsTest.java @@ -61,6 +61,76 @@ void testAliasMatching() { Assertions.assertEquals("matched-by-alias", config.getAliasValue()); } + // ------------------------------------------------------------------ + // Alias-resolution semantics. + // + // WHY these are pinned: connectors that assemble an SDK/BE payload used to resolve the same + // aliases with a hand-written "first non-blank of these keys" helper, reading the raw map a + // SECOND time next to the annotation binding. Folding that assembly onto the bound values only + // preserves behaviour because the binder resolves aliases the same way. If any of the three + // rules below changes, those connectors do not fail to compile — they silently resolve a + // different value than the one they validated. + // ------------------------------------------------------------------ + + /** + * A blank value counts as ABSENT, so a later alias wins over an earlier blank one. This is + * "first non-blank", not "first present": a map that carries {@code alias1=""} must still bind + * {@code alias2}. + */ + @Test + void testAliasSkipsBlankAndPicksFirstNonBlank() { + Map props = new HashMap<>(); + props.put("alias1", " "); + props.put("alias2", "matched-by-second-alias"); + + SampleConfig config = new SampleConfig(); + ConnectorPropertiesUtils.bindConnectorProperties(config, props); + + Assertions.assertEquals("matched-by-second-alias", config.getAliasValue()); + } + + /** Declaration order in {@code names()} IS the priority order when several aliases are set. */ + @Test + void testAliasDeclarationOrderIsPriorityOrder() { + Map props = new HashMap<>(); + props.put("alias1", "from-first"); + props.put("alias2", "from-second"); + + SampleConfig config = new SampleConfig(); + ConnectorPropertiesUtils.bindConnectorProperties(config, props); + + Assertions.assertEquals("from-first", config.getAliasValue()); + } + + /** All aliases blank leaves the field at its initial value; the binder never writes a blank. */ + @Test + void testAllBlankAliasesLeaveFieldAtInitialValue() { + Map props = new HashMap<>(); + props.put("alias1", ""); + props.put("alias2", " "); + + SampleConfig config = new SampleConfig(); + ConnectorPropertiesUtils.bindConnectorProperties(config, props); + + Assertions.assertNull(config.getAliasValue()); + } + + /** + * The bound value is TRIMMED. This is the one place the binder differs from the hand-written + * helpers it replaces (which returned the value verbatim), so a padded property reaches the + * downstream SDK/payload trimmed once assembly is folded onto the bound value. + */ + @Test + void testBoundValueIsTrimmed() { + Map props = new HashMap<>(); + props.put("string.key", " hello "); + + SampleConfig config = new SampleConfig(); + ConnectorPropertiesUtils.bindConnectorProperties(config, props); + + Assertions.assertEquals("hello", config.getStringValue()); + } + @Test void testUnsupportedFieldNotSet() { Map props = new HashMap<>(); diff --git a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy index a02c620862b9a2..0f4e6f9a5fd911 100644 --- a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy +++ b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy @@ -54,7 +54,8 @@ suite("test_hive_base_case_auth", "p0,auth_call") { connect(user, "${pwd}", context.config.jdbcUrl) { test { sql """create catalog if not exists ${catalogName} properties ( - 'type'='hms' + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' );""" exception "denied" } @@ -62,7 +63,8 @@ suite("test_hive_base_case_auth", "p0,auth_call") { assertTrue(ctl_res.size() == 1) } sql """create catalog if not exists ${catalogName} properties ( - 'type'='hms' + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' );""" sql """grant Create_priv on ${catalogName}.*.* to ${user}""" try_sql """drop catalog if exists ${catalogName}""" diff --git a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy index 50cd6f1279ab50..87edd8c6c31628 100644 --- a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy @@ -94,9 +94,15 @@ suite("test_catalogs_tvf", "p0,external") { // The two removed properties, kept verbatim for restoration (do not delete): // "hive.metastore.type" = "dlf", // "dlf.proxy.mode" = "DLF_ONLY", + // Removing "hive.metastore.type" turned this back into a plain thrift hms catalog, and such a + // catalog must name its metastore -- an hms catalog with no address can only ever fail on first + // use, so CREATE CATALOG refuses it. The address below is never dialled: this suite reads the + // catalog's properties through catalogs() and never queries it. Drop this line again together + // with restoring the two properties above. sql """ CREATE CATALOG catalog_tvf_test_dlf PROPERTIES ( "type"="hms", + "hive.metastore.uris" = "thrift://127.0.0.1:7004", "dlf.endpoint" = "dlf-vpc.cn-beijing.aliyuncs.com", "dlf.region" = "cn-beijing", "dlf.uid" = "123456789", From c1bab9e8cf80fc5e0c52914e9d4aa69717ea2e7a Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 7 Aug 2026 09:13:57 +0800 Subject: [PATCH 11/12] [refactor](be) move orc related serde method to specific datatypes (#66515) ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../data_type_serde/data_type_array_serde.cpp | 54 +- .../data_type_datetimev2_serde.cpp | 61 ++ .../data_type_datev2_serde.cpp | 43 + .../data_type_decimal_serde.cpp | 122 +++ .../data_type_serde/data_type_map_serde.cpp | 68 +- .../data_type_nullable_serde.cpp | 47 + .../data_type_number_serde.cpp | 128 +++ .../core/data_type_serde/data_type_serde.cpp | 901 ------------------ .../data_type_string_serde.cpp | 99 ++ .../data_type_struct_serde.cpp | 64 ++ .../data_type_timestamptz_serde.cpp | 51 + .../core/data_type_serde/orc_serde_utils.cpp | 161 ++++ be/src/core/data_type_serde/orc_serde_utils.h | 40 + .../transformer/vorc_transformer_test.cpp | 27 +- 14 files changed, 959 insertions(+), 907 deletions(-) create mode 100644 be/src/core/data_type_serde/orc_serde_utils.cpp diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp b/be/src/core/data_type_serde/data_type_array_serde.cpp index b63f081e594e16..bfab0f1eddbe5a 100644 --- a/be/src/core/data_type_serde/data_type_array_serde.cpp +++ b/be/src/core/data_type_serde/data_type_array_serde.cpp @@ -404,7 +404,7 @@ Status DataTypeArraySerDe::write_column_to_orc(const std::string& timezone, cons packed_nested_size, arena, options)); // String batches borrow their source bytes, but the packed column is local to this call; // keep only those borrowed leaves in the write Arena until Writer::add() consumes them. - copy_orc_string_data_to_arena(cur_batch->elements.get(), arena); + orc_serde_utils::copy_orc_string_data_to_arena(cur_batch->elements.get(), arena); cur_batch->elements->numElements = packed_nested_size; cur_batch->numElements = end - start; @@ -627,4 +627,56 @@ bool DataTypeArraySerDe::write_column_to_hive_text(const IColumn& column, Buffer return true; } +namespace { + +Status decode_list_orc_values(const DataTypeSerDeSPtr& nested_serde, IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_list = dynamic_cast(orc_view.batch); + if (orc_list == nullptr) { + return Status::InternalError("Unexpected ORC list batch type {}", + orc_view.batch->toString()); + } + DORIS_CHECK(orc_view.file_type != nullptr); + DORIS_CHECK(orc_view.selected_type != nullptr); + DORIS_CHECK(orc_view.file_type->getSubtypeCount() == 1); + DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == 1); + DORIS_CHECK(orc_list->elements != nullptr); + const auto* file_element_type = orc_view.file_type->getSubtype(0); + const auto* selected_element_type = orc_view.selected_type->getSubtype(0); + DORIS_CHECK(file_element_type != nullptr); + DORIS_CHECK(selected_element_type != nullptr); + + auto& array_column = assert_cast(nested_column); + size_t element_size = 0; + std::vector element_selection; + RETURN_IF_ERROR(orc_serde_utils::append_orc_offsets( + array_column.get_offsets(), orc_list->offsets, orc_view.rows, &element_size, + orc_view.selected_rows, &element_selection)); + auto element_column = array_column.get_data_ptr()->assert_mutable(); + const auto child_rows = orc_view.selected_rows == nullptr + ? element_size + : static_cast(orc_list->elements->numElements); + const auto* child_selection = orc_view.selected_rows == nullptr ? nullptr : &element_selection; + auto child_view = orc_serde_utils::make_child_orc_view( + orc_view, file_element_type, selected_element_type, orc_list->elements.get(), + child_rows, child_selection); + RETURN_IF_ERROR( + orc_serde_utils::read_orc_child_column(nested_serde, element_column, child_view)); + array_column.get_data_ptr() = std::move(element_column); + return Status::OK(); +} + +} // namespace + +Status DataTypeArraySerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::LIST); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_list_orc_values(nested_serde, column, view); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 7b5ad09391676c..78370a858c916a 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -31,6 +31,7 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" @@ -51,6 +52,52 @@ static const int64_t micro_to_nano_second = 1000; namespace { +Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view, + const cctz::time_zone& timezone) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC timestamp batch type {}", + orc_view.batch->toString()); + } + auto& data = assert_cast(nested_column).get_data(); + const size_t old_data_size = data.size(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + data.resize(old_data_size + output_rows); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + if (orc_serde_utils::orc_row_is_null(*orc_view.batch, source_row)) { + data[old_data_size + row] = DateV2Value {}; + continue; + } + auto& value = + reinterpret_cast&>(data[old_data_size + row]); + orc_serde_utils::RoundedOrcTimestamp timestamp; + auto status = orc_serde_utils::round_orc_timestamp_to_microseconds( + orc_batch->data[source_row], orc_batch->nanoseconds[source_row], ×tamp); + if (!status.ok()) { + data.resize(old_data_size); + return status; + } + value.from_unixtime(orc_batch->data[source_row], timezone); + if (!value.is_valid_date()) { + data.resize(old_data_size); + return Status::DataQualityError( + "Decoded ORC timestamp is outside the target timezone range"); + } + value.set_microsecond(timestamp.microseconds); + // Plain ORC TIMESTAMP is a civil value. Carry after timezone conversion so a fractional + // round does not jump backward or skip an hour at a daylight-saving transition. + if (timestamp.carry && + !value.date_add_interval(TimeInterval {TimeUnit::SECOND, 1, false})) { + data.resize(old_data_size); + return Status::DataQualityError( + "Decoded ORC timestamp is outside the target timezone range"); + } + } + return Status::OK(); +} + Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -926,4 +973,18 @@ template Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch( const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const; + +Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + const auto kind = view.file_type->getKind(); + DORIS_CHECK(kind == ::orc::TypeKind::TIMESTAMP || kind == ::orc::TypeKind::TIMESTAMP_INSTANT); + DORIS_CHECK(view.timezone != nullptr); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_timestamp_orc_values(column, view, *view.timezone); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp index 7b6b57fbd08c5c..be9c841d233d20 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp @@ -22,6 +22,7 @@ #include #include +#include #include "common/config.h" #include "core/column/column_const.h" @@ -30,11 +31,13 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datev2_impl.hpp" #include "exprs/function/cast/cast_to_string.h" +#include "storage/olap_common.h" namespace doris { @@ -43,6 +46,35 @@ static constexpr int32_t date_threshold = 719528; namespace { +constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528; + +Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC date batch type {}", + orc_view.batch->toString()); + } + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, DecodedValueKind::INT32); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector date_values; + date_values.resize(output_rows); + auto& date_dict = date_day_offset_dict::get(); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + const auto date = date_dict[cast_set(orc_batch->data[source_row])]; + date_values[row] = cast_set(date.daynr() - DORIS_DATE_EPOCH_DAYNR); + } + view.values = reinterpret_cast(date_values.data()); + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); +} + Status decode_parquet_date(int32_t encoded_date, DateV2Value* value) { DORIS_CHECK(value != nullptr); const int64_t day_number = static_cast(encoded_date) + date_threshold; @@ -696,4 +728,15 @@ template Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch( const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const; +Status DataTypeDateV2SerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DATE); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_date_orc_values(*this, column, view); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index 0b4452e3490350..99d32e0f1b8aec 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -37,6 +37,7 @@ #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" @@ -52,6 +53,115 @@ namespace doris { namespace { +constexpr int DECIMAL_PRECISION_FOR_HIVE11 = BeConsts::MAX_DECIMAL128_PRECISION; + +Int128 to_int128(::orc::Int128 value) { + const auto high_bits = static_cast<__uint128_t>(static_cast(value.getHighBits())); + const auto low_bits = static_cast<__uint128_t>(value.getLowBits()); + return static_cast((high_bits << 64) | low_bits); +} + +::orc::Int128 to_orc_int128(Int128 value) { + const auto unsigned_value = static_cast<__uint128_t>(value); + return ::orc::Int128(static_cast(static_cast(unsigned_value >> 64)), + static_cast(unsigned_value)); +} + +Status scale_decimal_value(Int128 value, int32_t source_scale, int32_t target_scale, + Int128* scaled_value) { + DORIS_CHECK(scaled_value != nullptr); + if (source_scale == target_scale) { + *scaled_value = value; + return Status::OK(); + } + if (source_scale < target_scale) { + bool overflow = false; + const auto scaled = ::orc::scaleUpInt128ByPowerOfTen(to_orc_int128(value), + target_scale - source_scale, overflow); + if (overflow) { + return Status::DataQualityError( + "ORC decimal value overflows when scaling from {} to {}", source_scale, + target_scale); + } + *scaled_value = to_int128(scaled); + return Status::OK(); + } + *scaled_value = to_int128( + ::orc::scaleDownInt128ByPowerOfTen(to_orc_int128(value), source_scale - target_scale)); + return Status::OK(); +} + +void fill_decimal_big_endian_value(Int128 value, std::array* bytes) { + DORIS_CHECK(bytes != nullptr); + const auto unsigned_value = static_cast<__uint128_t>(value); + for (size_t byte_idx = 0; byte_idx < bytes->size(); ++byte_idx) { + const auto shift = (bytes->size() - byte_idx - 1) * 8; + (*bytes)[byte_idx] = static_cast(unsigned_value >> shift); + } +} + +Status decode_decimal_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view, int32_t target_scale) { + DORIS_CHECK(orc_view.file_type != nullptr); + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, DecodedValueKind::FIXED_BINARY); + view.decimal_precision = orc_view.file_type->getPrecision() == 0 + ? DECIMAL_PRECISION_FOR_HIVE11 + : cast_set(orc_view.file_type->getPrecision()); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + view.fixed_length = sizeof(Int128); + + std::vector binary_values; + std::vector> decimal_values; + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + decimal_values.resize(output_rows); + binary_values.reserve(output_rows); + if (const auto* decimal64_batch = + dynamic_cast(orc_view.batch); + decimal64_batch != nullptr) { + view.decimal_scale = decimal64_batch->scale; + for (size_t row = 0; row < output_rows; ++row) { + Int128 value = 0; + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + if (!orc_serde_utils::orc_row_is_null(*orc_view.batch, source_row)) { + RETURN_IF_ERROR(scale_decimal_value(decimal64_batch->values[source_row], + decimal64_batch->scale, target_scale, &value)); + } + fill_decimal_big_endian_value(value, &decimal_values[row]); + binary_values.emplace_back(reinterpret_cast(decimal_values[row].data()), + decimal_values[row].size()); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); + } + + const auto* decimal128_batch = + dynamic_cast(orc_view.batch); + if (decimal128_batch == nullptr) { + return Status::InternalError("Unexpected ORC decimal batch type {}", + orc_view.batch->toString()); + } + view.decimal_scale = decimal128_batch->scale; + for (size_t row = 0; row < output_rows; ++row) { + Int128 value = 0; + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + if (!orc_serde_utils::orc_row_is_null(*orc_view.batch, source_row)) { + RETURN_IF_ERROR(scale_decimal_value(to_int128(decimal128_batch->values[source_row]), + decimal128_batch->scale, target_scale, &value)); + } + fill_decimal_big_endian_value(value, &decimal_values[row]); + binary_values.emplace_back(reinterpret_cast(decimal_values[row].data()), + decimal_values[row].size()); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); +} + template NativeType decode_big_endian_signed_integer(const uint8_t* data, int length) { if constexpr (std::is_same_v) { @@ -1360,6 +1470,18 @@ const uint8_t* DataTypeDecimalSerDe::deserialize_binary_to_field(const uint8_ return data; } +template +Status DataTypeDecimalSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DECIMAL); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_decimal_orc_values(*this, column, view, cast_set(scale)); +} + template class DataTypeDecimalSerDe; template class DataTypeDecimalSerDe; template class DataTypeDecimalSerDe; diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp b/be/src/core/data_type_serde/data_type_map_serde.cpp index cd1ec5cf984bc9..419006257a5b2a 100644 --- a/be/src/core/data_type_serde/data_type_map_serde.cpp +++ b/be/src/core/data_type_serde/data_type_map_serde.cpp @@ -36,6 +36,59 @@ namespace doris { class Arena; + +namespace { + +Status decode_map_orc_values(const DataTypeSerDeSPtr& key_serde, + const DataTypeSerDeSPtr& value_serde, IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_map = dynamic_cast(orc_view.batch); + if (orc_map == nullptr) { + return Status::InternalError("Unexpected ORC map batch type {}", + orc_view.batch->toString()); + } + DORIS_CHECK(orc_view.file_type != nullptr); + DORIS_CHECK(orc_view.selected_type != nullptr); + DORIS_CHECK(orc_view.file_type->getSubtypeCount() == 2); + DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == 2); + DORIS_CHECK(orc_map->keys != nullptr); + DORIS_CHECK(orc_map->elements != nullptr); + auto& map_column = assert_cast(nested_column); + size_t element_size = 0; + std::vector element_selection; + RETURN_IF_ERROR(orc_serde_utils::append_orc_offsets( + map_column.get_offsets(), orc_map->offsets, orc_view.rows, &element_size, + orc_view.selected_rows, &element_selection)); + const auto* file_key_type = orc_view.file_type->getSubtype(0); + const auto* selected_key_type = orc_view.selected_type->getSubtype(0); + DORIS_CHECK(file_key_type != nullptr); + DORIS_CHECK(selected_key_type != nullptr); + const auto child_rows = orc_view.selected_rows == nullptr + ? element_size + : static_cast(orc_map->keys->numElements); + const auto* child_selection = orc_view.selected_rows == nullptr ? nullptr : &element_selection; + auto key_column = map_column.get_keys_ptr()->assert_mutable(); + auto key_view = + orc_serde_utils::make_child_orc_view(orc_view, file_key_type, selected_key_type, + orc_map->keys.get(), child_rows, child_selection); + RETURN_IF_ERROR(orc_serde_utils::read_orc_child_column(key_serde, key_column, key_view)); + map_column.get_keys_ptr() = std::move(key_column); + const auto* file_value_type = orc_view.file_type->getSubtype(1); + const auto* selected_value_type = orc_view.selected_type->getSubtype(1); + DORIS_CHECK(file_value_type != nullptr); + DORIS_CHECK(selected_value_type != nullptr); + auto value_column = map_column.get_values_ptr()->assert_mutable(); + auto value_view = orc_serde_utils::make_child_orc_view( + orc_view, file_value_type, selected_value_type, orc_map->elements.get(), + orc_view.selected_rows == nullptr ? element_size + : static_cast(orc_map->elements->numElements), + child_selection); + RETURN_IF_ERROR(orc_serde_utils::read_orc_child_column(value_serde, value_column, value_view)); + map_column.get_values_ptr() = std::move(value_column); + return Status::OK(); +} + +} // namespace Status DataTypeMapSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, FormatOptions& options) const { @@ -484,8 +537,8 @@ Status DataTypeMapSerDe::write_column_to_orc(const std::string& timezone, const packed_nested_size, arena, options)); // String batches borrow their source bytes, but the packed columns are local to this call; // keep only those borrowed leaves in the write Arena until Writer::add() consumes them. - copy_orc_string_data_to_arena(cur_batch->keys.get(), arena); - copy_orc_string_data_to_arena(cur_batch->elements.get(), arena); + orc_serde_utils::copy_orc_string_data_to_arena(cur_batch->keys.get(), arena); + orc_serde_utils::copy_orc_string_data_to_arena(cur_batch->elements.get(), arena); cur_batch->keys->numElements = packed_nested_size; cur_batch->elements->numElements = packed_nested_size; @@ -722,4 +775,15 @@ bool DataTypeMapSerDe::write_column_to_hive_text(const IColumn& column, BufferWr return true; } +Status DataTypeMapSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::MAP); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_map_orc_values(key_serde, value_serde, column, view); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.cpp b/be/src/core/data_type_serde/data_type_nullable_serde.cpp index 2b4b6870581d76..c06bcf0af82837 100644 --- a/be/src/core/data_type_serde/data_type_nullable_serde.cpp +++ b/be/src/core/data_type_serde/data_type_nullable_serde.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/config.h" @@ -34,6 +35,7 @@ #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "exprs/function/cast/cast_base.h" #include "format/transformer/vcsv_transformer.h" #include "util/jsonb_document.h" @@ -42,6 +44,26 @@ namespace doris { class Arena; + +namespace { + +void append_orc_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, + const std::vector* selected_rows, NullMap* null_map) { + DORIS_CHECK(null_map != nullptr); + const auto output_rows = orc_serde_utils::orc_decode_row_count(rows, selected_rows); + const auto old_size = null_map->size(); + null_map->resize(old_size + output_rows); + if (batch.hasNulls) { + for (size_t row = 0; row < output_rows; ++row) { + (*null_map)[old_size + row] = + !batch.notNull[orc_serde_utils::orc_source_row_at(row, selected_rows)]; + } + return; + } + std::memset(null_map->data() + old_size, 0, output_rows); +} + +} // namespace Status DataTypeNullableSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, FormatOptions& options) const { @@ -591,4 +613,29 @@ Status DataTypeNullableSerDe::from_string_strict_mode(StringRef& str, IColumn& c null_column.get_null_map_data().push_back(0); return Status::OK(); } + +Status DataTypeNullableSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.selected_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == view.selected_type->getKind()); + auto& nullable_column = assert_cast(column); + const auto output_rows = orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows); + if (output_rows == 0) { + return Status::OK(); + } + + auto& null_map = nullable_column.get_null_map_data(); + const auto old_null_map_size = null_map.size(); + auto& nested_column = nullable_column.get_nested_column(); + const auto old_nested_size = nested_column.size(); + append_orc_null_map(*view.batch, view.rows, view.selected_rows, &null_map); + auto st = nested_serde->read_column_from_orc(nested_column, view); + if (!st.ok()) { + null_map.resize(old_null_map_size); + nested_column.resize(old_nested_size); + } + return st; +} } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index df9ccb529dae9a..ca30ec96d25310 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -23,7 +23,9 @@ #include #include #include +#include #include +#include #include "common/config.h" #include "common/exception.h" @@ -35,6 +37,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/packed_int128.h" #include "core/types.h" @@ -54,6 +57,95 @@ namespace doris { namespace { +template +void fill_selected_values(const SourceType* source_values, size_t rows, + const std::vector* selected_rows, + std::vector* selected_values) { + DORIS_CHECK(source_values != nullptr); + DORIS_CHECK(selected_values != nullptr); + const auto output_rows = orc_serde_utils::orc_decode_row_count(rows, selected_rows); + selected_values->resize(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + (*selected_values)[row] = + source_values[orc_serde_utils::orc_source_row_at(row, selected_rows)]; + } +} + +template +Status decode_fixed_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view, DecodedValueKind value_kind) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC scalar batch type {}", + orc_view.batch->toString()); + } + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, value_kind); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + std::vector selected_values; + if (orc_view.selected_rows == nullptr) { + view.values = reinterpret_cast(orc_batch->data.data()); + } else { + fill_selected_values(orc_batch->data.data(), orc_view.rows, orc_view.selected_rows, + &selected_values); + view.values = reinterpret_cast(selected_values.data()); + } + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_float_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC float batch type {}", + orc_view.batch->toString()); + } + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, DecodedValueKind::FLOAT); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector float_values; + float_values.resize(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + float_values[row] = static_cast( + orc_batch->data[orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows)]); + } + view.values = reinterpret_cast(float_values.data()); + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); +} + +Status decode_boolean_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC boolean batch type {}", + orc_view.batch->toString()); + } + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, DecodedValueKind::BOOL); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::unique_ptr bool_values = std::make_unique(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + bool_values[row] = + orc_batch->data[orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows)] != + 0; + } + view.values = reinterpret_cast(bool_values.get()); + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); +} + float parquet_half_to_float(uint16_t half) { const uint32_t sign = (half & 0x8000U) << 16; const uint32_t exponent = (half & 0x7C00U) >> 10; @@ -1747,6 +1839,42 @@ void DataTypeNumberSerDe::to_string_batch(const IColumn& column, ColumnString } } +template +Status DataTypeNumberSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + + if constexpr (T == TYPE_BOOLEAN) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::BOOLEAN); + return decode_boolean_orc_values(*this, column, view); + } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT) { + if constexpr (T == TYPE_TINYINT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::BYTE); + } else if constexpr (T == TYPE_SMALLINT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::SHORT); + } else if constexpr (T == TYPE_INT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::INT); + } else { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::LONG); + } + return decode_fixed_orc_values<::orc::LongVectorBatch, int64_t>(*this, column, view, + DecodedValueKind::INT64); + } else if constexpr (T == TYPE_FLOAT) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::FLOAT); + return decode_float_orc_values(*this, column, view); + } else if constexpr (T == TYPE_DOUBLE) { + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DOUBLE); + return decode_fixed_orc_values<::orc::DoubleVectorBatch, double>(*this, column, view, + DecodedValueKind::DOUBLE); + } + return DataTypeSerDe::read_column_from_orc(column, view); +} + /// Explicit template instantiations - to avoid code bloat in headers. template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index ba5426681c467a..ca5f2c206ada16 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -16,722 +16,28 @@ // under the License. #include "core/data_type_serde/data_type_serde.h" -#include - -#include -#include -#include -#include -#include #include -#include "common/cast_set.h" #include "common/check.h" -#include "common/consts.h" #include "common/exception.h" #include "common/status.h" #include "core/assert_cast.h" #include "core/column/column.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" #include "core/column/column_nullable.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" #include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" #include "core/data_type/storage_field_type.h" #include "core/data_type_serde/data_type_array_serde.h" -#include "core/data_type_serde/data_type_datetimev2_serde.h" -#include "core/data_type_serde/data_type_datev2_serde.h" #include "core/data_type_serde/data_type_decimal_serde.h" #include "core/data_type_serde/data_type_jsonb_serde.h" -#include "core/data_type_serde/data_type_nullable_serde.h" #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" -#include "core/data_type_serde/data_type_timestamptz_serde.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/field.h" -#include "core/types.h" -#include "core/value/timestamptz_value.h" -#include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_base.h" #include "runtime/descriptors.h" -#include "storage/olap_common.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" namespace doris { -namespace { - -constexpr int DECIMAL_PRECISION_FOR_HIVE11 = BeConsts::MAX_DECIMAL128_PRECISION; -constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528; - -bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row) { - return batch.hasNulls && !batch.notNull[row]; -} - -size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows) { - if (selected_rows == nullptr) { - return rows; - } - return selected_rows->size(); -} - -size_t orc_source_row_at(size_t row, const std::vector* selected_rows) { - if (selected_rows == nullptr) { - return row; - } - return (*selected_rows)[row]; -} - -DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view, - DecodedValueKind value_kind) { - DecodedColumnView view; - view.value_kind = value_kind; - view.row_count = cast_set(orc_decode_row_count(orc_view.rows, orc_view.selected_rows)); - view.timezone = orc_view.timezone; - return view; -} - -void fill_orc_decoded_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, - const std::vector* selected_rows, NullMap* null_map) { - DORIS_CHECK(null_map != nullptr); - if (!batch.hasNulls) { - return; - } - const auto output_rows = orc_decode_row_count(rows, selected_rows); - null_map->resize(output_rows); - for (size_t row = 0; row < output_rows; ++row) { - (*null_map)[row] = !batch.notNull[orc_source_row_at(row, selected_rows)]; - } -} - -void append_orc_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, - const std::vector* selected_rows, NullMap* null_map) { - DORIS_CHECK(null_map != nullptr); - const auto output_rows = orc_decode_row_count(rows, selected_rows); - const auto old_size = null_map->size(); - null_map->resize(old_size + output_rows); - if (batch.hasNulls) { - for (size_t row = 0; row < output_rows; ++row) { - (*null_map)[old_size + row] = !batch.notNull[orc_source_row_at(row, selected_rows)]; - } - return; - } - std::memset(null_map->data() + old_size, 0, output_rows); -} - -size_t trim_right_spaces(const char* value, size_t length) { - while (length > 0 && value[length - 1] == ' ') { - --length; - } - return length; -} - -Status append_orc_string_ref(const ::orc::Type& file_type, const char* data, int64_t length, - std::vector& binary_values) { - if (length < 0) { - return Status::Corruption("Invalid negative ORC string length {}", length); - } - auto value_length = static_cast(length); - if (file_type.getKind() == ::orc::TypeKind::CHAR) { - value_length = trim_right_spaces(data, value_length); - } - binary_values.emplace_back(value_length == 0 ? "" : data, value_length); - return Status::OK(); -} - -Int128 to_int128(::orc::Int128 value) { - const auto high_bits = static_cast<__uint128_t>(static_cast(value.getHighBits())); - const auto low_bits = static_cast<__uint128_t>(value.getLowBits()); - return static_cast((high_bits << 64) | low_bits); -} - -::orc::Int128 to_orc_int128(Int128 value) { - const auto unsigned_value = static_cast<__uint128_t>(value); - return ::orc::Int128(static_cast(static_cast(unsigned_value >> 64)), - static_cast(unsigned_value)); -} - -Status scale_decimal_value(Int128 value, int32_t source_scale, int32_t target_scale, - Int128* scaled_value) { - DORIS_CHECK(scaled_value != nullptr); - if (source_scale == target_scale) { - *scaled_value = value; - return Status::OK(); - } - if (source_scale < target_scale) { - bool overflow = false; - const auto scaled = ::orc::scaleUpInt128ByPowerOfTen(to_orc_int128(value), - target_scale - source_scale, overflow); - if (overflow) { - return Status::DataQualityError( - "ORC decimal value overflows when scaling from {} to {}", source_scale, - target_scale); - } - *scaled_value = to_int128(scaled); - return Status::OK(); - } - *scaled_value = to_int128( - ::orc::scaleDownInt128ByPowerOfTen(to_orc_int128(value), source_scale - target_scale)); - return Status::OK(); -} - -void fill_decimal_big_endian_value(Int128 value, std::array* bytes) { - DORIS_CHECK(bytes != nullptr); - const auto unsigned_value = static_cast<__uint128_t>(value); - for (size_t byte_idx = 0; byte_idx < bytes->size(); ++byte_idx) { - const auto shift = (bytes->size() - byte_idx - 1) * 8; - (*bytes)[byte_idx] = static_cast(unsigned_value >> shift); - } -} - -Status read_decoded_values(const DataTypeSerDe& serde, IColumn& column, DecodedColumnView* view) { - DORIS_CHECK(view != nullptr); - RETURN_IF_ERROR(serde.read_column_from_decoded_values(column, *view)); - return Status::OK(); -} - -template -void fill_selected_values(const SourceType* source_values, size_t rows, - const std::vector* selected_rows, - std::vector* selected_values) { - DORIS_CHECK(source_values != nullptr); - DORIS_CHECK(selected_values != nullptr); - const auto output_rows = orc_decode_row_count(rows, selected_rows); - selected_values->resize(output_rows); - for (size_t row = 0; row < output_rows; ++row) { - (*selected_values)[row] = source_values[orc_source_row_at(row, selected_rows)]; - } -} - -template -Status decode_fixed_orc_values(const DataTypeSerDe& serde, IColumn& column, - const OrcDecodedColumnView& orc_view, DecodedValueKind value_kind) { - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC scalar batch type {}", - orc_view.batch->toString()); - } - auto view = make_orc_decoded_view(orc_view, value_kind); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - std::vector selected_values; - if (orc_view.selected_rows == nullptr) { - view.values = reinterpret_cast(orc_batch->data.data()); - } else { - fill_selected_values(orc_batch->data.data(), orc_view.rows, orc_view.selected_rows, - &selected_values); - view.values = reinterpret_cast(selected_values.data()); - } - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); -} - -Status decode_float_orc_values(const DataTypeSerDe& serde, IColumn& column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC float batch type {}", - orc_view.batch->toString()); - } - auto view = make_orc_decoded_view(orc_view, DecodedValueKind::FLOAT); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - std::vector float_values; - float_values.resize(output_rows); - for (size_t row = 0; row < output_rows; ++row) { - float_values[row] = - static_cast(orc_batch->data[orc_source_row_at(row, orc_view.selected_rows)]); - } - view.values = reinterpret_cast(float_values.data()); - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); -} - -Status decode_boolean_orc_values(const DataTypeSerDe& serde, IColumn& column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC boolean batch type {}", - orc_view.batch->toString()); - } - auto view = make_orc_decoded_view(orc_view, DecodedValueKind::BOOL); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - std::unique_ptr bool_values = std::make_unique(output_rows); - for (size_t row = 0; row < output_rows; ++row) { - bool_values[row] = orc_batch->data[orc_source_row_at(row, orc_view.selected_rows)] != 0; - } - view.values = reinterpret_cast(bool_values.get()); - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); -} - -Status decode_string_orc_values(const DataTypeSerDe& serde, IColumn& column, - const OrcDecodedColumnView& orc_view) { - DORIS_CHECK(orc_view.file_type != nullptr); - if (const auto* encoded_batch = - dynamic_cast(orc_view.batch); - encoded_batch != nullptr && encoded_batch->isEncoded) { - if (encoded_batch->dictionary == nullptr) { - return Status::InternalError("Encoded ORC string batch has no dictionary"); - } - auto view = make_orc_decoded_view(orc_view, DecodedValueKind::BINARY); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, - &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - std::vector binary_values; - binary_values.reserve(output_rows); - for (size_t row = 0; row < output_rows; ++row) { - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - if (orc_row_is_null(*orc_view.batch, source_row)) { - binary_values.emplace_back("", 0); - continue; - } - char* data = nullptr; - int64_t length = 0; - encoded_batch->dictionary->getValueByIndex(encoded_batch->index[source_row], data, - length); - RETURN_IF_ERROR( - append_orc_string_ref(*orc_view.file_type, data, length, binary_values)); - } - view.binary_values = &binary_values; - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); - } - - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC string batch type {}", - orc_view.batch->toString()); - } - auto view = make_orc_decoded_view(orc_view, DecodedValueKind::BINARY); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - std::vector binary_values; - binary_values.reserve(output_rows); - for (size_t row = 0; row < output_rows; ++row) { - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - if (orc_row_is_null(*orc_view.batch, source_row)) { - binary_values.emplace_back("", 0); - continue; - } - RETURN_IF_ERROR(append_orc_string_ref(*orc_view.file_type, orc_batch->data[source_row], - orc_batch->length[source_row], binary_values)); - } - view.binary_values = &binary_values; - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); -} - -Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC date batch type {}", - orc_view.batch->toString()); - } - auto view = make_orc_decoded_view(orc_view, DecodedValueKind::INT32); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - std::vector date_values; - date_values.resize(output_rows); - auto& date_dict = date_day_offset_dict::get(); - for (size_t row = 0; row < output_rows; ++row) { - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - const auto date = date_dict[cast_set(orc_batch->data[source_row])]; - date_values[row] = cast_set(date.daynr() - DORIS_DATE_EPOCH_DAYNR); - } - view.values = reinterpret_cast(date_values.data()); - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); -} - -Status decode_decimal_orc_values(const DataTypeSerDe& serde, IColumn& column, - const OrcDecodedColumnView& orc_view, int32_t target_scale) { - DORIS_CHECK(orc_view.file_type != nullptr); - auto view = make_orc_decoded_view(orc_view, DecodedValueKind::FIXED_BINARY); - view.decimal_precision = orc_view.file_type->getPrecision() == 0 - ? DECIMAL_PRECISION_FOR_HIVE11 - : cast_set(orc_view.file_type->getPrecision()); - NullMap null_map; - fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, orc_view.selected_rows, &null_map); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - view.fixed_length = sizeof(Int128); - - std::vector binary_values; - std::vector> decimal_values; - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - decimal_values.resize(output_rows); - binary_values.reserve(output_rows); - if (const auto* decimal64_batch = - dynamic_cast(orc_view.batch); - decimal64_batch != nullptr) { - view.decimal_scale = decimal64_batch->scale; - for (size_t row = 0; row < output_rows; ++row) { - Int128 value = 0; - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - if (!orc_row_is_null(*orc_view.batch, source_row)) { - RETURN_IF_ERROR(scale_decimal_value(decimal64_batch->values[source_row], - decimal64_batch->scale, target_scale, &value)); - } - fill_decimal_big_endian_value(value, &decimal_values[row]); - binary_values.emplace_back(reinterpret_cast(decimal_values[row].data()), - decimal_values[row].size()); - } - view.binary_values = &binary_values; - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); - } - - const auto* decimal128_batch = - dynamic_cast(orc_view.batch); - if (decimal128_batch == nullptr) { - return Status::InternalError("Unexpected ORC decimal batch type {}", - orc_view.batch->toString()); - } - view.decimal_scale = decimal128_batch->scale; - for (size_t row = 0; row < output_rows; ++row) { - Int128 value = 0; - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - if (!orc_row_is_null(*orc_view.batch, source_row)) { - RETURN_IF_ERROR(scale_decimal_value(to_int128(decimal128_batch->values[source_row]), - decimal128_batch->scale, target_scale, &value)); - } - fill_decimal_big_endian_value(value, &decimal_values[row]); - binary_values.emplace_back(reinterpret_cast(decimal_values[row].data()), - decimal_values[row].size()); - } - view.binary_values = &binary_values; - RETURN_IF_ERROR(read_decoded_values(serde, column, &view)); - return Status::OK(); -} - -Status append_orc_offsets(ColumnArray::Offsets64& doris_offsets, - const ::orc::DataBuffer& orc_offsets, size_t rows, - size_t* element_size, const std::vector* selected_rows, - std::vector* element_selection) { - DORIS_CHECK(element_size != nullptr); - if (selected_rows != nullptr) { - DORIS_CHECK(element_selection != nullptr); - const auto prev_offset = doris_offsets.empty() ? 0 : doris_offsets.back(); - ColumnArray::Offset64 current_offset = prev_offset; - element_selection->clear(); - for (size_t row = 0; row < selected_rows->size(); ++row) { - const auto source_row = (*selected_rows)[row]; - DORIS_CHECK(source_row < rows); - const auto begin_offset = orc_offsets[source_row]; - const auto end_offset = orc_offsets[source_row + 1]; - if (end_offset < begin_offset) { - return Status::Corruption("Invalid ORC offsets"); - } - const auto delta = static_cast(end_offset - begin_offset); - for (size_t element_idx = 0; element_idx < delta; ++element_idx) { - element_selection->push_back(static_cast(begin_offset) + element_idx); - } - current_offset += static_cast(delta); - doris_offsets.push_back(current_offset); - } - *element_size = element_selection->size(); - return Status::OK(); - } - - const auto prev_offset = doris_offsets.empty() ? 0 : doris_offsets.back(); - const auto base_offset = orc_offsets[0]; - for (size_t idx = 1; idx <= rows; ++idx) { - const auto delta = orc_offsets[idx] - base_offset; - if (delta < 0) { - return Status::Corruption("Invalid ORC offsets"); - } - doris_offsets.push_back(prev_offset + static_cast(delta)); - } - const auto total_delta = orc_offsets[rows] - base_offset; - if (total_delta < 0) { - return Status::Corruption("Invalid ORC offsets"); - } - *element_size = static_cast(total_delta); - return Status::OK(); -} - -int64_t find_struct_child_index(const ::orc::Type& type, const std::string& field_name) { - DORIS_CHECK(type.getKind() == ::orc::TypeKind::STRUCT); - for (uint64_t child_idx = 0; child_idx < type.getSubtypeCount(); ++child_idx) { - if (type.getFieldName(child_idx) == field_name) { - return static_cast(child_idx); - } - } - return -1; -} - -struct RoundedOrcTimestamp { - int64_t seconds; - uint64_t microseconds; - bool carry; -}; - -Status round_orc_timestamp_to_microseconds(int64_t seconds, int64_t nanoseconds, - RoundedOrcTimestamp* result) { - constexpr int64_t NANOS_PER_SECOND = 1000000000; - constexpr int64_t NANOS_PER_MICROSECOND = 1000; - constexpr int64_t MICROS_PER_SECOND = 1000000; - DORIS_CHECK(result != nullptr); - DORIS_CHECK(nanoseconds >= 0 && nanoseconds < NANOS_PER_SECOND); - // Doris stores six fractional digits, so use half-up rounding and carry 999999500ns into the - // next second instead of silently truncating the ORC value. - const auto rounded_microseconds = - (nanoseconds + NANOS_PER_MICROSECOND / 2) / NANOS_PER_MICROSECOND; - // Validate the carry here, but validate Doris' calendar range after timezone conversion: - // a valid year-zero local timestamp may have a UTC epoch before year zero. - if (__builtin_add_overflow(seconds, rounded_microseconds / MICROS_PER_SECOND, - &result->seconds)) { - return Status::DataQualityError("ORC timestamp overflows after microsecond rounding"); - } - result->microseconds = cast_set(rounded_microseconds % MICROS_PER_SECOND); - result->carry = rounded_microseconds >= MICROS_PER_SECOND; - return Status::OK(); -} - -Status decode_timestamp_orc_values(IColumn& nested_column, const OrcDecodedColumnView& orc_view, - const cctz::time_zone& timezone) { - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC timestamp batch type {}", - orc_view.batch->toString()); - } - auto& data = assert_cast(nested_column).get_data(); - const size_t old_data_size = data.size(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - data.resize(old_data_size + output_rows); - for (size_t row = 0; row < output_rows; ++row) { - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - if (orc_row_is_null(*orc_view.batch, source_row)) { - data[old_data_size + row] = DateV2Value {}; - continue; - } - auto& value = - reinterpret_cast&>(data[old_data_size + row]); - RoundedOrcTimestamp timestamp; - auto status = round_orc_timestamp_to_microseconds( - orc_batch->data[source_row], orc_batch->nanoseconds[source_row], ×tamp); - if (!status.ok()) { - data.resize(old_data_size); - return status; - } - value.from_unixtime(orc_batch->data[source_row], timezone); - if (!value.is_valid_date()) { - data.resize(old_data_size); - return Status::DataQualityError( - "Decoded ORC timestamp is outside the target timezone range"); - } - value.set_microsecond(timestamp.microseconds); - // Plain ORC TIMESTAMP is a civil value. Carry after timezone conversion so a fractional - // round does not jump backward or skip an hour at a daylight-saving transition. - if (timestamp.carry && - !value.date_add_interval(TimeInterval {TimeUnit::SECOND, 1, false})) { - data.resize(old_data_size); - return Status::DataQualityError( - "Decoded ORC timestamp is outside the target timezone range"); - } - } - return Status::OK(); -} - -Status decode_timestamp_tz_orc_values(IColumn& nested_column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_batch = dynamic_cast(orc_view.batch); - if (orc_batch == nullptr) { - return Status::InternalError("Unexpected ORC timestamp batch type {}", - orc_view.batch->toString()); - } - auto& data = assert_cast(nested_column).get_data(); - const size_t old_data_size = data.size(); - const auto output_rows = orc_decode_row_count(orc_view.rows, orc_view.selected_rows); - data.resize(old_data_size + output_rows); - static const auto utc_time_zone = cctz::utc_time_zone(); - for (size_t row = 0; row < output_rows; ++row) { - const auto source_row = orc_source_row_at(row, orc_view.selected_rows); - if (orc_row_is_null(*orc_view.batch, source_row)) { - data[old_data_size + row] = TimestampTzValue {}; - continue; - } - auto& value = data[old_data_size + row]; - RoundedOrcTimestamp timestamp; - auto status = round_orc_timestamp_to_microseconds( - orc_batch->data[source_row], orc_batch->nanoseconds[source_row], ×tamp); - if (!status.ok()) { - data.resize(old_data_size); - return status; - } - value.from_unixtime(timestamp.seconds, utc_time_zone); - value.set_microsecond(timestamp.microseconds); - if (!value.is_valid_date()) { - data.resize(old_data_size); - return Status::DataQualityError( - "Decoded ORC TIMESTAMPTZ is outside the Doris 0000-9999 range"); - } - } - return Status::OK(); -} - -OrcDecodedColumnView make_child_orc_view(const OrcDecodedColumnView& parent_view, - const ::orc::Type* file_type, - const ::orc::Type* selected_type, - const ::orc::ColumnVectorBatch* batch, size_t rows, - const std::vector* selected_rows) { - OrcDecodedColumnView child_view = parent_view; - child_view.file_type = file_type; - child_view.selected_type = selected_type; - child_view.batch = batch; - child_view.rows = rows; - child_view.selected_rows = selected_rows; - return child_view; -} - -Status read_orc_child_column(const DataTypeSerDeSPtr& child_serde, MutableColumnPtr& child_column, - const OrcDecodedColumnView& child_view) { - DORIS_CHECK(child_serde != nullptr); - RETURN_IF_ERROR(child_serde->read_column_from_orc(*child_column, child_view)); - return Status::OK(); -} - -Status decode_list_orc_values(const DataTypeSerDeSPtr& nested_serde, IColumn& nested_column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_list = dynamic_cast(orc_view.batch); - if (orc_list == nullptr) { - return Status::InternalError("Unexpected ORC list batch type {}", - orc_view.batch->toString()); - } - DORIS_CHECK(orc_view.file_type != nullptr); - DORIS_CHECK(orc_view.selected_type != nullptr); - DORIS_CHECK(orc_view.file_type->getSubtypeCount() == 1); - DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == 1); - DORIS_CHECK(orc_list->elements != nullptr); - const auto* file_element_type = orc_view.file_type->getSubtype(0); - const auto* selected_element_type = orc_view.selected_type->getSubtype(0); - DORIS_CHECK(file_element_type != nullptr); - DORIS_CHECK(selected_element_type != nullptr); - - auto& array_column = assert_cast(nested_column); - size_t element_size = 0; - std::vector element_selection; - RETURN_IF_ERROR(append_orc_offsets(array_column.get_offsets(), orc_list->offsets, orc_view.rows, - &element_size, orc_view.selected_rows, &element_selection)); - auto element_column = array_column.get_data_ptr()->assert_mutable(); - const auto child_rows = orc_view.selected_rows == nullptr - ? element_size - : static_cast(orc_list->elements->numElements); - const auto* child_selection = orc_view.selected_rows == nullptr ? nullptr : &element_selection; - auto child_view = make_child_orc_view(orc_view, file_element_type, selected_element_type, - orc_list->elements.get(), child_rows, child_selection); - RETURN_IF_ERROR(read_orc_child_column(nested_serde, element_column, child_view)); - array_column.get_data_ptr() = std::move(element_column); - return Status::OK(); -} - -Status decode_map_orc_values(const DataTypeSerDeSPtr& key_serde, - const DataTypeSerDeSPtr& value_serde, IColumn& nested_column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_map = dynamic_cast(orc_view.batch); - if (orc_map == nullptr) { - return Status::InternalError("Unexpected ORC map batch type {}", - orc_view.batch->toString()); - } - DORIS_CHECK(orc_view.file_type != nullptr); - DORIS_CHECK(orc_view.selected_type != nullptr); - DORIS_CHECK(orc_view.file_type->getSubtypeCount() == 2); - DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == 2); - DORIS_CHECK(orc_map->keys != nullptr); - DORIS_CHECK(orc_map->elements != nullptr); - auto& map_column = assert_cast(nested_column); - size_t element_size = 0; - std::vector element_selection; - RETURN_IF_ERROR(append_orc_offsets(map_column.get_offsets(), orc_map->offsets, orc_view.rows, - &element_size, orc_view.selected_rows, &element_selection)); - - const auto* file_key_type = orc_view.file_type->getSubtype(0); - const auto* selected_key_type = orc_view.selected_type->getSubtype(0); - DORIS_CHECK(file_key_type != nullptr); - DORIS_CHECK(selected_key_type != nullptr); - const auto child_rows = orc_view.selected_rows == nullptr - ? element_size - : static_cast(orc_map->keys->numElements); - const auto* child_selection = orc_view.selected_rows == nullptr ? nullptr : &element_selection; - auto key_column = map_column.get_keys_ptr()->assert_mutable(); - auto key_view = make_child_orc_view(orc_view, file_key_type, selected_key_type, - orc_map->keys.get(), child_rows, child_selection); - RETURN_IF_ERROR(read_orc_child_column(key_serde, key_column, key_view)); - map_column.get_keys_ptr() = std::move(key_column); - - const auto* file_value_type = orc_view.file_type->getSubtype(1); - const auto* selected_value_type = orc_view.selected_type->getSubtype(1); - DORIS_CHECK(file_value_type != nullptr); - DORIS_CHECK(selected_value_type != nullptr); - auto value_column = map_column.get_values_ptr()->assert_mutable(); - auto value_view = make_child_orc_view( - orc_view, file_value_type, selected_value_type, orc_map->elements.get(), - orc_view.selected_rows == nullptr ? element_size - : static_cast(orc_map->elements->numElements), - child_selection); - RETURN_IF_ERROR(read_orc_child_column(value_serde, value_column, value_view)); - map_column.get_values_ptr() = std::move(value_column); - return Status::OK(); -} - -Status decode_struct_orc_values(const DataTypeSerDeSPtrs& elem_serdes_ptrs, IColumn& nested_column, - const OrcDecodedColumnView& orc_view) { - const auto* orc_struct = dynamic_cast(orc_view.batch); - if (orc_struct == nullptr) { - return Status::InternalError("Unexpected ORC struct batch type {}", - orc_view.batch->toString()); - } - DORIS_CHECK(orc_view.file_type != nullptr); - DORIS_CHECK(orc_view.selected_type != nullptr); - DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == orc_struct->fields.size()); - auto& struct_column = assert_cast(nested_column); - DORIS_CHECK(struct_column.tuple_size() == orc_view.selected_type->getSubtypeCount()); - DORIS_CHECK(elem_serdes_ptrs.size() == orc_view.selected_type->getSubtypeCount()); - - for (uint64_t selected_idx = 0; selected_idx < orc_view.selected_type->getSubtypeCount(); - ++selected_idx) { - const auto field_name = orc_view.selected_type->getFieldName(selected_idx); - const auto file_child_idx = find_struct_child_index(*orc_view.file_type, field_name); - if (file_child_idx < 0) { - return Status::InternalError("Selected ORC field {} is not in file struct", field_name); - } - const auto* file_child_type = - orc_view.file_type->getSubtype(static_cast(file_child_idx)); - const auto* selected_child_type = orc_view.selected_type->getSubtype(selected_idx); - DORIS_CHECK(file_child_type != nullptr); - DORIS_CHECK(selected_child_type != nullptr); - DORIS_CHECK(selected_idx < orc_struct->fields.size()); - auto child_column = - struct_column.get_column_ptr(static_cast(selected_idx))->assert_mutable(); - auto child_view = make_child_orc_view(orc_view, file_child_type, selected_child_type, - orc_struct->fields[selected_idx], orc_view.rows, - orc_view.selected_rows); - RETURN_IF_ERROR( - read_orc_child_column(elem_serdes_ptrs[selected_idx], child_column, child_view)); - struct_column.get_column_ptr(static_cast(selected_idx)) = std::move(child_column); - } - return Status::OK(); -} - -} // namespace - DataTypeSerDe::~DataTypeSerDe() = default; bool decoded_column_view_can_null_on_conversion_failure(const DecodedColumnView& view) { @@ -799,213 +105,6 @@ Status DataTypeSerDe::read_column_from_orc(IColumn& column, return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); } -Status DataTypeNullableSerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.selected_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == view.selected_type->getKind()); - auto& nullable_column = assert_cast(column); - const auto output_rows = orc_decode_row_count(view.rows, view.selected_rows); - if (output_rows == 0) { - return Status::OK(); - } - - auto& null_map = nullable_column.get_null_map_data(); - const auto old_null_map_size = null_map.size(); - auto& nested_column = nullable_column.get_nested_column(); - const auto old_nested_size = nested_column.size(); - append_orc_null_map(*view.batch, view.rows, view.selected_rows, &null_map); - auto st = nested_serde->read_column_from_orc(nested_column, view); - if (!st.ok()) { - null_map.resize(old_null_map_size); - nested_column.resize(old_nested_size); - } - return st; -} - -template -Status DataTypeNumberSerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - - if constexpr (T == TYPE_BOOLEAN) { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::BOOLEAN); - return decode_boolean_orc_values(*this, column, view); - } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || - T == TYPE_BIGINT) { - if constexpr (T == TYPE_TINYINT) { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::BYTE); - } else if constexpr (T == TYPE_SMALLINT) { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::SHORT); - } else if constexpr (T == TYPE_INT) { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::INT); - } else { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::LONG); - } - return decode_fixed_orc_values<::orc::LongVectorBatch, int64_t>(*this, column, view, - DecodedValueKind::INT64); - } else if constexpr (T == TYPE_FLOAT) { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::FLOAT); - return decode_float_orc_values(*this, column, view); - } else if constexpr (T == TYPE_DOUBLE) { - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DOUBLE); - return decode_fixed_orc_values<::orc::DoubleVectorBatch, double>(*this, column, view, - DecodedValueKind::DOUBLE); - } - return DataTypeSerDe::read_column_from_orc(column, view); -} - -template -Status DataTypeStringSerDeBase::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - const auto kind = view.file_type->getKind(); - DORIS_CHECK(kind == ::orc::TypeKind::STRING || kind == ::orc::TypeKind::BINARY || - kind == ::orc::TypeKind::VARCHAR || kind == ::orc::TypeKind::CHAR); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_string_orc_values(*this, column, view); -} - -template -Status DataTypeDecimalSerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DECIMAL); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_decimal_orc_values(*this, column, view, cast_set(scale)); -} - -Status DataTypeDateV2SerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::DATE); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_date_orc_values(*this, column, view); -} - -Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - const auto kind = view.file_type->getKind(); - DORIS_CHECK(kind == ::orc::TypeKind::TIMESTAMP || kind == ::orc::TypeKind::TIMESTAMP_INSTANT); - DORIS_CHECK(view.timezone != nullptr); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_timestamp_orc_values(column, view, *view.timezone); -} - -Status DataTypeTimeStampTzSerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::TIMESTAMP_INSTANT); - DORIS_CHECK(view.enable_mapping_timestamp_tz); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_timestamp_tz_orc_values(column, view); -} - -Status DataTypeArraySerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::LIST); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_list_orc_values(nested_serde, column, view); -} - -Status DataTypeMapSerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::MAP); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_map_orc_values(key_serde, value_serde, column, view); -} - -Status DataTypeStructSerDe::read_column_from_orc(IColumn& column, - const OrcDecodedColumnView& view) const { - DORIS_CHECK(view.file_type != nullptr); - DORIS_CHECK(view.batch != nullptr); - DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::STRUCT); - if (orc_decode_row_count(view.rows, view.selected_rows) == 0) { - return Status::OK(); - } - return decode_struct_orc_values(elem_serdes_ptrs, column, view); -} - -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeNumberSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; - -template Status DataTypeStringSerDeBase::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeStringSerDeBase::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeStringSerDeBase::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; - -template Status DataTypeDecimalSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeDecimalSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeDecimalSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeDecimalSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; -template Status DataTypeDecimalSerDe::read_column_from_orc( - IColumn& column, const OrcDecodedColumnView& view) const; - Status DataTypeSerDe::read_field_from_decoded_value(const IDataType& data_type, Field* field, const DecodedColumnView& view) const { DORIS_CHECK(field != nullptr); diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 24f7215d42e044..0fc469743e40ef 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -28,6 +28,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" @@ -36,6 +37,90 @@ namespace doris { namespace { +size_t trim_right_spaces(const char* value, size_t length) { + while (length > 0 && value[length - 1] == ' ') { + --length; + } + return length; +} + +Status append_orc_string_ref(const ::orc::Type& file_type, const char* data, int64_t length, + std::vector& binary_values) { + if (length < 0) { + return Status::Corruption("Invalid negative ORC string length {}", length); + } + auto value_length = static_cast(length); + if (file_type.getKind() == ::orc::TypeKind::CHAR) { + value_length = trim_right_spaces(data, value_length); + } + binary_values.emplace_back(value_length == 0 ? "" : data, value_length); + return Status::OK(); +} + +Status decode_string_orc_values(const DataTypeSerDe& serde, IColumn& column, + const OrcDecodedColumnView& orc_view) { + DORIS_CHECK(orc_view.file_type != nullptr); + if (const auto* encoded_batch = + dynamic_cast(orc_view.batch); + encoded_batch != nullptr && encoded_batch->isEncoded) { + if (encoded_batch->dictionary == nullptr) { + return Status::InternalError("Encoded ORC string batch has no dictionary"); + } + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, DecodedValueKind::BINARY); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector binary_values; + binary_values.reserve(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + if (orc_serde_utils::orc_row_is_null(*orc_view.batch, source_row)) { + binary_values.emplace_back("", 0); + continue; + } + char* data = nullptr; + int64_t length = 0; + encoded_batch->dictionary->getValueByIndex(encoded_batch->index[source_row], data, + length); + RETURN_IF_ERROR( + append_orc_string_ref(*orc_view.file_type, data, length, binary_values)); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); + } + + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC string batch type {}", + orc_view.batch->toString()); + } + auto view = orc_serde_utils::make_orc_decoded_view(orc_view, DecodedValueKind::BINARY); + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(*orc_view.batch, orc_view.rows, + orc_view.selected_rows, &null_map); + view.null_map = null_map.empty() ? nullptr : null_map.data(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + std::vector binary_values; + binary_values.reserve(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + if (orc_serde_utils::orc_row_is_null(*orc_view.batch, source_row)) { + binary_values.emplace_back("", 0); + continue; + } + RETURN_IF_ERROR(append_orc_string_ref(*orc_view.file_type, orc_batch->data[source_row], + orc_batch->length[source_row], binary_values)); + } + view.binary_values = &binary_values; + RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); + return Status::OK(); +} + template Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view) { if (view.binary_values == nullptr && decoded_column_view_has_non_null_value(view)) { @@ -772,6 +857,20 @@ Status DataTypeStringSerDeBase::from_olap_string(const std::string& return Status::OK(); } +template +Status DataTypeStringSerDeBase::read_column_from_orc( + IColumn& column, const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + const auto kind = view.file_type->getKind(); + DORIS_CHECK(kind == ::orc::TypeKind::STRING || kind == ::orc::TypeKind::BINARY || + kind == ::orc::TypeKind::VARCHAR || kind == ::orc::TypeKind::CHAR); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_string_orc_values(*this, column, view); +} + template class DataTypeStringSerDeBase; template class DataTypeStringSerDeBase; template class DataTypeStringSerDeBase; diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp b/be/src/core/data_type_serde/data_type_struct_serde.cpp index 15f40a54aa5887..8d7cf1fa30edb9 100644 --- a/be/src/core/data_type_serde/data_type_struct_serde.cpp +++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp @@ -28,6 +28,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/complex_type_deserialize_util.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/string_ref.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" @@ -36,6 +37,58 @@ namespace doris { class Arena; +namespace { + +int64_t find_struct_child_index(const ::orc::Type& type, const std::string& field_name) { + DORIS_CHECK(type.getKind() == ::orc::TypeKind::STRUCT); + for (uint64_t child_idx = 0; child_idx < type.getSubtypeCount(); ++child_idx) { + if (type.getFieldName(child_idx) == field_name) { + return static_cast(child_idx); + } + } + return -1; +} + +Status decode_struct_orc_values(const DataTypeSerDeSPtrs& elem_serdes_ptrs, IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_struct = dynamic_cast(orc_view.batch); + if (orc_struct == nullptr) { + return Status::InternalError("Unexpected ORC struct batch type {}", + orc_view.batch->toString()); + } + DORIS_CHECK(orc_view.file_type != nullptr); + DORIS_CHECK(orc_view.selected_type != nullptr); + DORIS_CHECK(orc_view.selected_type->getSubtypeCount() == orc_struct->fields.size()); + auto& struct_column = assert_cast(nested_column); + DORIS_CHECK(struct_column.tuple_size() == orc_view.selected_type->getSubtypeCount()); + DORIS_CHECK(elem_serdes_ptrs.size() == orc_view.selected_type->getSubtypeCount()); + for (uint64_t selected_idx = 0; selected_idx < orc_view.selected_type->getSubtypeCount(); + ++selected_idx) { + const auto field_name = orc_view.selected_type->getFieldName(selected_idx); + const auto file_child_idx = find_struct_child_index(*orc_view.file_type, field_name); + if (file_child_idx < 0) { + return Status::InternalError("Selected ORC field {} is not in file struct", field_name); + } + const auto* file_child_type = + orc_view.file_type->getSubtype(static_cast(file_child_idx)); + const auto* selected_child_type = orc_view.selected_type->getSubtype(selected_idx); + DORIS_CHECK(file_child_type != nullptr); + DORIS_CHECK(selected_child_type != nullptr); + DORIS_CHECK(selected_idx < orc_struct->fields.size()); + auto child_column = + struct_column.get_column_ptr(static_cast(selected_idx))->assert_mutable(); + auto child_view = orc_serde_utils::make_child_orc_view( + orc_view, file_child_type, selected_child_type, orc_struct->fields[selected_idx], + orc_view.rows, orc_view.selected_rows); + RETURN_IF_ERROR(orc_serde_utils::read_orc_child_column(elem_serdes_ptrs[selected_idx], + child_column, child_view)); + struct_column.get_column_ptr(static_cast(selected_idx)) = std::move(child_column); + } + return Status::OK(); +} + +} // namespace + std::string DataTypeStructSerDe::get_name() const { size_t size = elem_names.size(); std::stringstream s; @@ -667,4 +720,15 @@ bool DataTypeStructSerDe::write_column_to_hive_text(const IColumn& column, Buffe return true; } +Status DataTypeStructSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::STRUCT); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_struct_orc_values(elem_serdes_ptrs, column, view); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index e9968e0bc8748d..1e84fef5b05980 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -24,6 +24,7 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/orc_serde_utils.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/data_type_serde/parquet_timestamp.h" #include "core/value/timestamptz_value.h" @@ -35,6 +36,44 @@ namespace doris { namespace { +Status decode_timestamp_tz_orc_values(IColumn& nested_column, + const OrcDecodedColumnView& orc_view) { + const auto* orc_batch = dynamic_cast(orc_view.batch); + if (orc_batch == nullptr) { + return Status::InternalError("Unexpected ORC timestamp batch type {}", + orc_view.batch->toString()); + } + auto& data = assert_cast(nested_column).get_data(); + const size_t old_data_size = data.size(); + const auto output_rows = + orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); + data.resize(old_data_size + output_rows); + static const auto utc_time_zone = cctz::utc_time_zone(); + for (size_t row = 0; row < output_rows; ++row) { + const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); + if (orc_serde_utils::orc_row_is_null(*orc_view.batch, source_row)) { + data[old_data_size + row] = TimestampTzValue {}; + continue; + } + auto& value = data[old_data_size + row]; + orc_serde_utils::RoundedOrcTimestamp timestamp; + auto status = orc_serde_utils::round_orc_timestamp_to_microseconds( + orc_batch->data[source_row], orc_batch->nanoseconds[source_row], ×tamp); + if (!status.ok()) { + data.resize(old_data_size); + return status; + } + value.from_unixtime(timestamp.seconds, utc_time_zone); + value.set_microsecond(timestamp.microseconds); + if (!value.is_valid_date()) { + data.resize(old_data_size); + return Status::DataQualityError( + "Decoded ORC TIMESTAMPTZ is outside the Doris 0000-9999 range"); + } + } + return Status::OK(); +} + Status append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -604,4 +643,16 @@ void DataTypeTimeStampTzSerDe::write_one_cell_to_binary(const IColumn& src_colum data_ref.size); } +Status DataTypeTimeStampTzSerDe::read_column_from_orc(IColumn& column, + const OrcDecodedColumnView& view) const { + DORIS_CHECK(view.file_type != nullptr); + DORIS_CHECK(view.batch != nullptr); + DORIS_CHECK(view.file_type->getKind() == ::orc::TypeKind::TIMESTAMP_INSTANT); + DORIS_CHECK(view.enable_mapping_timestamp_tz); + if (orc_serde_utils::orc_decode_row_count(view.rows, view.selected_rows) == 0) { + return Status::OK(); + } + return decode_timestamp_tz_orc_values(column, view); +} + } // namespace doris diff --git a/be/src/core/data_type_serde/orc_serde_utils.cpp b/be/src/core/data_type_serde/orc_serde_utils.cpp new file mode 100644 index 00000000000000..5b7c88dbd63ce7 --- /dev/null +++ b/be/src/core/data_type_serde/orc_serde_utils.cpp @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/orc_serde_utils.h" + +#include "common/cast_set.h" +#include "common/check.h" +#include "core/column/column_array.h" + +namespace doris::orc_serde_utils { + +size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows) { + if (selected_rows == nullptr) { + return rows; + } + return selected_rows->size(); +} + +size_t orc_source_row_at(size_t row, const std::vector* selected_rows) { + if (selected_rows == nullptr) { + return row; + } + return (*selected_rows)[row]; +} + +bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row) { + return batch.hasNulls && !batch.notNull[row]; +} + +Status round_orc_timestamp_to_microseconds(int64_t seconds, int64_t nanoseconds, + RoundedOrcTimestamp* result) { + constexpr int64_t NANOS_PER_SECOND = 1000000000; + constexpr int64_t NANOS_PER_MICROSECOND = 1000; + constexpr int64_t MICROS_PER_SECOND = 1000000; + DORIS_CHECK(result != nullptr); + DORIS_CHECK(nanoseconds >= 0 && nanoseconds < NANOS_PER_SECOND); + // Doris stores six fractional digits, so use half-up rounding and carry 999999500ns into the + // next second instead of silently truncating the ORC value. + const auto rounded_microseconds = + (nanoseconds + NANOS_PER_MICROSECOND / 2) / NANOS_PER_MICROSECOND; + // Validate the carry here, but validate Doris' calendar range after timezone conversion: + // a valid year-zero local timestamp may have a UTC epoch before year zero. + if (__builtin_add_overflow(seconds, rounded_microseconds / MICROS_PER_SECOND, + &result->seconds)) { + return Status::DataQualityError("ORC timestamp overflows after microsecond rounding"); + } + result->microseconds = cast_set(rounded_microseconds % MICROS_PER_SECOND); + result->carry = rounded_microseconds >= MICROS_PER_SECOND; + return Status::OK(); +} + +DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view, + DecodedValueKind value_kind) { + DecodedColumnView view; + view.value_kind = value_kind; + view.row_count = cast_set(orc_decode_row_count(orc_view.rows, orc_view.selected_rows)); + view.timezone = orc_view.timezone; + return view; +} + +Status read_decoded_values(const DataTypeSerDe& serde, IColumn& column, DecodedColumnView* view) { + DORIS_CHECK(view != nullptr); + RETURN_IF_ERROR(serde.read_column_from_decoded_values(column, *view)); + return Status::OK(); +} + +void fill_orc_decoded_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, + const std::vector* selected_rows, NullMap* null_map) { + DORIS_CHECK(null_map != nullptr); + if (!batch.hasNulls) { + return; + } + const auto output_rows = orc_decode_row_count(rows, selected_rows); + null_map->resize(output_rows); + for (size_t row = 0; row < output_rows; ++row) { + (*null_map)[row] = !batch.notNull[orc_source_row_at(row, selected_rows)]; + } +} + +Status append_orc_offsets(ColumnArray::Offsets64& doris_offsets, + const ::orc::DataBuffer& orc_offsets, size_t rows, + size_t* element_size, const std::vector* selected_rows, + std::vector* element_selection) { + DORIS_CHECK(element_size != nullptr); + if (selected_rows != nullptr) { + DORIS_CHECK(element_selection != nullptr); + const auto prev_offset = doris_offsets.empty() ? 0 : doris_offsets.back(); + ColumnArray::Offset64 current_offset = prev_offset; + element_selection->clear(); + for (size_t row = 0; row < selected_rows->size(); ++row) { + const auto source_row = (*selected_rows)[row]; + DORIS_CHECK(source_row < rows); + const auto begin_offset = orc_offsets[source_row]; + const auto end_offset = orc_offsets[source_row + 1]; + if (end_offset < begin_offset) { + return Status::Corruption("Invalid ORC offsets"); + } + const auto delta = static_cast(end_offset - begin_offset); + for (size_t element_idx = 0; element_idx < delta; ++element_idx) { + element_selection->push_back(static_cast(begin_offset) + element_idx); + } + current_offset += static_cast(delta); + doris_offsets.push_back(current_offset); + } + *element_size = element_selection->size(); + return Status::OK(); + } + + const auto prev_offset = doris_offsets.empty() ? 0 : doris_offsets.back(); + const auto base_offset = orc_offsets[0]; + for (size_t idx = 1; idx <= rows; ++idx) { + const auto delta = orc_offsets[idx] - base_offset; + if (delta < 0) { + return Status::Corruption("Invalid ORC offsets"); + } + doris_offsets.push_back(prev_offset + static_cast(delta)); + } + const auto total_delta = orc_offsets[rows] - base_offset; + if (total_delta < 0) { + return Status::Corruption("Invalid ORC offsets"); + } + *element_size = static_cast(total_delta); + return Status::OK(); +} + +OrcDecodedColumnView make_child_orc_view(const OrcDecodedColumnView& parent_view, + const ::orc::Type* file_type, + const ::orc::Type* selected_type, + const ::orc::ColumnVectorBatch* batch, size_t rows, + const std::vector* selected_rows) { + OrcDecodedColumnView child_view = parent_view; + child_view.file_type = file_type; + child_view.selected_type = selected_type; + child_view.batch = batch; + child_view.rows = rows; + child_view.selected_rows = selected_rows; + return child_view; +} + +Status read_orc_child_column(const DataTypeSerDeSPtr& child_serde, MutableColumnPtr& child_column, + const OrcDecodedColumnView& child_view) { + DORIS_CHECK(child_serde != nullptr); + RETURN_IF_ERROR(child_serde->read_column_from_orc(*child_column, child_view)); + return Status::OK(); +} + +} // namespace doris::orc_serde_utils diff --git a/be/src/core/data_type_serde/orc_serde_utils.h b/be/src/core/data_type_serde/orc_serde_utils.h index 0803daab1b0e4b..06b9c4406eef74 100644 --- a/be/src/core/data_type_serde/orc_serde_utils.h +++ b/be/src/core/data_type_serde/orc_serde_utils.h @@ -19,10 +19,49 @@ #include #include +#include #include "core/arena.h" +#include "core/column/column_array.h" +#include "core/data_type_serde/data_type_serde.h" namespace doris { +namespace orc_serde_utils { + +size_t orc_decode_row_count(size_t rows, const std::vector* selected_rows); +size_t orc_source_row_at(size_t row, const std::vector* selected_rows); +bool orc_row_is_null(const ::orc::ColumnVectorBatch& batch, size_t row); + +struct RoundedOrcTimestamp { + int64_t seconds; + uint64_t microseconds; + bool carry; +}; + +Status round_orc_timestamp_to_microseconds(int64_t seconds, int64_t nanoseconds, + RoundedOrcTimestamp* result); + +DecodedColumnView make_orc_decoded_view(const OrcDecodedColumnView& orc_view, + DecodedValueKind value_kind); + +Status read_decoded_values(const DataTypeSerDe& serde, IColumn& column, DecodedColumnView* view); + +void fill_orc_decoded_null_map(const ::orc::ColumnVectorBatch& batch, size_t rows, + const std::vector* selected_rows, NullMap* null_map); + +Status append_orc_offsets(ColumnArray::Offsets64& doris_offsets, + const ::orc::DataBuffer& orc_offsets, size_t rows, + size_t* element_size, const std::vector* selected_rows, + std::vector* element_selection); + +OrcDecodedColumnView make_child_orc_view(const OrcDecodedColumnView& parent_view, + const ::orc::Type* file_type, + const ::orc::Type* selected_type, + const ::orc::ColumnVectorBatch* batch, size_t rows, + const std::vector* selected_rows); + +Status read_orc_child_column(const DataTypeSerDeSPtr& child_serde, MutableColumnPtr& child_column, + const OrcDecodedColumnView& child_view); inline void copy_orc_string_data_to_arena(orc::ColumnVectorBatch* batch, Arena& arena) { if (auto* strings = dynamic_cast(batch)) { @@ -84,4 +123,5 @@ inline void copy_orc_string_data_to_arena(orc::ColumnVectorBatch* batch, Arena& } } +} // namespace orc_serde_utils } // namespace doris diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp index ae87c60642748a..49af5e6624448a 100644 --- a/be/test/format/transformer/vorc_transformer_test.cpp +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -435,7 +435,7 @@ TEST(OrcSerdeUtilsTest, CopiesOnlyBorrowedStringData) { batch.length[1] = borrowed.size(); const size_t used_before_copy = arena.used_size(); - copy_orc_string_data_to_arena(&batch, arena); + orc_serde_utils::copy_orc_string_data_to_arena(&batch, arena); EXPECT_EQ(arena_owned, batch.data[0]); EXPECT_NE(borrowed.data(), batch.data[1]); @@ -450,7 +450,7 @@ TEST(OrcSerdeUtilsTest, PreservesEmptyStringAsPresentValue) { batch.data[0] = const_cast(""); batch.length[0] = 0; - copy_orc_string_data_to_arena(&batch, arena); + orc_serde_utils::copy_orc_string_data_to_arena(&batch, arena); EXPECT_NE(batch.data[0], nullptr); EXPECT_EQ(batch.length[0], 0); @@ -471,7 +471,7 @@ TEST(OrcSerdeUtilsTest, IgnoresBorrowedPayloadForNullString) { batch.length[1] = 0; const size_t used_before_copy = arena.used_size(); - copy_orc_string_data_to_arena(&batch, arena); + orc_serde_utils::copy_orc_string_data_to_arena(&batch, arena); EXPECT_EQ(used_before_copy, arena.used_size()); EXPECT_EQ(nullptr, batch.data[0]); @@ -479,6 +479,27 @@ TEST(OrcSerdeUtilsTest, IgnoresBorrowedPayloadForNullString) { EXPECT_NE(nullptr, batch.data[1]); } +TEST(OrcSerdeUtilsTest, AppliesSelectedRowsWhenBuildingNullMap) { + const std::vector selected_rows {3, 1}; + EXPECT_EQ(2, orc_serde_utils::orc_decode_row_count(4, &selected_rows)); + EXPECT_EQ(3, orc_serde_utils::orc_source_row_at(0, &selected_rows)); + EXPECT_EQ(1, orc_serde_utils::orc_source_row_at(1, &selected_rows)); + + orc::LongVectorBatch batch(4, *orc::getDefaultPool()); + batch.numElements = 4; + batch.hasNulls = true; + batch.notNull[0] = 1; + batch.notNull[1] = 0; + batch.notNull[2] = 1; + batch.notNull[3] = 1; + + NullMap null_map; + orc_serde_utils::fill_orc_decoded_null_map(batch, 4, &selected_rows, &null_map); + ASSERT_EQ(2, null_map.size()); + EXPECT_EQ(0, null_map[0]); + EXPECT_EQ(1, null_map[1]); +} + TEST_F(VOrcTransformerTest, PreservesNullableArrayStructChildPositions) { const auto int_type = make_nullable(std::make_shared()); const auto string_type = make_nullable(std::make_shared()); From 52d3bc4906eb601192dbf29851af2bf325a5caae Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 7 Aug 2026 10:01:56 +0800 Subject: [PATCH 12/12] [improvement](build) Fully exclude OBS/COS deps via --exclude-{obs,cos}-dependencies Turn the existing `--exclude-obs-dependencies` / `--exclude-cos-dependencies` flags from a scope=provided downgrade (which still resolved the jars from their remote repositories, only skipping the bundling step) into a full exclusion: the dependencies are no longer resolved, compiled, or bundled. This is driven by active-by-default Maven profiles (`obs` / `cos`, deactivated via -Ddisable.obs=true / -Ddisable.cos=true) that wrap every touch point: - fe-core: the fe-filesystem-obs / fe-filesystem-cos test couplings. - hadoop-deps: the hadoop-huaweicloud dependency and the Huawei OBS repository. - fe-filesystem: the fe-filesystem-obs / fe-filesystem-cos modules. build.sh maps the flags to -Ddisable.obs=true / -Ddisable.cos=true and drops the corresponding provider from both the -pl reactor list and the plugin packaging loop, so the build stays consistent when a provider is excluded. Default builds are unchanged (profiles active unless the flag is passed). Co-Authored-By: Claude Opus 4.8 (1M context) --- build.sh | 29 +++++- fe/be-java-extensions/hadoop-deps/pom.xml | 39 +++++--- fe/fe-core/pom.xml | 104 ++++++++++++++-------- fe/fe-filesystem/pom.xml | 39 +++++++- 4 files changed, 160 insertions(+), 51 deletions(-) diff --git a/build.sh b/build.sh index ceaa09a0779593..813c8bdfb08e7d 100755 --- a/build.sh +++ b/build.sh @@ -68,6 +68,11 @@ Usage: $0 --be-extension-ignore build be-java-extensions package, choose which modules to ignore. Multiple modules separated by commas. --enable-dynamic-arch enable dynamic CPU detection in OpenBLAS. Default ON. --disable-dynamic-arch disable dynamic CPU detection in OpenBLAS. + --exclude-obs-dependencies exclude all Huawei Cloud OBS (com.huaweicloud) dependencies and the + fe-filesystem-obs module; nothing from Huawei is resolved, compiled, + or bundled. Use when repo.huaweicloud.com is unreachable or forbidden. + --exclude-cos-dependencies exclude all Tencent Cloud COS dependencies and the fe-filesystem-cos + module; nothing from Tencent COS is resolved, compiled, or bundled. --clean clean and build target --compile-bench BE compile-speed benchmark: cold, cache-free BE-only build (fresh dedicated build dir, ccache disabled) with a per-phase @@ -802,6 +807,14 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then modules+=("fe-filesystem/fe-filesystem-api") modules+=("fe-filesystem/fe-filesystem-spi") for _fs_mod in s3-base s3 gcs minio ozone oss cos obs azure hdfs-base hdfs oss-hdfs jfs local broker http; do + # Skip the modules whose Maven profile is deactivated so the -pl list stays consistent with + # the reactor: obs is absent under -Ddisable.obs=true, cos under -Ddisable.cos=true. + if [[ "${_fs_mod}" == "obs" && "${BUILD_OBS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi + if [[ "${_fs_mod}" == "cos" && "${BUILD_COS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi if [[ -d "${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${_fs_mod}" ]]; then modules+=("fe-filesystem/fe-filesystem-${_fs_mod}") fi @@ -1070,10 +1083,14 @@ function build_fe_modules() { extra_mvn_opts=(${MVN_OPT}) fi if [[ "${BUILD_OBS_DEPENDENCIES}" -eq 0 ]]; then - dependency_mvn_opts+=("-Dobs.dependency.scope=provided") + # Deactivates the `obs` Maven profile in fe-core, hadoop-deps and fe-filesystem, so no + # com.huaweicloud artifact is resolved and the Huawei OBS module is not built or bundled. + dependency_mvn_opts+=("-Ddisable.obs=true") fi if [[ "${BUILD_COS_DEPENDENCIES}" -eq 0 ]]; then - dependency_mvn_opts+=("-Dcos.dependency.scope=provided") + # Deactivates the `cos` Maven profile in fe-core and fe-filesystem, so no Tencent COS + # artifact is resolved and the fe-filesystem-cos module is not built or bundled. + dependency_mvn_opts+=("-Ddisable.cos=true") fi if [[ -n "${USER_SETTINGS_MVN_REPO}" && -f "${USER_SETTINGS_MVN_REPO}" ]]; then user_settings_opts=(-gs "${USER_SETTINGS_MVN_REPO}") @@ -1191,6 +1208,14 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then if [ ! -d "${fs_module_dir}" ]; then continue fi + # These modules are not built when their Maven profile is deactivated, so their plugin zip + # does not exist; skip the unpack to keep packaging consistent with the reactor. + if [[ "${fs_module}" == "obs" && "${BUILD_OBS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi + if [[ "${fs_module}" == "cos" && "${BUILD_COS_DEPENDENCIES}" -eq 0 ]]; then + continue + fi mkdir -p "${fs_plugin_target}" # Unpack the self-contained plugin zip produced by maven-assembly-plugin. # Layout inside the zip: .jar at root + lib/*.jar for runtime deps. diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml b/fe/be-java-extensions/hadoop-deps/pom.xml index aba07779a80646..8c4c71aa486f8f 100644 --- a/fe/be-java-extensions/hadoop-deps/pom.xml +++ b/fe/be-java-extensions/hadoop-deps/pom.xml @@ -67,10 +67,8 @@ under the License. org.apache.hadoop.thirdparty hadoop-shaded-protobuf_3_25 - - com.huaweicloud - hadoop-huaweicloud - + org.apache.hadoop hadoop-client @@ -95,13 +93,32 @@ under the License. - - - - huawei-obs-sdk - https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ - - + + + + obs + + + !disable.obs + + + + + com.huaweicloud + hadoop-huaweicloud + + + + + + huawei-obs-sdk + https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ + + + + diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index b02fc027d1572e..404f0a2cac0715 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -70,6 +70,70 @@ under the License. ${env.FE_UT_PARALLEL} + + + obs + + + !disable.obs + + + + + ${project.groupId} + fe-filesystem-obs + ${project.version} + test + + + + com.squareup.okhttp3 + * + + + com.squareup.okio + * + + + + + + + + cos + + + !disable.cos + + + + + ${project.groupId} + fe-filesystem-cos + ${project.version} + test + + + + com.squareup.okhttp3 + * + + + com.squareup.okio + * + + + + + @@ -219,42 +283,10 @@ under the License. ${project.version} test - - ${project.groupId} - fe-filesystem-obs - ${project.version} - test - - - - com.squareup.okhttp3 - * - - - com.squareup.okio - * - - - - - ${project.groupId} - fe-filesystem-cos - ${project.version} - test - - - - com.squareup.okhttp3 - * - - - com.squareup.okio - * - - - + + ${project.groupId} fe-filesystem-gcs diff --git a/fe/fe-filesystem/pom.xml b/fe/fe-filesystem/pom.xml index 15392808f1a226..88fe040e7cb880 100644 --- a/fe/fe-filesystem/pom.xml +++ b/fe/fe-filesystem/pom.xml @@ -65,8 +65,10 @@ under the License. fe-filesystem-minio fe-filesystem-ozone fe-filesystem-oss - fe-filesystem-cos - fe-filesystem-obs + + fe-filesystem-local fe-filesystem-azure fe-filesystem-hdfs-base @@ -111,5 +113,38 @@ under the License. + + + + obs + + + !disable.obs + + + + fe-filesystem-obs + + + + + cos + + + !disable.cos + + + + fe-filesystem-cos + + + +