diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index d2de00f39d19ff..87f3b550cd2dbd 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -25,8 +25,6 @@ DEFINE_String(deploy_mode, ""); DEFINE_mString(cloud_unique_id, ""); DEFINE_mString(meta_service_endpoint, ""); DEFINE_mBool(enable_meta_service_endpoint_consistency_check, "true"); -DEFINE_Bool(meta_service_use_load_balancer, "false"); -DEFINE_mInt32(meta_service_rpc_timeout_ms, "10000"); DEFINE_Bool(meta_service_connection_pooled, "true"); DEFINE_mInt64(meta_service_connection_pool_size, "20"); DEFINE_mInt32(meta_service_connection_age_base_seconds, "30"); @@ -107,8 +105,6 @@ DEFINE_mBool(enable_use_cloud_unique_id_from_fe, "true"); DEFINE_mBool(enable_cloud_tablet_report, "true"); -DEFINE_mInt32(delete_bitmap_rpc_retry_times, "25"); - DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "100"); DEFINE_mInt32(meta_service_conflict_error_retry_times, "10"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 40b41715825965..c52e09bc0987a3 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -58,7 +58,6 @@ DECLARE_mInt32(meta_service_connection_age_base_seconds); // // Only works when meta_service_endpoint is set to a single host. DECLARE_mInt32(meta_service_idle_connection_timeout_ms); -DECLARE_mInt32(meta_service_rpc_timeout_ms); DECLARE_mInt32(meta_service_rpc_retry_times); // default brpc timeout DECLARE_mInt32(meta_service_brpc_timeout_ms); @@ -147,8 +146,6 @@ DECLARE_mBool(enable_use_cloud_unique_id_from_fe); DECLARE_Bool(enable_cloud_tablet_report); -DECLARE_mInt32(delete_bitmap_rpc_retry_times); - DECLARE_mInt64(meta_service_rpc_reconnect_interval_ms); DECLARE_mInt32(meta_service_conflict_error_retry_times); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 7b6db1636234aa..c07a3d0c2a7691 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -343,8 +344,6 @@ DEFINE_mInt32(doris_scan_range_max_mb, "1024"); DEFINE_mInt32(doris_scanner_row_num, "16384"); // single read execute fragment row bytes DEFINE_mInt32(doris_scanner_row_bytes, "10485760"); -// single read execute fragment max run time millseconds -DEFINE_mInt32(doris_scanner_max_run_time_ms, "1000"); DEFINE_mInt32(doris_scanner_dynamic_interval_ms, "100"); // (Advanced) Maximum size of per-query receive-side buffer DEFINE_mInt32(exchg_node_buffer_size_bytes, "20485760"); @@ -509,9 +508,6 @@ DEFINE_mInt64(cumulative_compaction_min_deltas, "5"); DEFINE_mInt64(cumulative_compaction_max_deltas, "1000"); DEFINE_mInt32(cumulative_compaction_max_deltas_factor, "10"); -// This config can be set to limit thread number in multiget thread pool. -DEFINE_mInt32(multi_get_max_threads, "10"); - // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. DEFINE_mInt64(total_permits_for_compaction_score, "1000000"); @@ -686,13 +682,10 @@ DEFINE_Int32(fragment_mgr_async_work_pool_thread_num_min, "16"); DEFINE_Int32(fragment_mgr_async_work_pool_thread_num_max, "2048"); DEFINE_Int32(fragment_mgr_async_work_pool_queue_size, "4096"); -// Control the number of disks on the machine. If 0, this comes from the system settings. -DEFINE_Int32(num_disks, "0"); // The read size is the size of the reads sent to os. // There is a trade off of latency and throughout, trying to keep disks busy but // not introduce seeks. The literature seems to agree that with 8 MB reads, random // io and sequential io perform similarly. -DEFINE_Int32(read_size, "8388608"); // 8 * 1024 * 1024, Read Size (in bytes) DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes) // for pprof @@ -748,9 +741,6 @@ DEFINE_Int32(load_process_soft_mem_limit_percent, "80"); // memtable memory limiter will do nothing. DEFINE_Int32(load_process_safe_mem_permit_percent, "5"); -// If there are a lot of memtable memory, then wait them flush finished. -DEFINE_mDouble(load_max_wg_active_memtable_percent, "0.6"); - // result buffer cancelled time (unit: second) DEFINE_mInt32(result_buffer_cancelled_interval_time, "300"); @@ -978,9 +968,6 @@ DEFINE_Int32(send_batch_thread_pool_queue_size, "102400"); DEFINE_mInt32(max_segment_num_per_rowset, "1000"); DEFINE_mInt32(segment_compression_threshold_kb, "256"); -// Time to clean up useless JDBC connection pool cache -DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800"); - // Global bitmap cache capacity for aggregation cache, size in bytes DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600"); // The default delete bitmap cache is set to 100MB, @@ -1054,9 +1041,6 @@ DEFINE_mInt32(merged_hdfs_min_io_size, "8192"); // OrcReader DEFINE_mInt32(orc_natural_read_size_mb, "8"); -DEFINE_mInt64(big_column_size_buffer, "65535"); -DEFINE_mInt64(small_column_size_buffer, "100"); - // Perform the always_true check at intervals determined by runtime_filter_sampling_frequency DEFINE_mInt32(runtime_filter_sampling_frequency, "32"); DEFINE_mInt32(execution_max_rpc_timeout_sec, "3600"); @@ -1090,13 +1074,6 @@ DEFINE_mInt64(nodechannel_pending_queue_max_bytes, "67108864"); // The batch size for sending data by brpc streaming client DEFINE_mInt64(brpc_streaming_client_batch_bytes, "262144"); -// Max waiting time to wait the "plan fragment start" rpc. -// If timeout, the fragment will be cancelled. -// This parameter is usually only used when the FE loses connection, -// and the BE can automatically cancel the relevant fragment after the timeout, -// so as to avoid occupying the execution thread for a long time. -DEFINE_mInt32(max_fragment_start_wait_time_seconds, "30"); - DEFINE_mInt32(fragment_mgr_cancel_worker_interval_seconds, "1"); // Node role tag for backend. Mix role is the default role, and computation role have no @@ -1156,21 +1133,11 @@ DEFINE_Bool(enable_debug_points, "false"); DEFINE_Int32(pipeline_executor_size, "0"); DEFINE_Int32(blocking_pipeline_executor_size, "0"); -DEFINE_Bool(enable_workload_group_for_scan, "false"); -DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000"); - -// Whether use schema dict in backend side instead of MetaService side(cloud mode) -DEFINE_mBool(variant_use_cloud_schema_dict_cache, "true"); -DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "2048"); DEFINE_mInt32(variant_max_json_key_length, "255"); DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false"); DEFINE_mBool(variant_enable_duplicate_json_path_check, "false"); DEFINE_mBool(enable_vertical_compact_variant_subcolumns, "true"); DEFINE_mBool(enable_variant_doc_sparse_write_subcolumns, "true"); -// Maximum depth of nested arrays to track with NestedGroup -// Reserved for future use when NestedGroup expansion moves to storage layer -// Deeper arrays will be stored as JSONB -DEFINE_mInt32(variant_nested_group_max_depth, "3"); DEFINE_mBool(variant_nested_group_discard_scalar_on_conflict, "false"); DEFINE_Validator(variant_max_json_key_length, @@ -1207,9 +1174,6 @@ DEFINE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold, "1000"); DEFINE_mBool(enable_read_cache_file_directly, "false"); DEFINE_mBool(file_cache_enable_evict_from_other_queue_by_size, "true"); -// If true, evict the ttl cache using LRU when full. -// Otherwise, only expiration can evict ttl and new data won't add to cache when full. -DEFINE_Bool(enable_ttl_cache_evict_using_lru, "true"); DEFINE_mBool(enbale_dump_error_file, "false"); // limit the max size of error log on disk DEFINE_mInt64(file_cache_error_log_limit_bytes, "209715200"); // 200MB @@ -1240,7 +1204,6 @@ DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "180000"); DEFINE_mInt64(file_cache_background_ttl_info_update_interval_ms, "180000"); DEFINE_mInt64(file_cache_background_tablet_id_flush_interval_ms, "1000"); -DEFINE_mInt64(file_cache_background_ttl_gc_batch, "1000"); DEFINE_mInt64(file_cache_background_lru_dump_interval_ms, "60000"); // dump queue only if the queue update specific times through several dump intervals DEFINE_mInt64(file_cache_background_lru_dump_update_cnt_threshold, "1000"); @@ -1342,9 +1305,6 @@ DEFINE_mBool(allow_zero_date, "false"); DEFINE_Bool(allow_invalid_decimalv2_literal, "false"); DEFINE_mString(kerberos_ccache_path, "/tmp/"); DEFINE_mString(kerberos_krb5_conf_path, "/etc/krb5.conf"); -// Deprecated -DEFINE_mInt32(kerberos_refresh_interval_second, "43200"); - // JDK-8153057: avoid StackOverflowError thrown from the UncaughtExceptionHandler in thread "process reaper" DEFINE_mBool(jdk_process_reaper_use_default_stack_size, "true"); @@ -1460,9 +1420,6 @@ DEFINE_mBool(enable_column_type_check, "true"); // 128 MB DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728"); -// Default 300s, if its value <= 0, then log is disabled -DEFINE_mInt64(enable_debug_log_timeout_secs, "0"); - // Tolerance for the number of partition id 0 in rowset, default 0 DEFINE_Int32(ignore_invalid_partition_id_rowset_num, "0"); @@ -1578,8 +1535,6 @@ DEFINE_Validator(paimon_file_system_scheme_mappings, DEFINE_mInt32(thrift_client_open_num_tries, "1"); -DEFINE_Bool(enable_index_compaction, "false"); - // http scheme in S3Client to use. E.g. http or https DEFINE_String(s3_client_http_scheme, "http"); DEFINE_Validator(s3_client_http_scheme, [](const std::string& config) -> bool { @@ -1693,8 +1648,6 @@ DEFINE_mInt32(max_automatic_compaction_num_per_round, "64"); DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300"); DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10"); DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true"); -DEFINE_mInt32(schema_dict_cache_capacity, "4096"); - // whether to prune rows with delete sign = 1 in base compaction // ATTN: this config is only for test DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true"); @@ -1787,8 +1740,6 @@ DEFINE_mInt32(file_handles_deplenish_frequency_times, "3"); // clang-format off #ifdef BE_TEST -// test s3 -DEFINE_String(test_s3_resource, "resource"); DEFINE_String(test_s3_ak, "ak"); DEFINE_String(test_s3_sk, "sk"); DEFINE_String(test_s3_endpoint, "endpoint"); @@ -2095,6 +2046,37 @@ std::ostream& operator<<(std::ostream& out, const std::vector& v) { continue; \ } +// Keys that were previously valid BE config entries but have been removed. +// If a user still has any of these in be.conf / be_custom.conf after upgrade, +// we emit a warning so they know the value is no longer honored, but BE will +// continue to start. +static const std::set kDeprecatedConfigKeys = { + "doris_scanner_max_run_time_ms", + "multi_get_max_threads", + "num_disks", + "read_size", + "load_max_wg_active_memtable_percent", + "jdbc_connection_pool_cache_clear_time_sec", + "big_column_size_buffer", + "small_column_size_buffer", + "max_fragment_start_wait_time_seconds", + "enable_workload_group_for_scan", + "workload_group_scan_task_wait_timeout_ms", + "variant_use_cloud_schema_dict_cache", + "variant_threshold_rows_to_estimate_sparse_column", + "variant_nested_group_max_depth", + "enable_ttl_cache_evict_using_lru", + "file_cache_background_ttl_gc_batch", + "kerberos_refresh_interval_second", + "enable_debug_log_timeout_secs", + "enable_index_compaction", + "schema_dict_cache_capacity", + "test_s3_resource", + "meta_service_use_load_balancer", + "meta_service_rpc_timeout_ms", + "delete_bitmap_rpc_retry_times", +}; + // init conf fields bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_to_default) { Properties props; @@ -2123,6 +2105,14 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t SET_FIELD(it.second, std::vector, fill_conf_map, set_to_default); } + // Warn about any deprecated keys still present in the conf file. + for (const auto& kv : props.conf_map()) { + if (kDeprecatedConfigKeys.count(kv.first) > 0) { + LOG(WARNING) << "BE config '" << kv.first << "' in " << conf_file + << " is deprecated and no longer takes effect; please remove it."; + } + } + if (config::is_cloud_mode()) { auto st = config::set_config("enable_file_cache", "true", true, true); LOG(INFO) << "set config enable_file_cache " diff --git a/be/src/common/config.h b/be/src/common/config.h index 427282a4452bc4..97283a3cafce57 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -402,8 +402,6 @@ DECLARE_mInt32(doris_scan_range_max_mb); DECLARE_mInt32(doris_scanner_row_num); // single read execute fragment row bytes DECLARE_mInt32(doris_scanner_row_bytes); -// Deprecated. single read execute fragment max run time millseconds -DECLARE_mInt32(doris_scanner_max_run_time_ms); // Minimum interval in milliseconds between adaptive scanner concurrency adjustments DECLARE_mInt32(doris_scanner_dynamic_interval_ms); // (Advanced) Maximum size of per-query receive-side buffer @@ -578,9 +576,6 @@ DECLARE_mInt64(cumulative_compaction_min_deltas); DECLARE_mInt64(cumulative_compaction_max_deltas); DECLARE_mInt32(cumulative_compaction_max_deltas_factor); -// This config can be set to limit thread number in multiget thread pool. -DECLARE_mInt32(multi_get_max_threads); - // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. DECLARE_mInt64(total_permits_for_compaction_score); @@ -743,13 +738,10 @@ DECLARE_Int32(fragment_mgr_async_work_pool_thread_num_min); DECLARE_Int32(fragment_mgr_async_work_pool_thread_num_max); DECLARE_Int32(fragment_mgr_async_work_pool_queue_size); -// Control the number of disks on the machine. If 0, this comes from the system settings. -DECLARE_Int32(num_disks); // The read size is the size of the reads sent to os. // There is a trade off of latency and throughout, trying to keep disks busy but // not introduce seeks. The literature seems to agree that with 8 MB reads, random // io and sequential io perform similarly. -DECLARE_Int32(read_size); // 8 * 1024 * 1024, Read Size (in bytes) DECLARE_Int32(min_buffer_size); // 1024, The minimum read buffer size (in bytes) // for pprof @@ -808,9 +800,6 @@ DECLARE_Int32(load_process_soft_mem_limit_percent); // memtable memory limiter will do nothing. DECLARE_Int32(load_process_safe_mem_permit_percent); -// If there are a lot of memtable memory, then wait them flush finished. -DECLARE_mDouble(load_max_wg_active_memtable_percent); - // result buffer cancelled time (unit: second) DECLARE_mInt32(result_buffer_cancelled_interval_time); @@ -1036,9 +1025,6 @@ DECLARE_mInt32(max_segment_num_per_rowset); // segment_compression_threshold_kb. DECLARE_mInt32(segment_compression_threshold_kb); -// Time to clean up useless JDBC connection pool cache -DECLARE_mInt32(jdbc_connection_pool_cache_clear_time_sec); - // Global bitmap cache capacity for aggregation cache, size in bytes DECLARE_Int64(delete_bitmap_agg_cache_capacity); DECLARE_String(delete_bitmap_dynamic_agg_cache_limit); @@ -1114,8 +1100,6 @@ DECLARE_mInt32(merged_hdfs_min_io_size); // OrcReader DECLARE_mInt32(orc_natural_read_size_mb); -DECLARE_mInt64(big_column_size_buffer); -DECLARE_mInt64(small_column_size_buffer); DECLARE_mInt32(runtime_filter_sampling_frequency); DECLARE_mInt32(execution_max_rpc_timeout_sec); @@ -1154,13 +1138,6 @@ DECLARE_Bool(enable_brpc_connection_check); DECLARE_mInt64(brpc_connection_check_timeout_ms); -// Max waiting time to wait the "plan fragment start" rpc. -// If timeout, the fragment will be cancelled. -// This parameter is usually only used when the FE loses connection, -// and the BE can automatically cancel the relevant fragment after the timeout, -// so as to avoid occupying the execution thread for a long time. -DECLARE_mInt32(max_fragment_start_wait_time_seconds); - DECLARE_Int32(fragment_mgr_cancel_worker_interval_seconds); // Node role tag for backend. Mix role is the default role, and computation role have no @@ -1248,9 +1225,6 @@ DECLARE_mInt64(file_cache_evict_in_advance_batch_bytes); DECLARE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold); DECLARE_mBool(enable_read_cache_file_directly); DECLARE_Bool(file_cache_enable_evict_from_other_queue_by_size); -// If true, evict the ttl cache using LRU when full. -// Otherwise, only expiration can evict ttl and new data won't add to cache when full. -DECLARE_Bool(enable_ttl_cache_evict_using_lru); DECLARE_mBool(enbale_dump_error_file); // limit the max size of error log on disk DECLARE_mInt64(file_cache_error_log_limit_bytes); @@ -1281,7 +1255,6 @@ DECLARE_mInt64(file_cache_background_monitor_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms); DECLARE_mInt64(file_cache_background_ttl_info_update_interval_ms); DECLARE_mInt64(file_cache_background_tablet_id_flush_interval_ms); -DECLARE_mInt64(file_cache_background_ttl_gc_batch); DECLARE_Int32(file_cache_downloader_thread_num_min); DECLARE_Int32(file_cache_downloader_thread_num_max); // used to persist lru information before be reboot and load the info back @@ -1391,8 +1364,6 @@ DECLARE_mBool(allow_invalid_decimalv2_literal); DECLARE_mString(kerberos_ccache_path); // set krb5.conf path, use "/etc/krb5.conf" by default DECLARE_mString(kerberos_krb5_conf_path); -// the interval for renew kerberos ticket cache -DECLARE_mInt32(kerberos_refresh_interval_second); // JDK-8153057: avoid StackOverflowError thrown from the UncaughtExceptionHandler in thread "process reaper" DECLARE_mBool(jdk_process_reaper_use_default_stack_size); @@ -1420,12 +1391,6 @@ DECLARE_mInt64(lookup_connection_cache_capacity); // level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT DECLARE_mInt64(LZ4_HC_compression_level); -// Threshold of a column as sparse column -// Notice: TEST ONLY -DECLARE_mBool(variant_use_cloud_schema_dict_cache); -// Threshold to estimate a column is sparsed -// Notice: TEST ONLY -DECLARE_mInt64(variant_threshold_rows_to_estimate_sparse_column); // Max json key length in bytes when parsing json into variant subcolumns/jsonb. DECLARE_mInt32(variant_max_json_key_length); // Treat invalid json format str as string, instead of throwing exception if false @@ -1435,9 +1400,6 @@ DECLARE_mBool(variant_enable_duplicate_json_path_check); // Enable vertical compact subcolumns of variant column DECLARE_mBool(enable_vertical_compact_variant_subcolumns); DECLARE_mBool(enable_variant_doc_sparse_write_subcolumns); -// Maximum depth of nested arrays to track with NestedGroup -// Reserved for future use when NestedGroup expansion moves to storage layer -DECLARE_mInt32(variant_nested_group_max_depth); // When true, discard scalar data that conflicts with NestedGroup array // data at the same path. This simplifies compaction by always prioritizing // nested structure over scalar. When false, report an error on conflict. @@ -1554,8 +1516,6 @@ DECLARE_mInt32(variant_max_merged_tablet_schema_size); DECLARE_mInt64(local_exchange_buffer_mem_limit); -DECLARE_mInt64(enable_debug_log_timeout_secs); - DECLARE_mBool(enable_column_type_check); // Tolerance for the number of partition id 0 in rowset, default 0 @@ -1752,7 +1712,6 @@ DECLARE_mInt32(max_automatic_compaction_num_per_round); DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds); DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n); DECLARE_mBool(enable_check_tablet_delete_bitmap_score); -DECLARE_mInt32(schema_dict_cache_capacity); // whether to prune rows with delete sign = 1 in base compaction // ATTN: this config is only for test @@ -1833,8 +1792,6 @@ DECLARE_mBool(enable_cloud_make_rs_visible_on_be); DECLARE_mInt32(file_handles_deplenish_frequency_times); #ifdef BE_TEST -// test s3 -DECLARE_String(test_s3_resource); DECLARE_String(test_s3_ak); DECLARE_String(test_s3_sk); DECLARE_String(test_s3_endpoint); @@ -1935,6 +1892,8 @@ class Properties { // dump props to conf file Status dump(const std::string& conffile); + const std::map& conf_map() const { return file_conf_map; } + private: std::map file_conf_map; }; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 5c242bd25fade8..ac7d9c978c4047 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -749,11 +749,6 @@ public class Config extends ConfigBase { "Whether to allow colocate balance between all groups."}) public static boolean disable_colocate_balance_between_groups = false; - /** - * The default user resource publishing timeout. - */ - @Deprecated - @ConfField public static int meta_publish_timeout_ms = 1000; @ConfField public static boolean proxy_auth_enable = false; @ConfField public static String proxy_auth_magic_prefix = "x@8"; /** @@ -772,12 +767,6 @@ public class Config extends ConfigBase { public static int expr_depth_limit = 3000; // Configurations for backup and restore - /** - * Plugins' path for BACKUP and RESTORE operations. Currently deprecated. - */ - @Deprecated - @ConfField public static String backup_plugin_path = "/tools/trans_file_tool/trans_files.sh"; - // For forward compatibility, will be removed later. // check token when download image file. @ConfField public static boolean enable_token_check = true; @@ -844,14 +833,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static long max_bytes_per_broker_scanner = 500 * 1024 * 1024 * 1024L; // 500G - /** - * Max number of load jobs, include PENDING、ETL、LOADING、QUORUM_FINISHED. - * If exceed this number, load job is not allowed to be submitted. - */ - @Deprecated - @ConfField(mutable = true, masterOnly = true) - public static long max_unfinished_load_job = 1000; - /** * If set to true, Planner will try to select replica of tablet on same host as this Frontend. * This may reduce network transmission in following case: @@ -1014,12 +995,6 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = true) public static long tablet_schedule_interval_ms = 1000; - /** - * Deprecated after 0.10 - */ - @Deprecated - @ConfField public static boolean use_new_tablet_scheduler = true; - /** * the threshold of cluster balance score, if a backend's load score is 10% lower than average score, * this backend will be marked as LOW load, if load score is 10% higher than average score, HIGH load @@ -1692,10 +1667,6 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = true) public static int partition_info_update_interval_secs = 60; - @Deprecated - @ConfField(masterOnly = true) - public static boolean enable_concurrent_update = false; - /** * This configuration can only be configured during cluster initialization and cannot be modified during cluster * restart and upgrade after initialization is complete. @@ -1805,33 +1776,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static long min_bytes_indicate_replica_too_large = 2 * 1024 * 1024 * 1024L; - // statistics - /* - * the max unfinished statistics job number - */ - @Deprecated - @ConfField(mutable = true, masterOnly = true) - public static int cbo_max_statistics_job_num = 20; - /* - * the max timeout of a statistics task - */ - @Deprecated - @ConfField(mutable = true, masterOnly = true) - public static int max_cbo_statistics_task_timeout_sec = 300; - /* - * the concurrency of statistics task - */ - @Deprecated - @ConfField(mutable = false, masterOnly = true) - public static int cbo_concurrency_statistics_task_num = 10; - /* - * default sample percentage - * The value from 0 ~ 100. The 100 means no sampling and fetch all data. - */ - @Deprecated - @ConfField(mutable = true, masterOnly = true) - public static int cbo_default_sample_percentage = 10; - /* * the system automatically checks the time interval for statistics */ @@ -1953,15 +1897,6 @@ public class Config extends ConfigBase { @ConfField public static int async_task_consumer_thread_num = 64; - /** - * When job is finished, it will be saved in job manager for a while. - * This configuration is used to control the max saved time. - * Default is 3 days. - */ - @Deprecated - @ConfField - public static int finish_job_max_saved_second = 60 * 60 * 24 * 3; - // enable_workload_group should be immutable and temporarily set to mutable during the development test phase @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_workload_group = true; @@ -2009,13 +1944,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static boolean enable_decimal_conversion = true; - /** - * Support complex data type ARRAY. - */ - @Deprecated - @ConfField(mutable = true, masterOnly = true) - public static boolean enable_array_type = false; - /** * The timeout of executing async remote fragment. * In normal case, the async remote fragment will be executed in a short time. If system are under high load @@ -2577,10 +2505,6 @@ public class Config extends ConfigBase { + "not guarded for correctness reasons; see the NOTE in BaseAnalysisTask."}) public static long statistics_max_string_column_length = 1024; - @Deprecated - @ConfField - public static final int period_analyze_simultaneously_running_task_num = 1; - @ConfField(mutable = false) public static boolean allow_analyze_statistics_info_polluting_file_cache = true; @@ -2593,10 +2517,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true, description = {"The maximum number of partitions allowed for an Export job"}) public static int maximum_number_of_export_partitions = 2000; - @Deprecated - @ConfField(mutable = true, description = {"The maximum parallelism allowed for an Export job"}) - public static int maximum_parallelism_of_export_job = 50; - @ConfField(mutable = true, description = {"Whether to use MySQL's BIGINT type to return Doris's LARGEINT type"}) public static boolean use_mysql_bigint_for_largeint = false; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index d6b9b2b361504f..5d0d5d07a38295 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -160,6 +160,37 @@ private void initConf(String confFile) throws Exception { } replacedByEnv(props); setFields(props, isLdapConfig); + warnDeprecatedKeys(confFile, props); + } + + // Keys that were previously valid configuration entries but have been removed. + // If a user still has any of these in fe.conf after upgrade, we emit a warning + // so they know the value is no longer honored, but the FE will continue to start. + private static final Set DEPRECATED_CONFIG_KEYS = Sets.newHashSet( + "meta_publish_timeout_ms", + "backup_plugin_path", + "max_unfinished_load_job", + "use_new_tablet_scheduler", + "enable_concurrent_update", + "cbo_max_statistics_job_num", + "max_cbo_statistics_task_timeout_sec", + "cbo_concurrency_statistics_task_num", + "cbo_default_sample_percentage", + "finish_job_max_saved_second", + "enable_array_type", + "period_analyze_simultaneously_running_task_num", + "maximum_parallelism_of_export_job"); + + private static void warnDeprecatedKeys(String confFile, Properties props) { + for (String key : props.stringPropertyNames()) { + if (DEPRECATED_CONFIG_KEYS.contains(key)) { + String msg = String.format( + "FE config '%s' in %s is deprecated and no longer takes effect; " + + "please remove it.", + key, confFile); + System.err.println("[WARN] " + msg); + } + } } public static HashMap dump() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 54095c9e08a08a..0203b33fb62bf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -80,7 +80,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; - /** * System variable. **/ @@ -213,8 +212,6 @@ public class SessionVariable implements Serializable, Writable { // user can set instance num after exchange, no need to be equal to nums of before exchange public static final String PARALLEL_EXCHANGE_INSTANCE_NUM = "parallel_exchange_instance_num"; public static final String SHOW_HIDDEN_COLUMNS = "show_hidden_columns"; - public static final String USE_V2_ROLLUP = "use_v2_rollup"; - public static final String REWRITE_COUNT_DISTINCT_TO_BITMAP_HLL = "rewrite_count_distinct_to_bitmap_hll"; public static final String EVENT_SCHEDULER = "event_scheduler"; public static final String STORAGE_ENGINE = "storage_engine"; // Compatible with datagrip mysql @@ -251,7 +248,6 @@ public class SessionVariable implements Serializable, Writable { public static final String runtime_filter_wait_infinitely = "runtime_filter_wait_infinitely"; // Maximum number of bloom runtime filters allowed per query - public static final String RUNTIME_FILTERS_MAX_NUM = "runtime_filters_max_num"; // Runtime filter type used, For testing, Corresponds to TRuntimeFilterType public static final String RUNTIME_FILTER_TYPE = "runtime_filter_type"; // if the right table is greater than this value in the hash join, we will ignore IN filter @@ -272,8 +268,6 @@ public class SessionVariable implements Serializable, Writable { public static final String DELETE_WITHOUT_PARTITION = "delete_without_partition"; - public static final String ENABLE_VARIANT_ACCESS_IN_ORIGINAL_PLANNER = "enable_variant_access_in_original_planner"; - // set the default parallelism for send batch when execute InsertStmt operation, // if the value for parallelism exceed `max_send_batch_parallelism_per_job` in BE config, // then the coordinator be will use the value of `max_send_batch_parallelism_per_job` @@ -297,14 +291,10 @@ public class SessionVariable implements Serializable, Writable { public static final String GLOBAL_PARTITION_TOPN_THRESHOLD = "global_partition_topn_threshold"; - public static final String ENABLE_INFER_PREDICATE = "enable_infer_predicate"; - public static final long DEFAULT_INSERT_VISIBLE_TIMEOUT_MS = 60_000; public static final String ENABLE_VECTORIZED_ENGINE = "enable_vectorized_engine"; - public static final String EXTRACT_WIDE_RANGE_EXPR = "extract_wide_range_expr"; - // If user set a very small value, use this value instead. public static final long MIN_INSERT_VISIBLE_TIMEOUT_MS = 1000; @@ -361,8 +351,6 @@ public class SessionVariable implements Serializable, Writable { public static final String BLOCK_ENCRYPTION_MODE = "block_encryption_mode"; - public static final String AUTO_BROADCAST_JOIN_THRESHOLD = "auto_broadcast_join_threshold"; - public static final String PARALLEL_PREPARE_THRESHOLD = "parallel_prepare_threshold"; public static final String ENABLE_PROJECTION = "enable_projection"; @@ -411,8 +399,6 @@ public class SessionVariable implements Serializable, Writable { public static final String REWRITE_OR_TO_IN_PREDICATE_THRESHOLD = "rewrite_or_to_in_predicate_threshold"; - public static final String NEREIDS_STAR_SCHEMA_SUPPORT = "nereids_star_schema_support"; - public static final String NEREIDS_CBO_PENALTY_FACTOR = "nereids_cbo_penalty_factor"; public static final String ENABLE_NEREIDS_TRACE = "enable_nereids_trace"; public static final String ENABLE_EXPR_TRACE = "enable_expr_trace"; @@ -465,12 +451,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_PUSH_DOWN_NO_GROUP_AGG = "enable_push_down_no_group_agg"; - public static final String ENABLE_CBO_STATISTICS = "enable_cbo_statistics"; - public static final String ENABLE_SAVE_STATISTICS_SYNC_JOB = "enable_save_statistics_sync_job"; - public static final String ENABLE_ELIMINATE_SORT_NODE = "enable_eliminate_sort_node"; - public static final String NEREIDS_TRACE_EVENT_MODE = "nereids_trace_event_mode"; public static final String PARTITION_PRUNING_EXPAND_THRESHOLD = "partition_pruning_expand_threshold"; @@ -507,7 +489,6 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_NO_NEED_READ_DATA_OPT = "enable_no_need_read_data_opt"; public static final String GROUP_BY_AND_HAVING_USE_ALIAS_FIRST = "group_by_and_having_use_alias_first"; - public static final String DROP_TABLE_IF_CTAS_FAILED = "drop_table_if_ctas_failed"; public static final String MAX_TABLE_COUNT_USE_CASCADES_JOIN_REORDER = "max_table_count_use_cascades_join_reorder"; public static final int MIN_JOIN_REORDER_TABLE_COUNT = 2; @@ -520,8 +501,6 @@ public class SessionVariable implements Serializable, Writable { public static final String MINIDUMP_PATH = "minidump_path"; - public static final String TRACE_NEREIDS = "trace_nereids"; - public static final String PLAN_NEREIDS_DUMP = "plan_nereids_dump"; public static final String DUMP_NEREIDS_MEMO = "dump_nereids_memo"; @@ -607,9 +586,6 @@ public class SessionVariable implements Serializable, Writable { public static final String JDBC_CLICKHOUSE_QUERY_FINAL = "jdbc_clickhouse_query_final"; - public static final String ENABLE_JDBC_ORACLE_NULL_PREDICATE_PUSH_DOWN - = "enable_jdbc_oracle_null_predicate_push_down"; - public static final String ENABLE_JDBC_CAST_PREDICATE_PUSH_DOWN = "enable_jdbc_cast_predicate_push_down"; public static final String ENABLE_MEMTABLE_ON_SINK_NODE = @@ -729,9 +705,6 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_MATERIALIZED_VIEW_NEST_REWRITE = "enable_materialized_view_nest_rewrite"; - public static final String ENABLE_SYNC_MV_COST_BASED_REWRITE - = "enable_sync_mv_cost_based_rewrite"; - public static final String MATERIALIZED_VIEW_REWRITE_DURATION_THRESHOLD_MS = "materialized_view_rewrite_duration_threshold_ms"; @@ -795,7 +768,6 @@ public class SessionVariable implements Serializable, Writable { public static final String SHORT_CIRCUIT_EVALUATION = "short_circuit_evaluation"; - // NOTE: if you want to add some debug variables, please disable sql cache in `CacheAnalyzer.commonCacheCondition`, // and set affectQueryResult=true public static final List DEBUG_VARIABLES = ImmutableList.of( @@ -811,14 +783,10 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_STATS = "enable_stats"; - public static final String LIMIT_ROWS_FOR_SINGLE_INSTANCE = "limit_rows_for_single_instance"; - public static final String FETCH_REMOTE_SCHEMA_TIMEOUT_SECONDS = "fetch_remote_schema_timeout_seconds"; public static final String MAX_FETCH_REMOTE_TABLET_COUNT = "max_fetch_remote_schema_tablet_count"; - public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant"; - // Deprecated legacy switch for flatten-nested variant behavior. // It is distinct from variant_enable_nested_group. // Default is false, which means do not flatten nested when create table. @@ -919,7 +887,6 @@ public static int getHotValueCollectCount() { public static final String SKEW_VALUE_THRESHOLD = "skew_value_threshold"; - @VarAttrDef.VarAttr(name = SKEW_VALUE_THRESHOLD, needForward = true, description = {"当列中某个特定值的出现次数大于等于（rowCount/ndv）× skewValueThreshold 时，该值即被视为热点值", "When the occurrence of a value in a column is greater than " @@ -1059,11 +1026,6 @@ public static double getHotValueThreshold() { "Whether to add the FINAL keyword to the query SQL when querying ClickHouse JDBC external tables."}) public boolean jdbcClickhouseQueryFinal = false; - @VarAttrDef.VarAttr(name = ENABLE_JDBC_ORACLE_NULL_PREDICATE_PUSH_DOWN, needForward = true, - description = {"是否允许将 NULL 谓词下推到 Oracle JDBC 外部表。", - "Whether to allow NULL predicates to be pushed down to Oracle JDBC external tables."}) - public boolean enableJdbcOracleNullPredicatePushDown = false; - @VarAttrDef.VarAttr(name = ENABLE_JDBC_CAST_PREDICATE_PUSH_DOWN, needForward = true, description = {"是否允许将带有 CAST 表达式的谓词下推到 JDBC 外部表。", "Whether to allow predicates with CAST expressions to be pushed down to JDBC external tables."}) @@ -1373,7 +1335,6 @@ public void checkQuerySlotCount(String slotCnt) { setter = "setPipelineTaskNum") public int parallelPipelineTaskNum = 0; - public enum IgnoreSplitType { NONE, IGNORE_JNI, @@ -1462,7 +1423,6 @@ public enum IgnoreSplitType { ) public boolean skipPrunePredicate = false; - @VarAttrDef.VarAttr(name = ENABLE_SQL_CACHE, fuzzy = true) public boolean enableSqlCache = true; @@ -1490,12 +1450,6 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = FORWARD_TO_MASTER) public boolean forwardToMaster = true; - @VarAttrDef.VarAttr(name = USE_V2_ROLLUP) - public boolean useV2Rollup = false; - - @VarAttrDef.VarAttr(name = REWRITE_COUNT_DISTINCT_TO_BITMAP_HLL) - public boolean rewriteCountDistinct = true; - // compatible with some mysql client connect, say DataGrip of JetBrains @VarAttrDef.VarAttr(name = EVENT_SCHEDULER) public String eventScheduler = "OFF"; @@ -1531,12 +1485,6 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = SEND_BATCH_PARALLELISM, needForward = true) public int sendBatchParallelism = 1; - @VarAttrDef.VarAttr(name = ENABLE_VARIANT_ACCESS_IN_ORIGINAL_PLANNER) - public boolean enableVariantAccessInOriginalPlanner = false; - - @VarAttrDef.VarAttr(name = EXTRACT_WIDE_RANGE_EXPR, needForward = true) - public boolean extractWideRangeExpr = true; - @VarAttrDef.VarAttr(name = ENABLE_NEREIDS_DML, varType = VariableAnnotation.REMOVED) public boolean enableNereidsDML = true; @@ -1626,12 +1574,6 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = TRIM_TAILING_SPACES_FOR_EXTERNAL_TABLE_QUERY, needForward = true) public boolean trimTailingSpacesForExternalTableQuery = false; - // the maximum size in bytes for a table that will be broadcast to all be nodes - // when performing a join, By setting this value to -1 broadcasting can be disabled. - // Default value is 1Gto - @VarAttrDef.VarAttr(name = AUTO_BROADCAST_JOIN_THRESHOLD) - public double autoBroadcastJoinThreshold = 0.8; - @VarAttrDef.VarAttr(name = PARALLEL_PREPARE_THRESHOLD, fuzzy = true) public int parallelPrepareThreshold = 32; @@ -1669,7 +1611,6 @@ public enum IgnoreSplitType { "Ignore the rf when it encounters an error" }) public boolean ignoreRuntimeFilterError = false; - @VarAttrDef.VarAttr(name = RUNTIME_FILTER_MODE, needForward = true) private String runtimeFilterMode = "GLOBAL"; @@ -1688,9 +1629,6 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = runtime_filter_wait_infinitely, fuzzy = true, needForward = true) private boolean runtimeFilterWaitInfinitely = false; - @VarAttrDef.VarAttr(name = RUNTIME_FILTERS_MAX_NUM, needForward = true) - private int runtimeFiltersMaxNum = 10; - // Set runtimeFilterType to IN_OR_BLOOM filter @VarAttrDef.VarAttr(name = RUNTIME_FILTER_TYPE, fuzzy = true, needForward = true) private int runtimeFilterType = 12; @@ -1761,9 +1699,6 @@ public static boolean getTopNLazyMaterializationUsingIndex() { } } - @VarAttrDef.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT, needForward = true) - private boolean disableInvertedIndexV1ForVaraint = true; - @Deprecated @VarAttrDef.VarAttr(name = ENABLE_VARIANT_FLATTEN_NESTED, needForward = true) private boolean enableVariantFlattenNested = false; @@ -1871,9 +1806,6 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { @VarAttrDef.VarAttr(name = GLOBAL_PARTITION_TOPN_THRESHOLD) private double globalPartitionTopNThreshold = 100; - @VarAttrDef.VarAttr(name = ENABLE_INFER_PREDICATE) - private boolean enableInferPredicate = true; - @VarAttrDef.VarAttr(name = RETURN_OBJECT_DATA_AS_BINARY) private boolean returnObjectDataAsBinary = false; @@ -1921,14 +1853,6 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { + "(comma-separated integer list)"}) public String requiredGroupIds = ""; - @VarAttrDef.VarAttr(name = LIMIT_ROWS_FOR_SINGLE_INSTANCE, - description = {"当一个 ScanNode 上没有过滤条件，且 limit 值小于这个阈值时，" - + "系统会将这个算子的并发度调整为 1，以减少简单查询的扇出", - "When a ScanNode has no filter conditions and the limit value is less than this threshold, " - + "the system will adjust the concurrency of this operator to 1 " - + "to reduce the fan-out of simple queries"}) - public long limitRowsForSingleInstance = 10000; - public boolean isEnableLeftZigZag() { return enableLeftZigZag; } @@ -2014,9 +1938,6 @@ public boolean isEnableHboNonStrictMatchingMode() { @VarAttrDef.VarAttr(name = "filter_cost_factor", needForward = true) public double filterCostFactor = 0.0001; - @VarAttrDef.VarAttr(name = NEREIDS_STAR_SCHEMA_SUPPORT) - private boolean nereidsStarSchemaSupport = true; - @VarAttrDef.VarAttr(name = ENABLE_NEREIDS_DISTRIBUTE_PLANNER, needForward = true, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, description = { "使用新的 nereids 的分布式规划器的开关，这个分布式规划器可以规划出一些更高效的查询计划，比如在某些情况下，" @@ -2182,11 +2103,6 @@ public boolean isEnableHboNonStrictMatchingMode() { * The current statistics are only used for CBO test, * and are not available to users. (work in progress) */ - @VarAttrDef.VarAttr(name = ENABLE_CBO_STATISTICS) - public boolean enableCboStatistics = false; - - @VarAttrDef.VarAttr(name = ENABLE_ELIMINATE_SORT_NODE) - public boolean enableEliminateSortNode = true; @VarAttrDef.VarAttr(name = PARTITION_PRUNING_EXPAND_THRESHOLD, fuzzy = true) public int partitionPruningExpandThreshold = 10; @@ -2309,10 +2225,6 @@ public boolean isEnableHboNonStrictMatchingMode() { + "Format: db1.tbl1,db2.tbl2 or * for all MOR tables."}) public String readMorAsDupTables = ""; - // Whether drop table when create table as select insert data appear error. - @VarAttrDef.VarAttr(name = DROP_TABLE_IF_CTAS_FAILED, needForward = true) - public boolean dropTableIfCtasFailed = true; - @VarAttrDef.VarAttr(name = MAX_TABLE_COUNT_USE_CASCADES_JOIN_REORDER, needForward = true) public int maxTableCountUseCascadesJoinReorder = 10; @@ -2387,9 +2299,6 @@ public static boolean isEagerAggregationOnJoin() { @VarAttrDef.VarAttr(name = ENABLE_FOLD_NONDETERMINISTIC_FN) public boolean enableFoldNondeterministicFn = false; - @VarAttrDef.VarAttr(name = TRACE_NEREIDS) - public boolean traceNereids = false; - @VarAttrDef.VarAttr(name = PLAN_NEREIDS_DUMP) public boolean planNereidsDump = false; @@ -2498,7 +2407,6 @@ public static boolean isEagerAggregationOnJoin() { setter = "setOrcTinyStripeThresholdBytes") public long orcTinyStripeThresholdBytes = 8L * 1024L * 1024L; - @VarAttrDef.VarAttr( name = ORC_ONCE_MAX_READ_BYTES, fuzzy = true, @@ -2512,7 +2420,6 @@ public static boolean isEagerAggregationOnJoin() { setter = "setOrcOnceMaxReadBytes") public long orcOnceMaxReadBytes = 8L * 1024L * 1024L; - @VarAttrDef.VarAttr( name = ORC_MAX_MERGE_DISTANCE_BYTES, fuzzy = true, @@ -2525,7 +2432,6 @@ public static boolean isEagerAggregationOnJoin() { setter = "setOrcMaxMergeDistanceBytes") public long orcMaxMergeDistanceBytes = 1024L * 1024L; - @VarAttrDef.VarAttr( name = ENABLE_PARQUET_FILTER_BY_MIN_MAX, fuzzy = true, @@ -2740,7 +2646,6 @@ public static boolean isEagerAggregationOnJoin() { }) public long hugeTableDefaultSampleRows = 4194304; - @VarAttrDef.VarAttr(name = HUGE_TABLE_LOWER_BOUND_SIZE_IN_BYTES, flag = VarAttrDef.GLOBAL, description = { "大小超过该值的表将会自动通过采样收集统计信息", @@ -2851,11 +2756,6 @@ public static boolean isEagerAggregationOnJoin() { "Whether enable materialized view nest rewrite"}) public boolean enableMaterializedViewNestRewrite = false; - @VarAttrDef.VarAttr(name = ENABLE_SYNC_MV_COST_BASED_REWRITE, needForward = true, - description = {"是否允许基于代价改写同步物化视图", - "Whether enable cost based rewrite for sync mv"}, varType = VariableAnnotation.REMOVED) - public boolean enableSyncMvCostBasedRewrite = true; - @VarAttrDef.VarAttr(name = MATERIALIZED_VIEW_REWRITE_DURATION_THRESHOLD_MS, needForward = true, description = {"物化视图透明改写允许的最长耗时，超过此时长不再进行透明改写", "The maximum duration allowed for transparent rewriting of materialized views; " @@ -2867,7 +2767,6 @@ public static boolean isEagerAggregationOnJoin() { "The maximum number of partitions created during table creation"}) public int createTablePartitionMaxNum = 10000; - @VarAttrDef.VarAttr(name = HIVE_PARQUET_USE_COLUMN_NAMES, affectQueryResultInExecution = true, description = {"默认情况下按名称访问 Parquet 列。将此属性设置为“false”可按 Hive 表定义中的序号位置访问列。", "Access Parquet columns by name by default. Set this property to `false` to access columns " @@ -3216,7 +3115,6 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { "用于控制结果反序列化时 thrift 字段的最大值，当遇到类似\"MaxMessageSize reached\"这样的错误时可以考虑修改该参数"}) public int maxMsgSizeOfResultReceiver = TConfiguration.DEFAULT_MAX_MESSAGE_SIZE; - // CLOUD_VARIABLES_BEGIN @VarAttrDef.VarAttr(name = CLOUD_CLUSTER, alias = {COMPUTE_GROUP}) public String cloudCluster = ""; @@ -3658,7 +3556,6 @@ public boolean isEnableESParallelScroll() { ) public boolean defaultVariantEnableDocMode = false; - @VarAttrDef.VarAttr( name = DEFAULT_VARIANT_DOC_MATERIALIZATION_MIN_ROWS, needForward = true, @@ -4039,7 +3936,6 @@ public int getInsertTimeoutS() { return insertTimeoutS; } - public void setInsertTimeoutS(int insertTimeoutS) { this.insertTimeoutS = insertTimeoutS; } @@ -4348,7 +4244,6 @@ private long checkFieldLongValue(String variableName, long minValue, String valu return val; } - private int checkFieldValue(String variableName, int minValue, String value) throws Exception { int val = Integer.valueOf(value); if (val < minValue) { @@ -4583,22 +4478,7 @@ public boolean getForwardToMaster() { return forwardToMaster; } - public boolean isUseV2Rollup() { - return useV2Rollup; - } - // for unit test - public void setUseV2Rollup(boolean useV2Rollup) { - this.useV2Rollup = useV2Rollup; - } - - public boolean isRewriteCountDistinct() { - return rewriteCountDistinct; - } - - public void setRewriteCountDistinct(boolean rewriteCountDistinct) { - this.rewriteCountDistinct = rewriteCountDistinct; - } public String getEventScheduler() { return eventScheduler; @@ -4716,14 +4596,6 @@ public void setRuntimeFilterWaitTimeMs(int runtimeFilterWaitTimeMs) { this.runtimeFilterWaitTimeMs = runtimeFilterWaitTimeMs; } - public int getRuntimeFiltersMaxNum() { - return runtimeFiltersMaxNum; - } - - public void setRuntimeFiltersMaxNum(int runtimeFiltersMaxNum) { - this.runtimeFiltersMaxNum = runtimeFiltersMaxNum; - } - public int getRuntimeFilterType() { return runtimeFilterType; } @@ -4768,10 +4640,6 @@ public boolean getEnableLocalExchange() { return enableLocalExchange; } - public boolean getEnableCboStatistics() { - return enableCboStatistics; - } - public long getFileSplitSize() { return fileSplitSize; } @@ -4872,14 +4740,14 @@ public boolean isRetryOriginSqlOnConvertFail() { return retryOriginSqlOnConvertFail; } - public String[] getSqlConvertorFeatures() { - return enableSqlConvertorFeatures.split(","); - } - public String getSqlConvertorConfig() { return sqlConvertorConfig; } + public String[] getSqlConvertorFeatures() { + return enableSqlConvertorFeatures.split(","); + } + public Dialect getSqlParseDialect() { return Dialect.getByName(sqlDialect); } @@ -4938,10 +4806,6 @@ public boolean isDeleteWithoutPartition() { return deleteWithoutPartition; } - public boolean isExtractWideRangeExpr() { - return extractWideRangeExpr; - } - public boolean isGroupByAndHavingUseAliasFirst() { return groupByAndHavingUseAliasFirst; } @@ -5002,14 +4866,6 @@ public void setReturnObjectDataAsBinary(boolean returnObjectDataAsBinary) { this.returnObjectDataAsBinary = returnObjectDataAsBinary; } - public boolean isEnableInferPredicate() { - return enableInferPredicate; - } - - public void setEnableInferPredicate(boolean enableInferPredicate) { - this.enableInferPredicate = enableInferPredicate; - } - public boolean isEnableProjection() { return enableProjection; } @@ -5283,10 +5139,6 @@ public void setReadHiveJsonInOneColumn(boolean readHiveJsonInOneColumn) { this.readHiveJsonInOneColumn = readHiveJsonInOneColumn; } - public boolean isDropTableIfCtasFailed() { - return dropTableIfCtasFailed; - } - public void checkQueryTimeoutValid(String newQueryTimeout) { int value = Integer.valueOf(newQueryTimeout); if (value <= 0) { @@ -5338,12 +5190,10 @@ public void setEnableInvertedIndexQuery(boolean enableInvertedIndexQuery) { this.enableInvertedIndexQuery = enableInvertedIndexQuery; } - public boolean isEnableCommonExprPushdownForInvertedIndex() { return enableCommonExpPushDownForInvertedIndex; } - public void setEnableCommonExprPushdownForInvertedIndex(boolean enableCommonExpPushDownForInvertedIndex) { this.enableCommonExpPushDownForInvertedIndex = enableCommonExpPushDownForInvertedIndex; } @@ -5700,7 +5550,6 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, root.toString()); } - public void readFields(DataInput in) throws IOException { String json = Text.readString(in); readFromJson(json); @@ -5897,14 +5746,6 @@ public String getTraceId() { return ""; } - public boolean isTraceNereids() { - return traceNereids; - } - - public void setTraceNereids(boolean traceNereids) { - this.traceNereids = traceNereids; - } - public boolean isPlayNereidsDump() { return planNereidsDump; } @@ -6240,7 +6081,6 @@ public void setEnableMaterializedViewRewrite(boolean enableMaterializedViewRewri this.enableMaterializedViewRewrite = enableMaterializedViewRewrite; } - public String getPreMaterializedViewRewriteStrategy() { return preMaterializedViewRewriteStrategy; } @@ -6327,7 +6167,6 @@ public String getIgnoreSplitType() { return ignoreSplitType; } - public void checkIgnoreSplitType(String value) { try { IgnoreSplitType.valueOf(value); @@ -6337,7 +6176,6 @@ public void checkIgnoreSplitType(String value) { } } - public boolean getUseConsistentHashForExternalScan() { return useConsistentHashForExternalScan; } @@ -6400,14 +6238,6 @@ public boolean isUseSerialExchange() { return useSerialExchange && getEnableLocalExchange(); } - public void setDisableInvertedIndexV1ForVaraint(boolean disableInvertedIndexV1ForVaraint) { - this.disableInvertedIndexV1ForVaraint = disableInvertedIndexV1ForVaraint; - } - - public boolean getDisableInvertedIndexV1ForVaraint() { - return disableInvertedIndexV1ForVaraint; - } - @Deprecated public void setEnableVariantFlattenNested(boolean enableVariantFlattenNested) { this.enableVariantFlattenNested = enableVariantFlattenNested; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 42bea5743e54b3..607b3b9033a44a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -40,6 +40,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -127,6 +128,33 @@ public class VariableMgr { private static final Lock rlock = rwlock.readLock(); private static final Lock wlock = rwlock.writeLock(); + // Session variables that have been removed from SessionVariable.java but may still appear in + // user scripts, JDBC client connection init, or older replayed-edit logs. Looking up these + // names by SET or SELECT @@ silently no-ops instead of throwing ERR_UNKNOWN_SYSTEM_VARIABLE, + // so a BE-then-FE rolling upgrade does not break existing workloads. All entries in this set + // must use lowercase to match the case-insensitive comparison. + private static final ImmutableSet REMOVED_SESSION_VAR_NAMES = ImmutableSet.of( + "enable_jdbc_oracle_null_predicate_push_down", + "use_v2_rollup", + "rewrite_count_distinct_to_bitmap_hll", + "enable_variant_access_in_original_planner", + "extract_wide_range_expr", + "auto_broadcast_join_threshold", + "runtime_filters_max_num", + "disable_inverted_index_v1_for_variant", + "enable_infer_predicate", + "limit_rows_for_single_instance", + "nereids_star_schema_support", + "enable_cbo_statistics", + "enable_eliminate_sort_node", + "drop_table_if_ctas_failed", + "trace_nereids", + "enable_sync_mv_cost_based_rewrite"); + + private static boolean isRemovedSessionVar(String varName) { + return varName != null && REMOVED_SESSION_VAR_NAMES.contains(varName.toLowerCase()); + } + // Form map from variable name to its field in Java class. static { // Session value @@ -280,6 +308,13 @@ public static void setVar(SessionVariable sessionVariable, SetVar setVar) throws DdlException { VarContext varCtx = getVarContext(setVar.getVariable()); if (varCtx == null) { + // Silently ignore variables that have been removed from SessionVariable. + // This preserves backward compatibility for clients still issuing SET on the old name + // during a BE-then-FE rolling upgrade. + if (isRemovedSessionVar(setVar.getVariable())) { + LOG.debug("Ignoring removed session variable: {}", setVar.getVariable()); + return; + } // Check if the variable is in the MySQL compatibility whitelist if (isInMySQLCompatWhitelist(setVar.getVariable())) { // Silently ignore whitelisted variables for MySQL compatibility @@ -325,6 +360,10 @@ public static void setVarForNonMasterFE(SessionVariable sessionVariable, SetVar throws DdlException { VarContext varCtx = getVarContext(setVar.getVariable()); if (varCtx == null) { + if (isRemovedSessionVar(setVar.getVariable())) { + LOG.debug("Ignoring removed session variable: {}", setVar.getVariable()); + return; + } // Check if the variable is in the MySQL compatibility whitelist if (isInMySQLCompatWhitelist(setVar.getVariable())) { // Silently ignore whitelisted variables for MySQL compatibility @@ -506,6 +545,13 @@ public static void replayGlobalVariableV2(GlobalVarPersistInfo info, boolean fro public static void fillValue(SessionVariable var, VariableExpr desc) throws AnalysisException { VarContext ctx = getVarContext(desc.getName()); if (ctx == null) { + if (isRemovedSessionVar(desc.getName())) { + // Treat removed session variables as empty string for read access, + // preserving compatibility with clients that still SELECT @@. + desc.setType(Type.VARCHAR); + desc.setStringValue(""); + return; + } ErrorReport.reportAnalysisException(ErrorCode.ERR_UNKNOWN_SYSTEM_VARIABLE, desc.getName()); } @@ -583,6 +629,9 @@ private static void fillValue(Object obj, Field field, VariableExpr desc) { private static String getValue(SessionVariable var, String name, SetType setType) throws AnalysisException { VarContext ctx = ctxByVarName.get(name); if (ctx == null) { + if (isRemovedSessionVar(name)) { + return ""; + } ErrorReport.reportAnalysisException(ErrorCode.ERR_UNKNOWN_SYSTEM_VARIABLE, name); } @@ -606,6 +655,12 @@ public static String getValue(SessionVariable var, VariableExpr desc) throws Ana // For Nereids optimizer public static @Nullable Literal getLiteral(SessionVariable var, String name, SetType setType) { + if (isRemovedSessionVar(name)) { + // Match the no-op behavior used by setVar/getValue for removed session + // variables so that SELECT @@ doesn't break clients during + // a BE-then-FE rolling upgrade. + return new org.apache.doris.nereids.trees.expressions.literal.StringLiteral(""); + } VarContext ctx = getVarContext(name); if (ctx == null) { return null;