Skip to content

[chore](conf) Remove deprecated/unused FE/BE configs and stale session variables #63744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jacktengg wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[chore](conf) Remove deprecated/unused FE/BE configs and stale session variables #63744

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions be/src/cloud/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ DEFINE_String(deploy_mode, "");
DEFINE_mString(cloud_unique_id, "");
DEFINE_mString(meta_service_endpoint, "");
DEFINE_mBool(enable_meta_service_endpoint_consistency_check, "true");
DEFINE_Bool(meta_service_use_load_balancer, "false");
DEFINE_mInt32(meta_service_rpc_timeout_ms, "10000");
DEFINE_Bool(meta_service_connection_pooled, "true");
DEFINE_mInt64(meta_service_connection_pool_size, "20");
DEFINE_mInt32(meta_service_connection_age_base_seconds, "30");
Expand Down Expand Up @@ -107,8 +105,6 @@ DEFINE_mBool(enable_use_cloud_unique_id_from_fe, "true");

DEFINE_mBool(enable_cloud_tablet_report, "true");

DEFINE_mInt32(delete_bitmap_rpc_retry_times, "25");

DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "100");

DEFINE_mInt32(meta_service_conflict_error_retry_times, "10");
Expand Down
3 changes: 0 additions & 3 deletions be/src/cloud/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ DECLARE_mInt32(meta_service_connection_age_base_seconds);
//
// Only works when meta_service_endpoint is set to a single host.
DECLARE_mInt32(meta_service_idle_connection_timeout_ms);
DECLARE_mInt32(meta_service_rpc_timeout_ms);
DECLARE_mInt32(meta_service_rpc_retry_times);
// default brpc timeout
DECLARE_mInt32(meta_service_brpc_timeout_ms);
Expand Down Expand Up @@ -147,8 +146,6 @@ DECLARE_mBool(enable_use_cloud_unique_id_from_fe);

DECLARE_Bool(enable_cloud_tablet_report);

DECLARE_mInt32(delete_bitmap_rpc_retry_times);

DECLARE_mInt64(meta_service_rpc_reconnect_interval_ms);

DECLARE_mInt32(meta_service_conflict_error_retry_times);
Expand Down
90 changes: 40 additions & 50 deletions be/src/common/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include <memory>
#include <mutex>
#include <random>
#include <set>
#include <string>
#include <string_view>
#include <utility>
Expand Down Expand Up @@ -343,8 +344,6 @@ DEFINE_mInt32(doris_scan_range_max_mb, "1024");
DEFINE_mInt32(doris_scanner_row_num, "16384");
// single read execute fragment row bytes
DEFINE_mInt32(doris_scanner_row_bytes, "10485760");
// single read execute fragment max run time millseconds
DEFINE_mInt32(doris_scanner_max_run_time_ms, "1000");
DEFINE_mInt32(doris_scanner_dynamic_interval_ms, "100");
// (Advanced) Maximum size of per-query receive-side buffer
DEFINE_mInt32(exchg_node_buffer_size_bytes, "20485760");
Expand Down Expand Up @@ -509,9 +508,6 @@ DEFINE_mInt64(cumulative_compaction_min_deltas, "5");
DEFINE_mInt64(cumulative_compaction_max_deltas, "1000");
DEFINE_mInt32(cumulative_compaction_max_deltas_factor, "10");

// This config can be set to limit thread number in multiget thread pool.
DEFINE_mInt32(multi_get_max_threads, "10");

// The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction.
DEFINE_mInt64(total_permits_for_compaction_score, "1000000");

Expand Down Expand Up @@ -686,13 +682,10 @@ DEFINE_Int32(fragment_mgr_async_work_pool_thread_num_min, "16");
DEFINE_Int32(fragment_mgr_async_work_pool_thread_num_max, "2048");
DEFINE_Int32(fragment_mgr_async_work_pool_queue_size, "4096");

// Control the number of disks on the machine. If 0, this comes from the system settings.
DEFINE_Int32(num_disks, "0");
// The read size is the size of the reads sent to os.
// There is a trade off of latency and throughout, trying to keep disks busy but
// not introduce seeks. The literature seems to agree that with 8 MB reads, random
// io and sequential io perform similarly.
DEFINE_Int32(read_size, "8388608"); // 8 * 1024 * 1024, Read Size (in bytes)
DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes)

// for pprof
Expand Down Expand Up @@ -748,9 +741,6 @@ DEFINE_Int32(load_process_soft_mem_limit_percent, "80");
// memtable memory limiter will do nothing.
DEFINE_Int32(load_process_safe_mem_permit_percent, "5");

// If there are a lot of memtable memory, then wait them flush finished.
DEFINE_mDouble(load_max_wg_active_memtable_percent, "0.6");

// result buffer cancelled time (unit: second)
DEFINE_mInt32(result_buffer_cancelled_interval_time, "300");

Expand Down Expand Up @@ -978,9 +968,6 @@ DEFINE_Int32(send_batch_thread_pool_queue_size, "102400");
DEFINE_mInt32(max_segment_num_per_rowset, "1000");
DEFINE_mInt32(segment_compression_threshold_kb, "256");

// Time to clean up useless JDBC connection pool cache
DEFINE_mInt32(jdbc_connection_pool_cache_clear_time_sec, "28800");

// Global bitmap cache capacity for aggregation cache, size in bytes
DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600");
// The default delete bitmap cache is set to 100MB,
Expand Down Expand Up @@ -1054,9 +1041,6 @@ DEFINE_mInt32(merged_hdfs_min_io_size, "8192");

// OrcReader
DEFINE_mInt32(orc_natural_read_size_mb, "8");
DEFINE_mInt64(big_column_size_buffer, "65535");
DEFINE_mInt64(small_column_size_buffer, "100");

// Perform the always_true check at intervals determined by runtime_filter_sampling_frequency
DEFINE_mInt32(runtime_filter_sampling_frequency, "32");
DEFINE_mInt32(execution_max_rpc_timeout_sec, "3600");
Expand Down Expand Up @@ -1090,13 +1074,6 @@ DEFINE_mInt64(nodechannel_pending_queue_max_bytes, "67108864");
// The batch size for sending data by brpc streaming client
DEFINE_mInt64(brpc_streaming_client_batch_bytes, "262144");

// Max waiting time to wait the "plan fragment start" rpc.
// If timeout, the fragment will be cancelled.
// This parameter is usually only used when the FE loses connection,
// and the BE can automatically cancel the relevant fragment after the timeout,
// so as to avoid occupying the execution thread for a long time.
DEFINE_mInt32(max_fragment_start_wait_time_seconds, "30");

DEFINE_mInt32(fragment_mgr_cancel_worker_interval_seconds, "1");

// Node role tag for backend. Mix role is the default role, and computation role have no
Expand Down Expand Up @@ -1156,21 +1133,11 @@ DEFINE_Bool(enable_debug_points, "false");

DEFINE_Int32(pipeline_executor_size, "0");
DEFINE_Int32(blocking_pipeline_executor_size, "0");
DEFINE_Bool(enable_workload_group_for_scan, "false");
DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000");

// Whether use schema dict in backend side instead of MetaService side(cloud mode)
DEFINE_mBool(variant_use_cloud_schema_dict_cache, "true");
DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "2048");
DEFINE_mInt32(variant_max_json_key_length, "255");
DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false");
DEFINE_mBool(variant_enable_duplicate_json_path_check, "false");
DEFINE_mBool(enable_vertical_compact_variant_subcolumns, "true");
DEFINE_mBool(enable_variant_doc_sparse_write_subcolumns, "true");
// Maximum depth of nested arrays to track with NestedGroup
// Reserved for future use when NestedGroup expansion moves to storage layer
// Deeper arrays will be stored as JSONB
DEFINE_mInt32(variant_nested_group_max_depth, "3");
DEFINE_mBool(variant_nested_group_discard_scalar_on_conflict, "false");

DEFINE_Validator(variant_max_json_key_length,
Expand Down Expand Up @@ -1207,9 +1174,6 @@ DEFINE_mInt64(file_cache_evict_in_advance_recycle_keys_num_threshold, "1000");

DEFINE_mBool(enable_read_cache_file_directly, "false");
DEFINE_mBool(file_cache_enable_evict_from_other_queue_by_size, "true");
// If true, evict the ttl cache using LRU when full.
// Otherwise, only expiration can evict ttl and new data won't add to cache when full.
DEFINE_Bool(enable_ttl_cache_evict_using_lru, "true");
DEFINE_mBool(enbale_dump_error_file, "false");
// limit the max size of error log on disk
DEFINE_mInt64(file_cache_error_log_limit_bytes, "209715200"); // 200MB
Expand Down Expand Up @@ -1240,7 +1204,6 @@ DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000");
DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "180000");
DEFINE_mInt64(file_cache_background_ttl_info_update_interval_ms, "180000");
DEFINE_mInt64(file_cache_background_tablet_id_flush_interval_ms, "1000");
DEFINE_mInt64(file_cache_background_ttl_gc_batch, "1000");
DEFINE_mInt64(file_cache_background_lru_dump_interval_ms, "60000");
// dump queue only if the queue update specific times through several dump intervals
DEFINE_mInt64(file_cache_background_lru_dump_update_cnt_threshold, "1000");
Expand Down Expand Up @@ -1342,9 +1305,6 @@ DEFINE_mBool(allow_zero_date, "false");
DEFINE_Bool(allow_invalid_decimalv2_literal, "false");
DEFINE_mString(kerberos_ccache_path, "/tmp/");
DEFINE_mString(kerberos_krb5_conf_path, "/etc/krb5.conf");
// Deprecated
DEFINE_mInt32(kerberos_refresh_interval_second, "43200");

// JDK-8153057: avoid StackOverflowError thrown from the UncaughtExceptionHandler in thread "process reaper"
DEFINE_mBool(jdk_process_reaper_use_default_stack_size, "true");

Expand Down Expand Up @@ -1460,9 +1420,6 @@ DEFINE_mBool(enable_column_type_check, "true");
// 128 MB
DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");

// Default 300s, if its value <= 0, then log is disabled
DEFINE_mInt64(enable_debug_log_timeout_secs, "0");

// Tolerance for the number of partition id 0 in rowset, default 0
DEFINE_Int32(ignore_invalid_partition_id_rowset_num, "0");

Expand Down Expand Up @@ -1578,8 +1535,6 @@ DEFINE_Validator(paimon_file_system_scheme_mappings,

DEFINE_mInt32(thrift_client_open_num_tries, "1");

DEFINE_Bool(enable_index_compaction, "false");

// http scheme in S3Client to use. E.g. http or https
DEFINE_String(s3_client_http_scheme, "http");
DEFINE_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
Expand Down Expand Up @@ -1693,8 +1648,6 @@ DEFINE_mInt32(max_automatic_compaction_num_per_round, "64");
DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300");
DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10");
DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true");
DEFINE_mInt32(schema_dict_cache_capacity, "4096");

// whether to prune rows with delete sign = 1 in base compaction
// ATTN: this config is only for test
DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
Expand Down Expand Up @@ -1787,8 +1740,6 @@ DEFINE_mInt32(file_handles_deplenish_frequency_times, "3");

// clang-format off
#ifdef BE_TEST
// test s3
DEFINE_String(test_s3_resource, "resource");
DEFINE_String(test_s3_ak, "ak");
DEFINE_String(test_s3_sk, "sk");
DEFINE_String(test_s3_endpoint, "endpoint");
Expand Down Expand Up @@ -2095,6 +2046,37 @@ std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
continue; \
}

// Keys that were previously valid BE config entries but have been removed.
// If a user still has any of these in be.conf / be_custom.conf after upgrade,
// we emit a warning so they know the value is no longer honored, but BE will
// continue to start.
static const std::set<std::string> kDeprecatedConfigKeys = {
"doris_scanner_max_run_time_ms",
"multi_get_max_threads",
"num_disks",
"read_size",
"load_max_wg_active_memtable_percent",
"jdbc_connection_pool_cache_clear_time_sec",
"big_column_size_buffer",
"small_column_size_buffer",
"max_fragment_start_wait_time_seconds",
"enable_workload_group_for_scan",
"workload_group_scan_task_wait_timeout_ms",
"variant_use_cloud_schema_dict_cache",
"variant_threshold_rows_to_estimate_sparse_column",
"variant_nested_group_max_depth",
"enable_ttl_cache_evict_using_lru",
"file_cache_background_ttl_gc_batch",
"kerberos_refresh_interval_second",
"enable_debug_log_timeout_secs",
"enable_index_compaction",
"schema_dict_cache_capacity",
"test_s3_resource",
"meta_service_use_load_balancer",
"meta_service_rpc_timeout_ms",
"delete_bitmap_rpc_retry_times",
};

// init conf fields
bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_to_default) {
Properties props;
Expand Down Expand Up @@ -2123,6 +2105,14 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t
SET_FIELD(it.second, std::vector<std::string>, fill_conf_map, set_to_default);
}

// Warn about any deprecated keys still present in the conf file.
for (const auto& kv : props.conf_map()) {
if (kDeprecatedConfigKeys.count(kv.first) > 0) {
LOG(WARNING) << "BE config '" << kv.first << "' in " << conf_file
<< " is deprecated and no longer takes effect; please remove it.";
}
}

if (config::is_cloud_mode()) {
auto st = config::set_config("enable_file_cache", "true", true, true);
LOG(INFO) << "set config enable_file_cache "
Expand Down
Loading
Loading