Merge branch 'master' into scanner_wait

be/src/agent/task_worker_pool.cpp

@@ -273,8 +273,8 @@ Status check_migrate_request(StorageEngine& engine, const TStorageMediumMigrateR
  // check local disk capacity
  int64_t tablet_size = tablet->tablet_local_size();
  if ((*dest_store)->reach_capacity_limit(tablet_size)) {
- return Status::InternalError("reach the capacity limit of path {}, tablet_size={}",
- (*dest_store)->path(), tablet_size);
+ return Status::Error<EXCEEDED_LIMIT>("reach the capacity limit of path {}, tablet_size={}",
+  (*dest_store)->path(), tablet_size);
  }
  return Status::OK();
 }

be/src/common/config.cpp

@@ -42,8 +42,7 @@
 #include "io/fs/file_writer.h"
 #include "io/fs/local_file_system.h"
 
-namespace doris {
-namespace config {
+namespace doris::config {
 
 // Dir of custom config file
 DEFINE_String(custom_config_dir, "${DORIS_HOME}/conf");
@@ -109,10 +108,6 @@ DEFINE_mInt32(hash_table_double_grow_degree, "31");
 DEFINE_mInt32(max_fill_rate, "2");
 
 DEFINE_mInt32(double_resize_threshold, "23");
-// Expand the hash table before inserting data, the maximum expansion size.
-// There are fewer duplicate keys, reducing the number of resize hash tables
-// There are many duplicate keys, and the hash table filled bucket is far less than the hash table build bucket.
-DEFINE_mInt64(hash_table_pre_expanse_max_rows, "65535");
 
 // The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default 1.6G,
 // actual low water mark=min(1.6G, MemTotal * 10%), avoid wasting too much memory on machines
@@ -846,16 +841,6 @@ DEFINE_String(function_service_protocol, "h2:grpc");
 // use which load balancer to select server to connect
 DEFINE_String(rpc_load_balancer, "rr");
 
-// The maximum buffer/queue size to collect span. After the size is reached, spans are dropped.
-// An export will be triggered when the number of spans in the queue reaches half of the maximum.
-DEFINE_Int32(max_span_queue_size, "2048");
-
-// The maximum batch size of every export spans. It must be smaller or equal to max_queue_size.
-DEFINE_Int32(max_span_export_batch_size, "512");
-
-// The time interval between two consecutive export spans.
-DEFINE_Int32(export_span_schedule_delay_millis, "500");
-
 // a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
 // so we set a soft limit, default is 1MB
 DEFINE_mInt32(string_type_length_soft_limit_bytes, "1048576");
@@ -868,10 +853,6 @@ DEFINE_mInt32(jsonb_type_length_soft_limit_bytes, "1048576");
 DEFINE_Validator(jsonb_type_length_soft_limit_bytes,
  [](const int config) -> bool { return config > 0 && config <= 2147483643; });
 
-// used for olap scanner to save memory, when the size of unused_object_pool
-// is greater than object_pool_buffer_size, release the object in the unused_object_pool.
-DEFINE_Int32(object_pool_buffer_size, "100");
-
 // Threshold of reading a small file into memory
 DEFINE_mInt32(in_memory_file_size, "1048576"); // 1MB
 
@@ -907,7 +888,7 @@ DEFINE_Int32(concurrency_per_dir, "2");
 // "whole_file_cache": the whole file.
 DEFINE_mString(file_cache_type, "file_block_cache");
 DEFINE_Validator(file_cache_type, [](std::string_view config) -> bool {
- return config == "" || config == "file_block_cache";
+ return config.empty() || config == "file_block_cache";
 });
 
 DEFINE_Int32(s3_transfer_executor_pool_size, "2");
@@ -971,8 +952,8 @@ DEFINE_Bool(enable_fuzzy_mode, "false");
 DEFINE_Bool(enable_debug_points, "false");
 
 DEFINE_Int32(pipeline_executor_size, "0");
-// 128 MB
-DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");
+DEFINE_Bool(enable_workload_group_for_scan, "false");
+DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000");
 
 // Temp config. True to use optimization for bitmap_index apply predicate except leaf node of the and node.
 // Will remove after fully test.
@@ -1153,6 +1134,9 @@ DEFINE_Bool(enable_snapshot_action, "false");
 
 DEFINE_mInt32(variant_max_merged_tablet_schema_size, "2048");
 
+// 128 MB
+DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3
@@ -1205,7 +1189,7 @@ bool replaceenv(std::string& s) {
  std::size_t pos = 0;
  std::size_t start = 0;
  while ((start = s.find("${", pos)) != std::string::npos) {
- std::size_t end = s.find("}", start + 2);
+ std::size_t end = s.find('}', start + 2);
  if (end == std::string::npos) {
  return false;
  }
@@ -1243,9 +1227,9 @@ bool strtox(const std::string& valstr, std::vector<T>& retval) {
 }
 
 bool strtox(const std::string& valstr, bool& retval) {
- if (valstr.compare("true") == 0) {
+ if (valstr == "true") {
  retval = true;
- } else if (valstr.compare("false") == 0) {
+ } else if (valstr == "false") {
  retval = false;
  } else {
  return false;
@@ -1605,18 +1589,17 @@ std::vector<std::vector<std::string>> get_config_info() {
  std::vector<std::string> _config;
  _config.push_back(it.first);
 
- _config.push_back(field_it->second.type);
+ _config.emplace_back(field_it->second.type);
  if (0 == strcmp(field_it->second.type, "bool")) {
- _config.push_back(it.second == "1" ? "true" : "false");
+ _config.emplace_back(it.second == "1" ? "true" : "false");
  } else {
  _config.push_back(it.second);
  }
- _config.push_back(field_it->second.valmutable ? "true" : "false");
+ _config.emplace_back(field_it->second.valmutable ? "true" : "false");
 
  configs.push_back(_config);
  }
  return configs;
 }
 
-} // namespace config
-} // namespace doris
+} // namespace doris::config

be/src/common/config.h

@@ -152,11 +152,6 @@ DECLARE_mInt32(max_fill_rate);
 
 DECLARE_mInt32(double_resize_threshold);
 
-// Expand the hash table before inserting data, the maximum expansion size.
-// There are fewer duplicate keys, reducing the number of resize hash tables
-// There are many duplicate keys, and the hash table filled bucket is far less than the hash table build bucket.
-DECLARE_mInt64(hash_table_pre_expanse_max_rows);
-
 // The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default 1.6G,
 // actual low water mark=min(1.6G, MemTotal * 10%), avoid wasting too much memory on machines
 // with large memory larger than 16G.
@@ -908,26 +903,12 @@ DECLARE_String(function_service_protocol);
 // use which load balancer to select server to connect
 DECLARE_String(rpc_load_balancer);
 
-// The maximum buffer/queue size to collect span. After the size is reached, spans are dropped.
-// An export will be triggered when the number of spans in the queue reaches half of the maximum.
-DECLARE_Int32(max_span_queue_size);
-
-// The maximum batch size of every export spans. It must be smaller or equal to max_queue_size.
-DECLARE_Int32(max_span_export_batch_size);
-
-// The time interval between two consecutive export spans.
-DECLARE_Int32(export_span_schedule_delay_millis);
-
 // a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
 // so we set a soft limit, default is 1MB
 DECLARE_mInt32(string_type_length_soft_limit_bytes);
 
 DECLARE_mInt32(jsonb_type_length_soft_limit_bytes);
 
-// used for olap scanner to save memory, when the size of unused_object_pool
-// is greater than object_pool_buffer_size, release the object in the unused_object_pool.
-DECLARE_Int32(object_pool_buffer_size);
-
 // Threshold fo reading a small file into memory
 DECLARE_mInt32(in_memory_file_size);
 

be/src/common/status.h

@@ -8,8 +8,8 @@
 #include <gen_cpp/Status_types.h> // for TStatus
 #include <gen_cpp/types.pb.h>
 #include <glog/logging.h>
-#include <stdint.h>
 
+#include <cstdint>
 #include <iostream>
 #include <memory>
 #include <string>
@@ -93,7 +93,8 @@ namespace ErrorCode {
  E(VERSION_NOT_EXIST, -214, false); \
  E(TABLE_NOT_FOUND, -215, true); \
  E(TRY_LOCK_FAILED, -216, false); \
- E(OUT_OF_BOUND, -218, true); \
+ E(EXCEEDED_LIMIT, -217, false); \
+ E(OUT_OF_BOUND, -218, false); \
  E(INVALID_ROOT_PATH, -222, true); \
  E(NO_AVAILABLE_ROOT_PATH, -223, true); \
  E(CHECK_LINES_ERROR, -224, true); \