Skip to content

Commit

Permalink
Merge branch 'master' into scanner_wait
Browse files Browse the repository at this point in the history
  • Loading branch information
@dataroaring
dataroaring authored Dec 25, 2023
2 parents c7ea8d4 + 6085a61 commit 3db2567
Show file tree
Hide file tree
Showing 142 changed files with 2,092 additions and 686 deletions.
4 changes: 2 additions & 2 deletions be/src/agent/task_worker_pool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,8 +273,8 @@ Status check_migrate_request(StorageEngine& engine, const TStorageMediumMigrateR
// check local disk capacity
int64_t tablet_size = tablet->tablet_local_size();
if ((*dest_store)->reach_capacity_limit(tablet_size)) {
return Status::InternalError("reach the capacity limit of path {}, tablet_size={}",
(*dest_store)->path(), tablet_size);
return Status::Error<EXCEEDED_LIMIT>("reach the capacity limit of path {}, tablet_size={}",
(*dest_store)->path(), tablet_size);
}
return Status::OK();
}
Expand Down
45 changes: 14 additions & 31 deletions be/src/common/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"

namespace doris {
namespace config {
namespace doris::config {

// Dir of custom config file
DEFINE_String(custom_config_dir, "${DORIS_HOME}/conf");
Expand Down Expand Up @@ -109,10 +108,6 @@ DEFINE_mInt32(hash_table_double_grow_degree, "31");
DEFINE_mInt32(max_fill_rate, "2");

DEFINE_mInt32(double_resize_threshold, "23");
// Expand the hash table before inserting data, the maximum expansion size.
// There are fewer duplicate keys, reducing the number of resize hash tables
// There are many duplicate keys, and the hash table filled bucket is far less than the hash table build bucket.
DEFINE_mInt64(hash_table_pre_expanse_max_rows, "65535");

// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default 1.6G,
// actual low water mark=min(1.6G, MemTotal * 10%), avoid wasting too much memory on machines
Expand Down Expand Up @@ -846,16 +841,6 @@ DEFINE_String(function_service_protocol, "h2:grpc");
// use which load balancer to select server to connect
DEFINE_String(rpc_load_balancer, "rr");

// The maximum buffer/queue size to collect span. After the size is reached, spans are dropped.
// An export will be triggered when the number of spans in the queue reaches half of the maximum.
DEFINE_Int32(max_span_queue_size, "2048");

// The maximum batch size of every export spans. It must be smaller or equal to max_queue_size.
DEFINE_Int32(max_span_export_batch_size, "512");

// The time interval between two consecutive export spans.
DEFINE_Int32(export_span_schedule_delay_millis, "500");

// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
// so we set a soft limit, default is 1MB
DEFINE_mInt32(string_type_length_soft_limit_bytes, "1048576");
Expand All @@ -868,10 +853,6 @@ DEFINE_mInt32(jsonb_type_length_soft_limit_bytes, "1048576");
DEFINE_Validator(jsonb_type_length_soft_limit_bytes,
[](const int config) -> bool { return config > 0 && config <= 2147483643; });

// used for olap scanner to save memory, when the size of unused_object_pool
// is greater than object_pool_buffer_size, release the object in the unused_object_pool.
DEFINE_Int32(object_pool_buffer_size, "100");

// Threshold of reading a small file into memory
DEFINE_mInt32(in_memory_file_size, "1048576"); // 1MB

Expand Down Expand Up @@ -907,7 +888,7 @@ DEFINE_Int32(concurrency_per_dir, "2");
// "whole_file_cache": the whole file.
DEFINE_mString(file_cache_type, "file_block_cache");
DEFINE_Validator(file_cache_type, [](std::string_view config) -> bool {
return config == "" || config == "file_block_cache";
return config.empty() || config == "file_block_cache";
});

DEFINE_Int32(s3_transfer_executor_pool_size, "2");
Expand Down Expand Up @@ -971,8 +952,8 @@ DEFINE_Bool(enable_fuzzy_mode, "false");
DEFINE_Bool(enable_debug_points, "false");

DEFINE_Int32(pipeline_executor_size, "0");
// 128 MB
DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");
DEFINE_Bool(enable_workload_group_for_scan, "false");
DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000");

// Temp config. True to use optimization for bitmap_index apply predicate except leaf node of the and node.
// Will remove after fully test.
Expand Down Expand Up @@ -1153,6 +1134,9 @@ DEFINE_Bool(enable_snapshot_action, "false");

DEFINE_mInt32(variant_max_merged_tablet_schema_size, "2048");

// 128 MB
DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");

// clang-format off
#ifdef BE_TEST
// test s3
Expand Down Expand Up @@ -1205,7 +1189,7 @@ bool replaceenv(std::string& s) {
std::size_t pos = 0;
std::size_t start = 0;
while ((start = s.find("${", pos)) != std::string::npos) {
std::size_t end = s.find("}", start + 2);
std::size_t end = s.find('}', start + 2);
if (end == std::string::npos) {
return false;
}
Expand Down Expand Up @@ -1243,9 +1227,9 @@ bool strtox(const std::string& valstr, std::vector<T>& retval) {
}

bool strtox(const std::string& valstr, bool& retval) {
if (valstr.compare("true") == 0) {
if (valstr == "true") {
retval = true;
} else if (valstr.compare("false") == 0) {
} else if (valstr == "false") {
retval = false;
} else {
return false;
Expand Down Expand Up @@ -1605,18 +1589,17 @@ std::vector<std::vector<std::string>> get_config_info() {
std::vector<std::string> _config;
_config.push_back(it.first);

_config.push_back(field_it->second.type);
_config.emplace_back(field_it->second.type);
if (0 == strcmp(field_it->second.type, "bool")) {
_config.push_back(it.second == "1" ? "true" : "false");
_config.emplace_back(it.second == "1" ? "true" : "false");
} else {
_config.push_back(it.second);
}
_config.push_back(field_it->second.valmutable ? "true" : "false");
_config.emplace_back(field_it->second.valmutable ? "true" : "false");

configs.push_back(_config);
}
return configs;
}

} // namespace config
} // namespace doris
} // namespace doris::config
19 changes: 0 additions & 19 deletions be/src/common/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,6 @@ DECLARE_mInt32(max_fill_rate);

DECLARE_mInt32(double_resize_threshold);

// Expand the hash table before inserting data, the maximum expansion size.
// There are fewer duplicate keys, reducing the number of resize hash tables
// There are many duplicate keys, and the hash table filled bucket is far less than the hash table build bucket.
DECLARE_mInt64(hash_table_pre_expanse_max_rows);

// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit byte, default 1.6G,
// actual low water mark=min(1.6G, MemTotal * 10%), avoid wasting too much memory on machines
// with large memory larger than 16G.
Expand Down Expand Up @@ -908,26 +903,12 @@ DECLARE_String(function_service_protocol);
// use which load balancer to select server to connect
DECLARE_String(rpc_load_balancer);

// The maximum buffer/queue size to collect span. After the size is reached, spans are dropped.
// An export will be triggered when the number of spans in the queue reaches half of the maximum.
DECLARE_Int32(max_span_queue_size);

// The maximum batch size of every export spans. It must be smaller or equal to max_queue_size.
DECLARE_Int32(max_span_export_batch_size);

// The time interval between two consecutive export spans.
DECLARE_Int32(export_span_schedule_delay_millis);

// a soft limit of string type length, the hard limit is 2GB - 4, but if too long will cause very low performance,
// so we set a soft limit, default is 1MB
DECLARE_mInt32(string_type_length_soft_limit_bytes);

DECLARE_mInt32(jsonb_type_length_soft_limit_bytes);

// used for olap scanner to save memory, when the size of unused_object_pool
// is greater than object_pool_buffer_size, release the object in the unused_object_pool.
DECLARE_Int32(object_pool_buffer_size);

// Threshold fo reading a small file into memory
DECLARE_mInt32(in_memory_file_size);

Expand Down
5 changes: 3 additions & 2 deletions be/src/common/status.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
#include <gen_cpp/Status_types.h> // for TStatus
#include <gen_cpp/types.pb.h>
#include <glog/logging.h>
#include <stdint.h>

#include <cstdint>
#include <iostream>
#include <memory>
#include <string>
Expand Down Expand Up @@ -93,7 +93,8 @@ namespace ErrorCode {
E(VERSION_NOT_EXIST, -214, false); \
E(TABLE_NOT_FOUND, -215, true); \
E(TRY_LOCK_FAILED, -216, false); \
E(OUT_OF_BOUND, -218, true); \
E(EXCEEDED_LIMIT, -217, false); \
E(OUT_OF_BOUND, -218, false); \
E(INVALID_ROOT_PATH, -222, true); \
E(NO_AVAILABLE_ROOT_PATH, -223, true); \
E(CHECK_LINES_ERROR, -224, true); \
Expand Down
Loading

0 comments on commit 3db2567

Please sign in to comment.