Merge branch 'apache:master' into master

.github/workflows/auto-cherry-pick.yml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+on:
+  pull_request:
+    branches:
+      - master
+    types: ["closed"]
+
+jobs:
+  cherry_pick_branch_2.1:
+    runs-on: ubuntu-latest
+    name: Cherry pick into branch-2.1
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') && github.event.pull_request.merged == true }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Cherry pick into branch-2.1
+        uses: carloscastrojumo/github-cherry-pick-action@v1.0.1
+        with:
+          branch: branch-2.1
+          labels: |
+            cherry-pick
+          reviewers: |
+            yiguolei
+  cherry_pick_branch-3.0:
+    runs-on: ubuntu-latest
+    name: Cherry pick into branch-3.0
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') && github.event.pull_request.merged == true }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Cherry pick into branch-3.0
+        uses: carloscastrojumo/github-cherry-pick-action@v1.0.1
+        with:
+          branch: branch-3.0
+          labels: |
+            cherry-pick
+          reviewers: |
+            dataroaring
+          title: '[cherry-pick] {old_title}'
+          body: 'Cherry picking #{old_pull_request_id} onto this branch'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

be/CMakeLists.txt

@@ -299,8 +299,6 @@ if (COMPILER_CLANG)
                         -Wno-implicit-float-conversion
                         -Wno-implicit-int-conversion
                         -Wno-sign-conversion
-                        -Wno-missing-field-initializers
-                        -Wno-unused-const-variable
                         -Wno-shorten-64-to-32)
     if (USE_LIBCPP)
         add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)

be/src/cloud/cloud_meta_mgr.cpp

@@ -880,6 +880,7 @@ Status CloudMetaMgr::abort_txn(const StreamLoadContext& ctx) {
     AbortTxnRequest req;
     AbortTxnResponse res;
     req.set_cloud_unique_id(config::cloud_unique_id);
+    req.set_reason(std::string(ctx.status.msg().substr(0, 1024)));
     if (ctx.db_id > 0 && !ctx.label.empty()) {
         req.set_db_id(ctx.db_id);
         req.set_label(ctx.label);

be/src/cloud/cloud_storage_engine.cpp

@@ -558,14 +558,16 @@ std::vector<CloudTabletSPtr> CloudStorageEngine::_generate_cloud_compaction_task
     } else if (config::enable_parallel_cumu_compaction) {
         filter_out = [&tablet_preparing_cumu_compaction](CloudTablet* t) {
             return tablet_preparing_cumu_compaction.contains(t->tablet_id()) ||
-                   (t->tablet_state() != TABLET_RUNNING && t->alter_version() == -1);
+                   (t->tablet_state() != TABLET_RUNNING &&
+                    (!config::enable_new_tablet_do_compaction || t->alter_version() == -1));
         };
     } else {
         filter_out = [&tablet_preparing_cumu_compaction,
                       &submitted_cumu_compactions](CloudTablet* t) {
             return tablet_preparing_cumu_compaction.contains(t->tablet_id()) ||
                    submitted_cumu_compactions.contains(t->tablet_id()) ||
-                   (t->tablet_state() != TABLET_RUNNING && t->alter_version() == -1);
+                   (t->tablet_state() != TABLET_RUNNING &&
+                    (!config::enable_new_tablet_do_compaction || t->alter_version() == -1));
         };
     }
 

be/src/common/compile_check_begin.h

@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#pragma once
-
 #ifdef __clang__
 #pragma clang diagnostic push
-#pragma clang diagnostic error "-Wshorten-64-to-32"
+#pragma clang diagnostic error "-Wconversion"
+#pragma clang diagnostic ignored "-Wsign-conversion"
+#pragma clang diagnostic ignored "-Wfloat-conversion"
 #endif
 //#include "common/compile_check_begin.h"

be/src/common/compile_check_end.h

@@ -15,8 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#pragma once
-
 #ifdef __clang__
 #pragma clang diagnostic pop
 #endif

be/src/common/config.cpp

@@ -1797,6 +1797,9 @@ Status set_fuzzy_configs() {
             ((distribution(*generator) % 2) == 0) ? "true" : "false";
     fuzzy_field_and_value["enable_shrink_memory"] =
             ((distribution(*generator) % 2) == 0) ? "true" : "false";
+    fuzzy_field_and_value["enable_vertical_compaction"] =
+            ((distribution(*generator) % 2) == 0) ? "true" : "false";
+
     fuzzy_field_and_value["string_overflow_size"] =
             ((distribution(*generator) % 2) == 0) ? "10" : "4294967295";
 

be/src/exprs/runtime_filter.cpp

@@ -1283,16 +1283,17 @@ void IRuntimeFilter::set_filter_timer(std::shared_ptr<pipeline::RuntimeFilterTim
     _filter_timer.push_back(timer);
 }
 
-void IRuntimeFilter::set_dependency(std::shared_ptr<pipeline::Dependency> dependency) {
+void IRuntimeFilter::set_finish_dependency(
+        const std::shared_ptr<pipeline::CountedFinishDependency>& dependency) {
     _dependency = dependency;
-    ((pipeline::CountedFinishDependency*)_dependency.get())->add();
+    _dependency->add();
     CHECK(_dependency);
 }
 
 void IRuntimeFilter::set_synced_size(uint64_t global_size) {
     _synced_size = global_size;
     if (_dependency) {
-        ((pipeline::CountedFinishDependency*)_dependency.get())->sub();
+        _dependency->sub();
     }
 }
 

be/src/exprs/runtime_filter.h

@@ -69,6 +69,7 @@ struct RuntimeFilterContextSPtr;
 
 namespace pipeline {
 class RuntimeFilterTimer;
+class CountedFinishDependency;
 } // namespace pipeline
 
 enum class RuntimeFilterType {
@@ -352,7 +353,8 @@ class IRuntimeFilter {
 
     void set_synced_size(uint64_t global_size);
 
-    void set_dependency(std::shared_ptr<pipeline::Dependency> dependency);
+    void set_finish_dependency(
+            const std::shared_ptr<pipeline::CountedFinishDependency>& dependency);
 
     int64_t get_synced_size() const { return _synced_size; }
 
@@ -422,7 +424,7 @@ class IRuntimeFilter {
     std::vector<std::shared_ptr<pipeline::RuntimeFilterTimer>> _filter_timer;
 
     int64_t _synced_size = -1;
-    std::shared_ptr<pipeline::Dependency> _dependency;
+    std::shared_ptr<pipeline::CountedFinishDependency> _dependency;
 };
 
 // avoid expose RuntimePredicateWrapper

be/src/exprs/runtime_filter_slots.h

@@ -42,17 +42,17 @@ class VRuntimeFilterSlots {
     }
 
     Status send_filter_size(RuntimeState* state, uint64_t hash_table_size,
-                            std::shared_ptr<pipeline::Dependency> dependency) {
+                            std::shared_ptr<pipeline::CountedFinishDependency> dependency) {
         if (_runtime_filters.empty()) {
             return Status::OK();
         }
         for (auto runtime_filter : _runtime_filters) {
             if (runtime_filter->need_sync_filter_size()) {
-                runtime_filter->set_dependency(dependency);
+                runtime_filter->set_finish_dependency(dependency);
             }
         }
 
-        // send_filter_size may call dependency->sub(), so we call set_dependency firstly for all rf to avoid dependency set_ready repeatedly
+        // send_filter_size may call dependency->sub(), so we call set_finish_dependency firstly for all rf to avoid dependency set_ready repeatedly
         for (auto runtime_filter : _runtime_filters) {
             if (runtime_filter->need_sync_filter_size()) {
                 RETURN_IF_ERROR(runtime_filter->send_filter_size(state, hash_table_size));

be/src/http/default_path_handlers.cpp

@@ -103,7 +103,7 @@ void memory_info_handler(std::stringstream* output) {
               << "<a "
                  "href=https://doris.apache.org/zh-CN/docs/dev/admin-manual/memory-management/"
                  "memory-issue-faq>Memory Issue FAQ</a>\n"
-              << "\n---\n";
+              << "\n---\n\n";
 
     (*output) << "<h4 id=\"memoryPropertiesTitle\">Memory Properties</h4>\n"
               << "System Physical Mem: "
@@ -122,7 +122,7 @@ void memory_info_handler(std::stringstream* output) {
               << "Cgroup Mem Usage: "
               << PrettyPrinter::print(MemInfo::cgroup_mem_usage(), TUnit::BYTES) << std::endl
               << "Cgroup Mem Refresh State: " << MemInfo::cgroup_mem_refresh_state() << std::endl
-              << "\n---\n";
+              << "\n---\n\n";
 
     (*output) << "<h4 id=\"memoryOptionSettingsTitle\">Memory Option Settings</h4>\n";
     {
@@ -135,7 +135,7 @@ void memory_info_handler(std::stringstream* output) {
             }
         }
     }
-    (*output) << "\n---\n";
+    (*output) << "\n---\n\n";
 
     (*output) << "<h4 id=\"jemallocProfilesTitle\">Jemalloc Profiles</h4>\n";
 #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
@@ -162,11 +162,45 @@ void memory_info_handler(std::stringstream* output) {
 
 // Registered to handle "/profile".
 void process_profile_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) {
-    (*output) << "<h2 id=\"processProfileTitle\">Process Profile</h2>\n";
+    (*output) << "<h4>Copy Process Profile To Clipboard (拷贝 Process Profile 到剪切板) </h4>";
+    (*output) << "<button id=\"copyToClipboard\">Copy Page Text</button>" << std::endl;
+    (*output) << "<script>" << std::endl;
+    (*output) << "$('#copyToClipboard').click(function () {" << std::endl;
+    // create a hidden textarea element
+    (*output) << "     var textarea = document.createElement('textarea');" << std::endl;
+    (*output) << "     textarea.style.position = 'absolute';" << std::endl;
+    (*output) << "     textarea.style.left = '-9999px';" << std::endl;
+    // get the content to copy
+    (*output) << "     var contentToCopy = document.getElementById('allPageText').innerHTML;"
+              << std::endl;
+    (*output) << "     textarea.value = contentToCopy;"
+              << std::endl; // set the content to the textarea
+    (*output) << "     document.body.appendChild(textarea);" << std::endl;
+    (*output) << "     textarea.select();" << std::endl;
+    (*output) << "     textarea.setSelectionRange(0, 99999);"
+              << std::endl; // compatible with mobile devices
+    (*output) << "try {" << std::endl;
+    (*output) << "          document.execCommand('copy');"
+              << std::endl; //copy the selected text to the clipboard
+    (*output) << "          alert('Process profile copied to clipboard!');" << std::endl;
+    (*output) << "      } catch (err) {" << std::endl;
+    (*output) << "          alert('Failed to copy process profile! ' + err);" << std::endl;
+    (*output) << "      }" << std::endl;
+    (*output) << "});" << std::endl;
+    (*output) << "</script>" << std::endl;
+
     doris::ProcessProfile::instance()->refresh_profile();
+
+    (*output) << "<div id=\"allPageText\">" << std::endl;
+    (*output) << "<h2 id=\"processProfileTitle\">Process Profile</h2>" << std::endl;
     (*output) << "<pre id=\"processProfile\">"
-              << doris::ProcessProfile::instance()->print_process_profile_no_root() << "</pre>";
+              << doris::ProcessProfile::instance()->print_process_profile_no_root() << "</pre>"
+              << "\n\n---\n\n";
     memory_info_handler(output);
+
+    // TODO, expect more information about process status, CPU, IO, etc.
+
+    (*output) << "</div>" << std::endl;
 }
 
 void display_tablets_callback(const WebPageHandler::ArgumentMap& args, EasyJson* ej) {

be/src/io/cache/block_file_cache.cpp

@@ -244,15 +244,12 @@ void BlockFileCache::use_cell(const FileBlockCell& cell, FileBlocks* result, boo
         result->push_back(cell.file_block);
     }
 
-    if (cell.file_block->cache_type() != FileCacheType::TTL ||
-        config::enable_ttl_cache_evict_using_lru) {
-        auto& queue = get_queue(cell.file_block->cache_type());
-        DCHECK(cell.queue_iterator) << "impossible";
-        /// Move to the end of the queue. The iterator remains valid.
-        if (move_iter_flag) {
-            queue.move_to_end(*cell.queue_iterator, cache_lock);
-        }
+    auto& queue = get_queue(cell.file_block->cache_type());
+    /// Move to the end of the queue. The iterator remains valid.
+    if (cell.queue_iterator && move_iter_flag) {
+        queue.move_to_end(*cell.queue_iterator, cache_lock);
     }
+
     cell.update_atime();
     cell.is_deleted = false;
 }
@@ -358,7 +355,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte
                 auto st = cell.file_block->change_cache_type_between_ttl_and_others(
                         FileCacheType::NORMAL);
                 if (st.ok()) {
-                    if (config::enable_ttl_cache_evict_using_lru) {
+                    if (cell.queue_iterator) {
                         auto& ttl_queue = get_queue(FileCacheType::TTL);
                         ttl_queue.remove(cell.queue_iterator.value(), cache_lock);
                     }
@@ -1056,7 +1053,7 @@ bool BlockFileCache::remove_if_ttl_file_unlock(const UInt128Wrapper& file_key, b
                 auto st = cell.file_block->change_cache_type_between_ttl_and_others(
                         FileCacheType::NORMAL);
                 if (st.ok()) {
-                    if (config::enable_ttl_cache_evict_using_lru) {
+                    if (cell.queue_iterator) {
                         ttl_queue.remove(cell.queue_iterator.value(), cache_lock);
                     }
                     auto& queue = get_queue(FileCacheType::NORMAL);
@@ -1133,7 +1130,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper& hash, size_t offset, size
            _files.find(hash)->second.find(offset) != _files.find(hash)->second.end());
     FileBlockCell* cell = get_cell(hash, offset, cache_lock);
     DCHECK(cell != nullptr);
-    if (cell->file_block->cache_type() != FileCacheType::TTL) {
+    if (cell->queue_iterator) {
         auto& queue = get_queue(cell->file_block->cache_type());
         DCHECK(queue.contains(hash, offset, cache_lock));
         auto iter = queue.get(hash, offset, cache_lock);
@@ -1809,10 +1806,8 @@ std::string BlockFileCache::clear_file_cache_directly() {
        << " time_elapsed=" << duration_cast<milliseconds>(steady_clock::now() - start).count()
        << " num_files=" << num_files << " cache_size=" << cache_size
        << " index_queue_size=" << index_queue_size << " normal_queue_size=" << normal_queue_size
-       << " disposible_queue_size=" << disposible_queue_size;
-    if (config::enable_ttl_cache_evict_using_lru) {
-        ss << "ttl_queue_size=" << ttl_queue_size;
-    }
+       << " disposible_queue_size=" << disposible_queue_size << "ttl_queue_size=" << ttl_queue_size;
+
     auto msg = ss.str();
     LOG(INFO) << msg;
     return msg;