From 7376f0cd07f2b0973aad77822a8bc83c0458ebc9 Mon Sep 17 00:00:00 2001 From: "zhe.yang" Date: Wed, 9 Oct 2024 15:45:11 +0800 Subject: [PATCH 001/101] add log --- validator/impl/liteserver.cpp | 406 +++++++++++++++++++++------------- validator/impl/liteserver.hpp | 27 ++- validator/impl/scope_guard.h | 37 ++++ 3 files changed, 312 insertions(+), 158 deletions(-) create mode 100644 validator/impl/scope_guard.h diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 7bedf7fe4..859289562 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -44,9 +44,16 @@ #include #include "td/actor/MultiPromise.h" #include "collator-impl.h" +#include "scope_guard.h" -namespace ton { +#define LOG_TIME_EST(hint) \ + SCOPE_EXIT([start = std::chrono::steady_clock::now(), hint]() { \ + const auto end{std::chrono::steady_clock::now()}; \ + const auto elapsed = std::chrono::duration_cast(end - start).count(); \ + LOG(INFO) << hint << elapsed << "μs"; \ + }); +namespace ton { namespace validator { using td::Ref; using namespace std::literals::string_literals; @@ -56,15 +63,16 @@ td::int32 get_tl_tag(td::Slice slice) { } void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId manager, - td::actor::ActorId cache, - td::Promise promise) { + td::actor::ActorId cache, td::Promise promise) { td::actor::create_actor("litequery", std::move(data), std::move(manager), std::move(cache), std::move(promise)) .release(); } -void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) { +void LiteQuery::fetch_account_state( + WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise) { td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); } @@ -74,8 +82,9 @@ LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } -LiteQuery::LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise) +LiteQuery::LiteQuery( + WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> promise) : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } @@ -162,32 +171,30 @@ void LiteQuery::start_up() { use_cache_ = use_cache(); if (use_cache_) { cache_key_ = td::sha256_bits256(query_); - td::actor::send_closure( - cache_, &LiteServerCache::lookup, cache_key_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &LiteQuery::perform); - } else { - td::actor::send_closure(SelfId, &LiteQuery::finish_query, R.move_as_ok(), true); - } - }); + td::actor::send_closure(cache_, &LiteServerCache::lookup, cache_key_, + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &LiteQuery::perform); + } else { + td::actor::send_closure(SelfId, &LiteQuery::finish_query, R.move_as_ok(), true); + } + }); } else { perform(); } } -bool LiteQuery::use_cache() { +bool LiteQuery::use_cache() { if (cache_.empty()) { return false; } bool use = false; - lite_api::downcast_call( - *query_obj_, - td::overloaded( - [&](lite_api::liteServer_runSmcMethod& q) { - // wc=-1, seqno=-1 means "use latest mc block" - use = q.id_->workchain_ != masterchainId || q.id_->seqno_ != -1; - }, - [&](auto& obj) { use = false; })); + lite_api::downcast_call(*query_obj_, td::overloaded( + [&](lite_api::liteServer_runSmcMethod& q) { + // wc=-1, seqno=-1 means "use latest mc block" + use = q.id_->workchain_ != masterchainId || q.id_->seqno_ != -1; + }, + [&](auto& obj) { use = false; })); return use; } @@ -236,7 +243,8 @@ void LiteQuery::perform() { this->perform_lookupBlock(ton::create_block_id_simple(q.id_), q.mode_, q.lt_, q.utime_); }, [&](lite_api::liteServer_lookupBlockWithProof& q) { - this->perform_lookupBlockWithProof(ton::create_block_id_simple(q.id_), ton::create_block_id(q.mc_block_id_), q.mode_, q.lt_, q.utime_); + this->perform_lookupBlockWithProof(ton::create_block_id_simple(q.id_), ton::create_block_id(q.mc_block_id_), + q.mode_, q.lt_, q.utime_); }, [&](lite_api::liteServer_listBlockTransactions& q) { this->perform_listBlockTransactions(ton::create_block_id(q.id_), q.mode_, q.count_, @@ -245,8 +253,8 @@ void LiteQuery::perform() { }, [&](lite_api::liteServer_listBlockTransactionsExt& q) { this->perform_listBlockTransactionsExt(ton::create_block_id(q.id_), q.mode_, q.count_, - (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), - static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); + (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), + static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); }, [&](lite_api::liteServer_getConfigParams& q) { this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xffff) | 0x10000, q.param_list_); @@ -268,15 +276,11 @@ void LiteQuery::perform() { this->perform_runSmcMethod(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, q.mode_, q.method_id_, std::move(q.params_)); }, - [&](lite_api::liteServer_getLibraries& q) { - this->perform_getLibraries(q.library_list_); - }, + [&](lite_api::liteServer_getLibraries& q) { this->perform_getLibraries(q.library_list_); }, [&](lite_api::liteServer_getLibrariesWithProof& q) { this->perform_getLibrariesWithProof(ton::create_block_id(q.id_), q.mode_, q.library_list_); }, - [&](lite_api::liteServer_getShardBlockProof& q) { - this->perform_getShardBlockProof(create_block_id(q.id_)); - }, + [&](lite_api::liteServer_getShardBlockProof& q) { this->perform_getShardBlockProof(create_block_id(q.id_)); }, [&](lite_api::liteServer_nonfinal_getCandidate& q) { this->perform_nonfinal_getCandidate(q.id_->creator_, create_block_id(q.id_->block_id_), q.id_->collated_data_hash_); @@ -313,22 +317,22 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { fatal_error("unsupported getMasterchainInfo mode"); return; } - td::actor::send_closure_later( - manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, - [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - auto pair = res.move_as_ok(); - auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; - td::actor::send_closure_later(Self, func, std::move(pair.first), - pair.second, mode); - } - }); + td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, + [Self = actor_id(this), return_state = bool(acc_state_promise_), + mode](td::Result, BlockIdExt>> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + auto pair = res.move_as_ok(); + auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState + : &LiteQuery::continue_getMasterchainInfo; + td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); + } + }); } void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, - int mode) { + int mode) { perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); } @@ -571,6 +575,12 @@ void LiteQuery::get_block_handle_checked(BlockIdExt blkid, td::Promise(end - start).count(); + LOG(INFO) << "request_mc_block_data cost" << elapsed << "μs" + << ". counter" << counter; + })); if (!blkid.is_masterchain() || !blkid.is_valid_full()) { return fatal_error("reference block must belong to the masterchain"); } @@ -651,6 +661,12 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { } bool LiteQuery::request_mc_block_data_state(BlockIdExt blkid) { + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "request_mc_block_data_state cost" << elapsed << "μs" + << ". counter" << counter; + })); return request_mc_block_data(blkid) && request_mc_block_state(blkid); } @@ -774,8 +790,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { return; } td::actor::send_closure_later( - manager, &ValidatorManager::get_zero_state, blkid, - [=](td::Result res) { + manager, &ValidatorManager::get_zero_state, blkid, [=](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); @@ -787,9 +802,23 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { return true; } +static std::atomic_uint64_t counter; + void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode) { + auto c = counter.fetch_add(1, std::memory_order_seq_cst); LOG(INFO) << "started a getAccountState(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " - << mode << ") liteserver query"; + << mode << ") liteserver query." + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << c; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), c]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "perform_getAccountState cost " << elapsed << "μs" + << ". counter" << c; + })); if (blkid.id.workchain != masterchainId && blkid.id.workchain != workchain) { fatal_error("reference block for a getAccountState() must belong to the masterchain"); return; @@ -810,11 +839,14 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, acc_workchain_ = workchain; acc_addr_ = addr; mode_ = mode; + counter_ = c; if (blkid.id.workchain != masterchainId) { base_blk_id_ = blkid; + LOG(INFO) << "started a getAccountState: set finish_getAccountState"; set_continuation([&]() -> void { finish_getAccountState({}); }); request_block_data_state(blkid); } else if (blkid.id.seqno != ~0U) { + LOG(INFO) << "started a getAccountState: set continue_getAccountState"; set_continuation([&]() -> void { continue_getAccountState(); }); request_mc_block_data_state(blkid); } else { @@ -835,6 +867,11 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, void LiteQuery::continue_getAccountState_0(Ref mc_state, BlockIdExt blkid) { LOG(INFO) << "obtained last masterchain block = " << blkid.to_str(); + SCOPE_EXIT([start = std::chrono::steady_clock::now()]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "continue_getAccountState_0 cost" << elapsed << "μs"; + }); base_blk_id_ = blkid; CHECK(mc_state.not_null()); mc_state_ = Ref(std::move(mc_state)); @@ -894,22 +931,24 @@ void LiteQuery::perform_getLibraries(std::vector library_list) { LOG(INFO) << "too many libraries requested, returning only first 16"; library_list.resize(16); } - sort( library_list.begin(), library_list.end() ); - library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + sort(library_list.begin(), library_list.end()); + library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); td::actor::send_closure_later( manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, - [Self = actor_id(this), library_list](td::Result, BlockIdExt>> res) -> void { + [Self = actor_id(this), + library_list](td::Result, BlockIdExt>> res) -> void { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); - td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), - pair.second, library_list); + td::actor::send_closure_later(Self, &LiteQuery::continue_getLibraries, std::move(pair.first), pair.second, + library_list); } }); } -void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list) { +void LiteQuery::continue_getLibraries(Ref mc_state, BlockIdExt blkid, + std::vector library_list) { LOG(INFO) << "obtained last masterchain block = " << blkid.to_str(); base_blk_id_ = blkid; CHECK(mc_state.not_null()); @@ -929,7 +968,7 @@ void LiteQuery::continue_getLibraries(Ref mc_s LOG(INFO) << "\n" << os.str(); auto lib_dict = std::make_unique(config->get_libraries_root(), 256); - for (auto k: *lib_dict) { + for (auto k : *lib_dict) { std::ostringstream oss; k.second->print_rec(oss); LOG(INFO) << "library " << k.first.to_hex(256) << ": \n" << oss.str(); @@ -961,8 +1000,8 @@ void LiteQuery::perform_getLibrariesWithProof(BlockIdExt blkid, int mode, std::v LOG(INFO) << "too many libraries requested, returning only first 16"; library_list.resize(16); } - sort( library_list.begin(), library_list.end() ); - library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + sort(library_list.begin(), library_list.end()); + library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); set_continuation([this, library_list, mode]() -> void { continue_getLibrariesWithProof(library_list, mode); }); request_mc_block_data_state(blkid); @@ -1002,8 +1041,9 @@ void LiteQuery::continue_getLibrariesWithProof(std::vector library_ // include first 16 publishers in the proof auto publishers_dict = vm::Dictionary{vm::DictNonEmpty(), libdescr.publishers, 256}; auto iter = publishers_dict.begin(); - constexpr int max_publishers = 15; // set to 15 because publishers_dict.begin() counts as the first visit - for (int i = 0; i < max_publishers && iter != publishers_dict.end(); ++i, ++iter) {} + constexpr int max_publishers = 15; // set to 15 because publishers_dict.begin() counts as the first visit + for (int i = 0; i < max_publishers && iter != publishers_dict.end(); ++i, ++iter) { + } } result_hashes.push_back(hash); @@ -1029,7 +1069,7 @@ void LiteQuery::continue_getLibrariesWithProof(std::vector library_ } if (!libdescr.lib->get_hash().bits().equals(hash.bits(), 256)) { LOG(ERROR) << "public library hash mismatch: expected " << hash.to_hex() << " , found " - << libdescr.lib->get_hash().to_hex(); + << libdescr.lib->get_hash().to_hex(); continue; } td::BufferSlice libdata; @@ -1044,8 +1084,9 @@ void LiteQuery::continue_getLibrariesWithProof(std::vector library_ result.push_back(ton::create_tl_object(hash, std::move(libdata))); } - auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(base_blk_id_), mode, std::move(result), - state_proof_boc.move_as_ok(), data_proof_boc.move_as_ok()); + auto b = ton::create_serialize_tl_object( + ton::create_tl_lite_block_id(base_blk_id_), mode, std::move(result), state_proof_boc.move_as_ok(), + data_proof_boc.move_as_ok()); finish_query(std::move(b)); } @@ -1068,7 +1109,13 @@ void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchai } void LiteQuery::got_block_state(BlockIdExt blkid, Ref state) { - LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ") needed by a liteserver query"; + LOG(INFO) << "got_block_state(" << blkid.to_str() << ") needed by a liteserver query"; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "got_block_state cost " << elapsed << "μs" + << ". counter" << counter; + })); CHECK(state.not_null()); state_ = Ref(std::move(state)); CHECK(state_.not_null()); @@ -1077,7 +1124,14 @@ void LiteQuery::got_block_state(BlockIdExt blkid, Ref state) { } void LiteQuery::got_mc_block_state(BlockIdExt blkid, Ref state) { - LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ") needed by a liteserver query"; + LOG(INFO) << "got_mc_block_state(" << blkid.to_str() << ") needed by a liteserver query" + << ". counter" << counter_; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "got_mc_block_state cost " << elapsed << "μs" + << ". counter" << counter; + })); CHECK(state.not_null()); mc_state_ = Ref(std::move(state)); CHECK(mc_state_.not_null()); @@ -1087,6 +1141,12 @@ void LiteQuery::got_mc_block_state(BlockIdExt blkid, Ref state) { void LiteQuery::got_block_data(BlockIdExt blkid, Ref data) { LOG(INFO) << "obtained data for getBlock(" << blkid.to_str() << ") needed by a liteserver query"; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "got_block_data cost " << elapsed << "μs" + << ". counter" << counter; + })); CHECK(data.not_null()); block_ = Ref(std::move(data)); CHECK(block_.not_null()); @@ -1095,7 +1155,14 @@ void LiteQuery::got_block_data(BlockIdExt blkid, Ref data) { } void LiteQuery::got_mc_block_data(BlockIdExt blkid, Ref data) { - LOG(INFO) << "obtained data for getBlock(" << blkid.to_str() << ") needed by a liteserver query"; + LOG(INFO) << "got_mc_block_data(" << blkid.to_str() << ") needed by a liteserver query" + << ". counter" << counter_; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "got_mc_block_data cost " << elapsed << "μs" + << ". counter" << counter; + })); CHECK(data.not_null()); mc_block_ = Ref(std::move(data)); CHECK(mc_block_.not_null()); @@ -1182,7 +1249,11 @@ bool LiteQuery::make_state_root_proof(Ref& proof, Ref state_ block::gen::BlkPrevInfo(info.after_merge).validate_ref(info.prev_ref))) { return fatal_error("cannot unpack block header"); } + const auto start{std::chrono::steady_clock::now()}; vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "create vm::CellSlice const" << elapsed << "μs"; if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update && upd_cs.size_ext() == 0x20228)) { return fatal_error("invalid Merkle update in block"); @@ -1269,7 +1340,19 @@ bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref st } void LiteQuery::continue_getAccountState() { - LOG(INFO) << "continue getAccountState() query"; + LOG(INFO) << "continue getAccountState() query. " + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << counter_; + + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "continue_getAccountState cost " << elapsed << "μs" + << ". counter" << counter; + })); if (acc_workchain_ == masterchainId) { blk_id_ = base_blk_id_; block_ = mc_block_; @@ -1304,7 +1387,18 @@ void LiteQuery::continue_getAccountState() { } void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { - LOG(INFO) << "completing getAccountState() query"; + LOG(INFO) << "completing getAccountState() query. " + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << counter_; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "finish_getAccountState cost " << elapsed << "μs" + << ". counter" << counter; + })); Ref proof1, proof2; if (!make_state_root_proof(proof1)) { return; @@ -1325,9 +1419,8 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { } auto rconfig = config.move_as_ok(); rconfig->set_block_id_ext(mc_state_->get_block_id()); - acc_state_promise_.set_value(std::make_tuple( - std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) - )); + acc_state_promise_.set_value( + std::make_tuple(std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig))); return; } @@ -1501,9 +1594,9 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice } LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** - auto r_config = block::ConfigInfo::extract_config( - mc_state_->root_cell(), - block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks); + auto r_config = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries | + block::ConfigInfo::needCapabilities | + block::ConfigInfo::needPrevBlocks); if (r_config.is_error()) { fatal_error(r_config.move_as_error()); return; @@ -2025,7 +2118,8 @@ void LiteQuery::continue_getAllShardsInfo() { finish_query(std::move(b)); } -void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, int mode, LogicalTime lt, UnixTime utime) { +void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, int mode, LogicalTime lt, + UnixTime utime) { if (!((1 << (mode & 7)) & 0x16)) { fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set"); return; @@ -2049,30 +2143,33 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, << lt << ", " << utime << ") liteserver query"; ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; - auto P = td::PromiseCreator::lambda( - [Self = actor_id(this), mc_blkid, manager = manager_, mode, pfx](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - return; - } - auto handle = res.move_as_ok(); - if (!handle->inited_masterchain_ref_block()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); - return; - } - if (handle->masterchain_ref_block() > mc_blkid.seqno()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("specified mc block is older than block's masterchain ref")); - return; - } - LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, - [Self, mc_ref_blkid = handle->masterchain_ref_block(), mc_blkid, pfx, mode](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, res.move_as_ok(), pfx, mc_ref_blkid); - } - }); + auto P = td::PromiseCreator::lambda([Self = actor_id(this), mc_blkid, manager = manager_, mode, + pfx](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + auto handle = res.move_as_ok(); + if (!handle->inited_masterchain_ref_block()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); + return; + } + if (handle->masterchain_ref_block() > mc_blkid.seqno()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, + td::Status::Error("specified mc block is older than block's masterchain ref")); + return; + } + LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_data_from_db, handle, + [Self, mc_ref_blkid = handle->masterchain_ref_block(), mc_blkid, pfx, mode](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, + res.move_as_ok(), pfx, mc_ref_blkid); + } + }); }); if (mode & 2) { @@ -2086,7 +2183,9 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, } } -void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno) { +void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref block, + AccountIdPrefixFull req_prefix, + BlockSeqno masterchain_ref_seqno) { blk_id_ = block->block_id(); LOG(INFO) << "obtained data for getBlockHeader(" << blk_id_.to_str() << ", " << mode_ << ")"; CHECK(block.not_null()); @@ -2121,26 +2220,30 @@ void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref R) mutable { + get_block_handle_checked(prev_blkid, [Self = actor_id(this), masterchain_ref_seqno, + manager = manager_](td::Result R) mutable { if (R.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } - td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), [Self, masterchain_ref_seqno](td::Result> res) mutable { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - return; - } - td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData, res.move_as_ok(), masterchain_ref_seqno); - }); + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + td::actor::send_closure(Self, + &LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData, + res.move_as_ok(), masterchain_ref_seqno); + }); }); } else { continue_lookupBlockWithProof_gotPrevBlockData(Ref(), masterchain_ref_seqno); } } -void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, BlockSeqno masterchain_ref_seqno) { +void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, + BlockSeqno masterchain_ref_seqno) { if (prev_block.not_null()) { CHECK(prev_block.not_null()); if (prev_block->root_cell().is_null()) { @@ -2164,29 +2267,33 @@ void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref pr if (!blk_id_.is_masterchain()) { ton::AccountIdPrefixFull pfx{ton::masterchainId, ton::shardIdAll}; - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno, + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno, [manager = manager_, Self = actor_id(this)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), - [Self](td::Result> res) mutable { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - return; - } - td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, res.move_as_ok(), std::vector>>()); - }); - }); + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [Self](td::Result> res) mutable { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + td::actor::send_closure( + Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, + res.move_as_ok(), std::vector>>()); + }); + }); } else { base_blk_id_alt_ = blk_id_; - td::actor::send_closure(actor_id(this), &LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState, std::vector>>()); + td::actor::send_closure(actor_id(this), &LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState, + std::vector>>()); } } -void LiteQuery::continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, - std::vector>> result) { +void LiteQuery::continue_lookupBlockWithProof_buildProofLinks( + td::Ref cur_block, std::vector>> result) { BlockIdExt cur_id = cur_block->block_id(); BlockIdExt prev_id; vm::MerkleProofBuilder mpb{cur_block->root_cell()}; @@ -2262,19 +2369,20 @@ void LiteQuery::continue_lookupBlockWithProof_buildProofLinks(td::Ref td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, R.move_as_ok(), std::move(result)); } - }); + }); } -void LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector>> links) { - set_continuation([this, links = std::move(links)]() -> void { - continue_lookupBlockWithProof_getMcBlockPrev(std::move(links)); - }); +void LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState( + std::vector>> links) { + set_continuation( + [this, links = std::move(links)]() -> void { continue_lookupBlockWithProof_getMcBlockPrev(std::move(links)); }); request_mc_block_data_state(base_blk_id_); } -void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev(std::vector>> links) { +void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev( + std::vector>> links) { td::BufferSlice mc_state_proof_buf, client_mc_blk_proof_buf; - + if (base_blk_id_alt_ != base_blk_id_) { vm::MerkleProofBuilder mpb{mc_state_->root_cell()}; auto prev_blocks_dict = block::get_prev_blocks_dict(mpb.root()); @@ -2313,17 +2421,17 @@ void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev(std::vector(create_tl_lite_block_id(p.first), prev_block_proof.move_as_ok())); + links_res.push_back(create_tl_object(create_tl_lite_block_id(p.first), + prev_block_proof.move_as_ok())); } - auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(blk_id_), - mode_, ton::create_tl_lite_block_id(base_blk_id_alt_), std::move(mc_state_proof_buf), std::move(client_mc_blk_proof_buf), - std::move(links_res), std::move(lookup_header_proof_), std::move(lookup_prev_header_proof_)); + auto b = ton::create_serialize_tl_object( + ton::create_tl_lite_block_id(blk_id_), mode_, ton::create_tl_lite_block_id(base_blk_id_alt_), + std::move(mc_state_proof_buf), std::move(client_mc_blk_proof_buf), std::move(links_res), + std::move(lookup_header_proof_), std::move(lookup_prev_header_proof_)); finish_query(std::move(b)); } - void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime) { if (!((1 << (mode & 7)) & 0x16)) { fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set"); @@ -2358,14 +2466,13 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; if (mode & 2) { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_for_litequery, pfx, lt, - std::move(P)); + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_for_litequery, pfx, lt, std::move(P)); } else if (mode & 4) { td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_unix_time_for_litequery, pfx, utime, std::move(P)); } else { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, - blkid.seqno, std::move(P)); + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, blkid.seqno, + std::move(P)); } } @@ -2406,7 +2513,8 @@ static td::Result> get_i } block::tlb::MsgEnvelope::Record_std env; if (!block::tlb::unpack_cell(std::move(msg_env), env)) { - return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " << in_msg_hash.to_hex()); + return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " + << in_msg_hash.to_hex()); } if (!env.metadata) { return nullptr; @@ -2528,7 +2636,8 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { finish_query(std::move(b)); } -void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt) { +void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, + LogicalTime lt) { LOG(INFO) << "started a listBlockTransactionsExt(" << blkid.to_str() << ", " << mode << ", " << count << ", " << account.to_hex() << ", " << lt << ") liteserver query"; base_blk_id_ = blkid; @@ -2674,7 +2783,7 @@ void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { fatal_error(res.move_as_error()); return; } - + auto b = ton::create_serialize_tl_object( ton::create_tl_lite_block_id(base_blk_id_), req_count, !eof, res.move_as_ok(), std::move(proof_data)); LOG(INFO) << "listBlockTransactionsExt() query completed"; @@ -3432,12 +3541,11 @@ void LiteQuery::finish_getBlockOutMsgQueueSize() { finish_query(std::move(b)); } - void LiteQuery::perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash) { LOG(INFO) << "started a nonfinal.getCandidate liteserver query"; td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_candidate_for_litequery, PublicKey{pubkeys::Ed25519{source}}, blkid, collated_data_hash, - [Self = actor_id(this)](td::Result R) { + manager_, &ValidatorManager::get_block_candidate_for_litequery, PublicKey{pubkeys::Ed25519{source}}, blkid, + collated_data_hash, [Self = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 2d75dc61c..7b6b33127 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -41,7 +41,8 @@ class LiteQuery : public td::actor::Actor { td::Timestamp timeout_; td::Promise promise_; - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + acc_state_promise_; tl_object_ptr query_obj_; bool use_cache_{false}; @@ -51,6 +52,7 @@ class LiteQuery : public td::actor::Actor { int mode_{0}; WorkchainId acc_workchain_; StdSmcAddress acc_addr_; + std::uint64_t counter_; LogicalTime trans_lt_; Bits256 trans_hash_; BlockIdExt base_blk_id_, base_blk_id_alt_, blk_id_; @@ -84,13 +86,16 @@ class LiteQuery : public td::actor::Actor { }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); - LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); - static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, - td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); + static void fetch_account_state( + WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, + td::Promise, UnixTime, LogicalTime, std::unique_ptr>> + promise); private: bool fatal_error(td::Status error); @@ -143,11 +148,15 @@ class LiteQuery : public td::actor::Actor { void perform_getConfigParams(BlockIdExt blkid, int mode, std::vector param_list = {}); void continue_getConfigParams(int mode, std::vector param_list); void perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime); - void perform_lookupBlockWithProof(BlockId blkid, BlockIdExt client_mc_blkid, int mode, LogicalTime lt, UnixTime utime); - void continue_lookupBlockWithProof_getHeaderProof(Ref block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno); + void perform_lookupBlockWithProof(BlockId blkid, BlockIdExt client_mc_blkid, int mode, LogicalTime lt, + UnixTime utime); + void continue_lookupBlockWithProof_getHeaderProof(Ref block, + AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno); void continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, BlockSeqno masterchain_ref_seqno); - void continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, std::vector>> result); - void continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector>> links); + void continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, + std::vector>> result); + void continue_lookupBlockWithProof_getClientMcBlockDataState( + std::vector>> links); void continue_lookupBlockWithProof_getMcBlockPrev(std::vector>> links); void perform_listBlockTransactions(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt); void finish_listBlockTransactions(int mode, int count); diff --git a/validator/impl/scope_guard.h b/validator/impl/scope_guard.h new file mode 100644 index 000000000..d2d33eae6 --- /dev/null +++ b/validator/impl/scope_guard.h @@ -0,0 +1,37 @@ +#ifndef __SCOPE_GUARD_HPP__ +#define __SCOPE_GUARD_HPP__ + +#include + +class ScopeGuard { + public: + ScopeGuard(std::function& on_scope_exit) { + this->on_scope_exit = on_scope_exit; + } + ScopeGuard(std::function&& on_scope_exit) { + this->on_scope_exit = std::move(on_scope_exit); + } + + void dismiss() { + is_dismiss = true; + } + ~ScopeGuard() { + if (!is_dismiss) + on_scope_exit(); + } + + ScopeGuard(const ScopeGuard&) = delete; + ScopeGuard& operator=(const ScopeGuard&) = delete; + ScopeGuard(ScopeGuard&&) = delete; + ScopeGuard& operator=(ScopeGuard&&) = delete; + + protected: + bool is_dismiss = false; + std::function on_scope_exit; +}; + +#define SCOPEGUARD_LINENAME_CAT(name, line) name##line +#define SCOPEGUARD_LINENAME(name, line) SCOPEGUARD_LINENAME_CAT(name, line) +#define SCOPE_EXIT(x) ScopeGuard SCOPEGUARD_LINENAME(scope_exit_, __LINE__)((x)) + +#endif /*__SCOPE_GUARD_HPP__*/ \ No newline at end of file From 8eda34440eab5de8e4ce717d90ac5e57b3d06e8a Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 11 Oct 2024 17:07:45 +0800 Subject: [PATCH 002/101] update --- validator/manager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.hpp b/validator/manager.hpp index 99aa4e0e1..388cb99a2 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -678,7 +678,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::unique_ptr callback_; td::actor::ActorOwn db_; - bool started_ = false; + bool started_ = true; bool allow_validate_ = false; private: From 45285b719a580d673b5a9a71b6caaac49927e26b Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 11 Oct 2024 17:17:44 +0800 Subject: [PATCH 003/101] update --- build.sh | 1 + validator/impl/liteserver.cpp | 5 +++++ 2 files changed, 6 insertions(+) create mode 100755 build.sh diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..4970290af --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +docker build -t ton -f ./Dockerfile . diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 7bedf7fe4..f065e5745 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -522,6 +522,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { } void LiteQuery::continue_getState(BlockIdExt blkid, Ref state) { + td::PerfWarningTimer timer{"zjg-continue_getState", 0.01}; LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ")"; CHECK(state.not_null()); auto res = state->serialize(); @@ -1068,6 +1069,7 @@ void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchai } void LiteQuery::got_block_state(BlockIdExt blkid, Ref state) { + td::PerfWarningTimer timer{"zjg-got_block_state", 0.01}; LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ") needed by a liteserver query"; CHECK(state.not_null()); state_ = Ref(std::move(state)); @@ -1077,6 +1079,7 @@ void LiteQuery::got_block_state(BlockIdExt blkid, Ref state) { } void LiteQuery::got_mc_block_state(BlockIdExt blkid, Ref state) { + td::PerfWarningTimer timer{"zjg-got_mc_block_state", 0.01}; LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ") needed by a liteserver query"; CHECK(state.not_null()); mc_state_ = Ref(std::move(state)); @@ -1269,6 +1272,7 @@ bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref st } void LiteQuery::continue_getAccountState() { + td::PerfWarningTimer timer{"zjg-continue_getAccountState", 0.01}; LOG(INFO) << "continue getAccountState() query"; if (acc_workchain_ == masterchainId) { blk_id_ = base_blk_id_; @@ -1304,6 +1308,7 @@ void LiteQuery::continue_getAccountState() { } void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { + td::PerfWarningTimer timer{"zjg-finish_getAccountState", 0.01}; LOG(INFO) << "completing getAccountState() query"; Ref proof1, proof2; if (!make_state_root_proof(proof1)) { From b97364e0b081b78aae24e0e490a99db80194b00a Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 11 Oct 2024 17:55:21 +0800 Subject: [PATCH 004/101] update --- validator/db/archive-manager.cpp | 6 ++++++ validator/db/archive-slice.cpp | 2 ++ validator/impl/liteserver.cpp | 7 +++++++ validator/manager-disk.cpp | 7 +++++++ validator/manager-hardfork.cpp | 4 ++++ 5 files changed, 26 insertions(+) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 14d3ec469..96a8f16f6 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -151,6 +151,8 @@ void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice d } void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise promise) { + td::PerfWarningTimer timer{"zjg-get_handle", 0.01}; + LOG(INFO) << "zjg-get_handle"; auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), false); if (f) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_max_temp_file_desc_idx(), @@ -168,6 +170,8 @@ void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise pr } void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Promise promise) { + td::PerfWarningTimer timer{"zjg-get_handle_cont", 0.01}; + LOG(INFO) << "zjg-get_handle_cont"; if (idx.is_empty()) { promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db")); return; @@ -189,6 +193,8 @@ void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Pro } void ArchiveManager::get_handle_finish(BlockHandle handle, td::Promise promise) { + td::PerfWarningTimer timer{"zjg-get_handle_cont", 0.01}; + LOG(INFO) << "zjg-get_handle_cont"; auto f = get_file_desc_by_seqno(handle->id().shard_full(), handle->id().seqno(), false); if (!f) { promise.set_value(std::move(handle)); diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index d392431a9..56f83e84b 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -314,6 +314,8 @@ void ArchiveSlice::add_file_cont(size_t idx, FileReference ref_id, td::uint64 of } void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise promise) { + td::PerfWarningTimer timer{"zjg-get_handle_finish", 0.01}; + LOG(INFO) << "zjg-get_handle_finish"; if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index f065e5745..2079fd5f9 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -630,6 +630,8 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { } bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { + td::PerfWarningTimer timer{"zjg-request_mc_block_state", 0.01}; + LOG(INFO) << "zjg-request_mc_block_state"; if (!blkid.is_masterchain() || !blkid.is_valid_full()) { return fatal_error("reference block must belong to the masterchain"); } @@ -661,6 +663,8 @@ bool LiteQuery::request_block_data_state(BlockIdExt blkid) { } bool LiteQuery::request_block_state(BlockIdExt blkid) { + td::PerfWarningTimer timer{"zjg-request_block_state", 0.01}; + LOG(INFO) << "zjg-request_block_state"; if (!blkid.is_valid_full()) { return fatal_error("invalid block id requested"); } @@ -683,6 +687,8 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { } bool LiteQuery::request_block_data(BlockIdExt blkid) { + td::PerfWarningTimer timer{"zjg-request_block_data", 0.01}; + LOG(INFO) << "zjg-request_block_data"; if (!blkid.is_valid_full()) { return fatal_error("invalid block id requested"); } @@ -789,6 +795,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { } void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode) { + td::PerfWarningTimer timer{"zjg-perform_getAccountState", 0.01}; LOG(INFO) << "started a getAccountState(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " << mode << ") liteserver query"; if (blkid.id.workchain != masterchainId && blkid.id.workchain != workchain) { diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 5678408c6..b342eea4c 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -571,6 +571,9 @@ void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td:: void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) { + + td::PerfWarningTimer timer{"zjg-get_shard_state_from_db_short", 0.01}; + LOG(INFO) << "zjg-get_shard_state_from_db_short"; auto P = td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -821,6 +824,8 @@ void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref sta } void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promise promise) { + td::PerfWarningTimer timer{"zjg-get_block_handle", 0.01}; + LOG(INFO) << "zjg-get_block_handle"; auto it = handles_.find(id); if (it != handles_.end()) { auto handle = it->second.lock(); @@ -853,6 +858,8 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi } void ValidatorManagerImpl::register_block_handle(BlockHandle handle, td::Promise promise) { + td::PerfWarningTimer timer{"zjg-register_block_handle", 0.01}; + LOG(INFO) << "zjg-register_block_handle"; auto it = handles_.find(handle->id()); if (it != handles_.end()) { auto h = it->second.lock(); diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 49d27085f..b995b2fff 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -397,6 +397,8 @@ void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td:: void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) { + td::PerfWarningTimer timer{"zjg-get_shard_state_from_db_short", 0.01}; + LOG(INFO) << "zjg-get_shard_state_from_db_short"; auto P = td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -503,6 +505,8 @@ void ValidatorManagerImpl::finished_wait_data(BlockIdExt block_id, td::Result promise) { + td::PerfWarningTimer timer{"zjg-get_shard_state_from_db_short", 0.01}; + LOG(INFO) << "zjg-get_shard_state_from_db_short"; auto it = handles_.find(id); if (it != handles_.end()) { auto handle = it->second.lock(); From 71573f179f82647b53aaba09f1acb31955c1989d Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 12 Oct 2024 09:51:36 +0800 Subject: [PATCH 005/101] update --- validator/db/archive-manager.cpp | 12 ++++++------ validator/db/archive-slice.cpp | 4 ++-- validator/impl/liteserver.cpp | 26 ++++++++++++++------------ validator/impl/liteserver.hpp | 1 + validator/manager-disk.cpp | 12 ++++++------ validator/manager-hardfork.cpp | 8 ++++---- 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 96a8f16f6..bac3a0290 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -151,8 +151,8 @@ void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice d } void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise promise) { - td::PerfWarningTimer timer{"zjg-get_handle", 0.01}; - LOG(INFO) << "zjg-get_handle"; + td::PerfWarningTimer timer{"okxdebug-get_handle", 0.01}; + LOG(INFO) << "okxdebug-get_handle"; auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), false); if (f) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_max_temp_file_desc_idx(), @@ -170,8 +170,8 @@ void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise pr } void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Promise promise) { - td::PerfWarningTimer timer{"zjg-get_handle_cont", 0.01}; - LOG(INFO) << "zjg-get_handle_cont"; + td::PerfWarningTimer timer{"okxdebug-get_handle_cont", 0.01}; + LOG(INFO) << "okxdebug-get_handle_cont"; if (idx.is_empty()) { promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db")); return; @@ -193,8 +193,8 @@ void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Pro } void ArchiveManager::get_handle_finish(BlockHandle handle, td::Promise promise) { - td::PerfWarningTimer timer{"zjg-get_handle_cont", 0.01}; - LOG(INFO) << "zjg-get_handle_cont"; + td::PerfWarningTimer timer{"okxdebug-get_handle_cont", 0.01}; + LOG(INFO) << "okxdebug-get_handle_cont"; auto f = get_file_desc_by_seqno(handle->id().shard_full(), handle->id().seqno(), false); if (!f) { promise.set_value(std::move(handle)); diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 56f83e84b..292931e6b 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -314,8 +314,8 @@ void ArchiveSlice::add_file_cont(size_t idx, FileReference ref_id, td::uint64 of } void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise promise) { - td::PerfWarningTimer timer{"zjg-get_handle_finish", 0.01}; - LOG(INFO) << "zjg-get_handle_finish"; + td::PerfWarningTimer timer{"okxdebug-get_handle_finish", 0.01}; + LOG(INFO) << "okxdebug-get_handle_finish"; if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 2079fd5f9..7454b77fd 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -55,6 +55,8 @@ td::int32 get_tl_tag(td::Slice slice) { return slice.size() >= 4 ? td::as(slice.data()) : -1; } +int global_count = 0; + void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise) { @@ -522,7 +524,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { } void LiteQuery::continue_getState(BlockIdExt blkid, Ref state) { - td::PerfWarningTimer timer{"zjg-continue_getState", 0.01}; + td::PerfWarningTimer timer{"okxdebug-continue_getState", 0.01}; LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ")"; CHECK(state.not_null()); auto res = state->serialize(); @@ -630,8 +632,8 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { } bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { - td::PerfWarningTimer timer{"zjg-request_mc_block_state", 0.01}; - LOG(INFO) << "zjg-request_mc_block_state"; + td::PerfWarningTimer timer{"okxdebug-request_mc_block_state", 0.01}; + LOG(INFO) << "okxdebug-request_mc_block_state"; if (!blkid.is_masterchain() || !blkid.is_valid_full()) { return fatal_error("reference block must belong to the masterchain"); } @@ -663,8 +665,8 @@ bool LiteQuery::request_block_data_state(BlockIdExt blkid) { } bool LiteQuery::request_block_state(BlockIdExt blkid) { - td::PerfWarningTimer timer{"zjg-request_block_state", 0.01}; - LOG(INFO) << "zjg-request_block_state"; + td::PerfWarningTimer timer{"okxdebug-request_block_state", 0.01}; + LOG(INFO) << "okxdebug-request_block_state"; if (!blkid.is_valid_full()) { return fatal_error("invalid block id requested"); } @@ -687,8 +689,8 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { } bool LiteQuery::request_block_data(BlockIdExt blkid) { - td::PerfWarningTimer timer{"zjg-request_block_data", 0.01}; - LOG(INFO) << "zjg-request_block_data"; + td::PerfWarningTimer timer{"okxdebug-request_block_data", 0.01}; + LOG(INFO) << "okxdebug-request_block_data"; if (!blkid.is_valid_full()) { return fatal_error("invalid block id requested"); } @@ -795,7 +797,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { } void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode) { - td::PerfWarningTimer timer{"zjg-perform_getAccountState", 0.01}; + td::PerfWarningTimer timer{"okxdebug-perform_getAccountState", 0.01}; LOG(INFO) << "started a getAccountState(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << ", " << mode << ") liteserver query"; if (blkid.id.workchain != masterchainId && blkid.id.workchain != workchain) { @@ -1076,7 +1078,7 @@ void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchai } void LiteQuery::got_block_state(BlockIdExt blkid, Ref state) { - td::PerfWarningTimer timer{"zjg-got_block_state", 0.01}; + td::PerfWarningTimer timer{"okxdebug-got_block_state", 0.01}; LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ") needed by a liteserver query"; CHECK(state.not_null()); state_ = Ref(std::move(state)); @@ -1086,7 +1088,7 @@ void LiteQuery::got_block_state(BlockIdExt blkid, Ref state) { } void LiteQuery::got_mc_block_state(BlockIdExt blkid, Ref state) { - td::PerfWarningTimer timer{"zjg-got_mc_block_state", 0.01}; + td::PerfWarningTimer timer{"okxdebug-got_mc_block_state", 0.01}; LOG(INFO) << "obtained data for getState(" << blkid.to_str() << ") needed by a liteserver query"; CHECK(state.not_null()); mc_state_ = Ref(std::move(state)); @@ -1279,7 +1281,7 @@ bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref st } void LiteQuery::continue_getAccountState() { - td::PerfWarningTimer timer{"zjg-continue_getAccountState", 0.01}; + td::PerfWarningTimer timer{"okxdebug-continue_getAccountState", 0.01}; LOG(INFO) << "continue getAccountState() query"; if (acc_workchain_ == masterchainId) { blk_id_ = base_blk_id_; @@ -1315,7 +1317,7 @@ void LiteQuery::continue_getAccountState() { } void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { - td::PerfWarningTimer timer{"zjg-finish_getAccountState", 0.01}; + td::PerfWarningTimer timer{"okxdebug-finish_getAccountState", 0.01}; LOG(INFO) << "completing getAccountState() query"; Ref proof1, proof2; if (!make_state_root_proof(proof1)) { diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 2d75dc61c..f58af609e 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -39,6 +39,7 @@ class LiteQuery : public td::actor::Actor { td::actor::ActorId manager_; td::actor::ActorId cache_; td::Timestamp timeout_; + string query_index_; td::Promise promise_; td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index b342eea4c..75c83b04d 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -572,8 +572,8 @@ void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td:: void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) { - td::PerfWarningTimer timer{"zjg-get_shard_state_from_db_short", 0.01}; - LOG(INFO) << "zjg-get_shard_state_from_db_short"; + td::PerfWarningTimer timer{"okxdebug-get_shard_state_from_db_short", 0.01}; + LOG(INFO) << "okxdebug-get_shard_state_from_db_short"; auto P = td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -824,8 +824,8 @@ void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref sta } void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promise promise) { - td::PerfWarningTimer timer{"zjg-get_block_handle", 0.01}; - LOG(INFO) << "zjg-get_block_handle"; + td::PerfWarningTimer timer{"okxdebug-get_block_handle", 0.01}; + LOG(INFO) << "okxdebug-get_block_handle"; auto it = handles_.find(id); if (it != handles_.end()) { auto handle = it->second.lock(); @@ -858,8 +858,8 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi } void ValidatorManagerImpl::register_block_handle(BlockHandle handle, td::Promise promise) { - td::PerfWarningTimer timer{"zjg-register_block_handle", 0.01}; - LOG(INFO) << "zjg-register_block_handle"; + td::PerfWarningTimer timer{"okxdebug-register_block_handle", 0.01}; + LOG(INFO) << "okxdebug-register_block_handle"; auto it = handles_.find(handle->id()); if (it != handles_.end()) { auto h = it->second.lock(); diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index b995b2fff..d1e10779e 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -397,8 +397,8 @@ void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td:: void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) { - td::PerfWarningTimer timer{"zjg-get_shard_state_from_db_short", 0.01}; - LOG(INFO) << "zjg-get_shard_state_from_db_short"; + td::PerfWarningTimer timer{"okxdebug-get_shard_state_from_db_short", 0.01}; + LOG(INFO) << "okxdebug-get_shard_state_from_db_short"; auto P = td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -505,8 +505,8 @@ void ValidatorManagerImpl::finished_wait_data(BlockIdExt block_id, td::Result promise) { - td::PerfWarningTimer timer{"zjg-get_shard_state_from_db_short", 0.01}; - LOG(INFO) << "zjg-get_shard_state_from_db_short"; + td::PerfWarningTimer timer{"okxdebug-get_shard_state_from_db_short", 0.01}; + LOG(INFO) << "okxdebug-get_shard_state_from_db_short"; auto it = handles_.find(id); if (it != handles_.end()) { auto handle = it->second.lock(); From c6dc185052e31b92715933709d68a92a75d2d377 Mon Sep 17 00:00:00 2001 From: "zhe.yang" Date: Sat, 12 Oct 2024 10:04:41 +0800 Subject: [PATCH 006/101] add log2 --- ton/ton-types.h | 1 + validator-engine/validator-engine.cpp | 220 +++++++++++++------------- validator/db/archive-manager.cpp | 93 +++++++++-- validator/db/archive-slice.cpp | 50 ++++-- validator/db/rootdb.cpp | 5 + validator/impl/liteserver.cpp | 1 + validator/manager-disk.hpp | 12 +- validator/manager-hardfork.hpp | 12 +- validator/manager.cpp | 103 +++++++++--- 9 files changed, 326 insertions(+), 171 deletions(-) diff --git a/ton/ton-types.h b/ton/ton-types.h index 915682655..533486d86 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -224,6 +224,7 @@ struct BlockIdExt { BlockId id; RootHash root_hash; FileHash file_hash; + std::uint64_t counter_; BlockIdExt(WorkchainId workchain, ShardId shard, BlockSeqno seqno, const RootHash& root_hash, const FileHash& file_hash) : id{workchain, shard, seqno}, root_hash(root_hash), file_hash(file_hash) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index bb7574105..80ab387db 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1247,10 +1247,12 @@ void ValidatorEngine::schedule_shutdown(double at) { LOG(DEBUG) << "Scheduled shutdown is in past (" << at << ")"; } else { LOG(INFO) << "Schedule shutdown for " << at << " (in " << ts.in() << "s)"; - ton::delay_action([]() { - LOG(WARNING) << "Shutting down as scheduled"; - std::_Exit(0); - }, ts); + ton::delay_action( + []() { + LOG(WARNING) << "Shutting down as scheduled"; + std::_Exit(0); + }, + ts); } } void ValidatorEngine::start_up() { @@ -2434,8 +2436,7 @@ void ValidatorEngine::try_del_proxy(td::uint32 ip, td::int32 port, std::vector(); + custom_overlays_config_ = ton::create_tl_object(); auto data_R = td::read_file(custom_overlays_config_file()); if (data_R.is_error()) { return; @@ -2488,7 +2489,7 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi static td::Result> parse_collator_options(td::MutableSlice json_str) { td::Ref ref{true}; - ton::validator::CollatorOptions& opts = ref.write(); + ton::validator::CollatorOptions &opts = ref.write(); // Set default values (from_json leaves missing fields as is) ton::ton_api::engine_validator_collatorOptions f; @@ -2531,11 +2532,11 @@ static td::Result> parse_collator_optio } else { opts.dispatch_phase_3_max_per_initiator = {}; } - for (const std::string& s : f.whitelist_) { + for (const std::string &s : f.whitelist_) { TRY_RESULT(addr, block::StdAddress::parse(s)); opts.whitelist.emplace(addr.workchain, addr.addr); } - for (const std::string& s : f.prioritylist_) { + for (const std::string &s : f.prioritylist_) { TRY_RESULT(addr, block::StdAddress::parse(s)); opts.prioritylist.emplace(addr.workchain, addr.addr); } @@ -3427,7 +3428,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } //TODO force Overlays::update_certificate to return result @@ -3442,17 +3443,15 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer }); */ td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, - ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, - ton::overlay::OverlayIdShort{query.overlay_id_}, - ton::PublicKeyHash{query.signed_key_->key_hash_}, - r.move_as_ok()); - promise.set_value( - ton::serialize_tl_object(ton::create_tl_object(), true) - ); + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok()); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3467,7 +3466,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { @@ -3479,12 +3478,13 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha } }); ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3506,11 +3506,11 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard promise.set_value(R.move_as_ok()); } }); - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, + std::move(P)); } - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3553,42 +3553,46 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi return; } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { - const std::vector times{60, 300, 3600}; - double now = td::Time::now(); - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - auto r = R.move_as_ok(); - std::vector> by_name; - for (const auto &stats : r) { - if (stats.name == query.name_ || query.name_.empty()) { - std::vector> by_time; - for (const auto &t : times) { - double min = std::numeric_limits::lowest(); - double max = std::numeric_limits::max(); - double sum = 0; - int cnt = 0; - for (const auto &stat : stats.stats) { - double time = stat.first; - double duration = stat.second; - if (now - time <= static_cast(t)) { - min = td::min(min, duration); - max = td::max(max, duration); - sum += duration; - ++cnt; - } - } - by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), query = std::move(query)]( + td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; } - by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); } + by_time.push_back(ton::create_tl_object( + t, min, sum / static_cast(cnt), max)); } - promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + by_name.push_back(ton::create_tl_object( + stats.name, std::move(by_time))); } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); + } + promise.set_value( + ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, + std::move(P)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, @@ -3711,9 +3715,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMes }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3753,9 +3756,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3783,9 +3785,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3795,8 +3796,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto return; } - promise.set_value(ton::serialize_tl_object( - custom_overlays_config_, true)); + promise.set_value( + ton::serialize_tl_object(custom_overlays_config_, true)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, @@ -4189,24 +4190,25 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); td::uint32 threads = 7; - p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { - td::int32 v; - try { - v = std::stoi(arg.str()); - } catch (...) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); - } - if (v <= 0) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); - } - if (v > 127) { - LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; - v = 127; - } - threads = v; - return td::Status::OK(); - }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << threads << ")", + [&](td::Slice arg) { + td::int32 v; + try { + v = std::stoi(arg.str()); + } catch (...) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); + } + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; + } + threads = v; + threads = 127; + return td::Status::OK(); + }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); p.add_checked_option('\0', "shutdown-at", "stop validator at the given time (unix timestamp)", [&](td::Slice arg) { TRY_RESULT(at, td::to_integer_safe(arg)); @@ -4223,7 +4225,8 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - '\0', "max-archive-fd", "limit for a number of open file descriptirs in archive manager. 0 is unlimited (default)", + '\0', "max-archive-fd", + "limit for a number of open file descriptirs in archive manager. 0 is unlimited (default)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_open_archive_files, v); }); @@ -4258,13 +4261,16 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); - p.add_option( - '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); }); - p.add_option( - '\0', "celldb-preload-all", - "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); + p.add_option('\0', "celldb-direct-io", + "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); + }); + p.add_option('\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and " + "celldb-direct-io)", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { @@ -4275,16 +4281,18 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); return td::Status::OK(); }); - p.add_checked_option( - '\0', "catchain-max-block-delay-slow", "max extended catchain block delay (for too long rounds), (default: 1.0)", - [&](td::Slice s) -> td::Status { - auto v = td::to_double(s); - if (v < 0) { - return td::Status::Error("catchain-max-block-delay-slow should be non-negative"); - } - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay_slow, v); }); - return td::Status::OK(); - }); + p.add_checked_option('\0', "catchain-max-block-delay-slow", + "max extended catchain block delay (for too long rounds), (default: 1.0)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay-slow should be non-negative"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay_slow, v); + }); + return td::Status::OK(); + }); p.add_option( '\0', "fast-state-serializer", "faster persistent state serializer, but requires more RAM (enabled automatically on machines with >= 90GB RAM)", diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 14d3ec469..83ddbbc03 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -22,6 +22,7 @@ #include "files-async.hpp" #include "td/db/RocksDb.h" #include "common/delay.h" +#include "validator/impl/scope_guard.h" namespace ton { @@ -151,23 +152,54 @@ void ArchiveManager::add_temp_file_short(FileReference ref_id, td::BufferSlice d } void ArchiveManager::get_handle(BlockIdExt block_id, td::Promise promise) { + LOG(INFO) << "ArchiveManager::get_handle. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; + auto f = get_file_desc_by_seqno(block_id.shard_full(), block_id.seqno(), false); if (f) { + LOG(INFO) << "ArchiveManager::get_handle: f is valid" + << ". counter" << block_id.counter_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_max_temp_file_desc_idx(), promise = std::move(promise)](td::Result R) mutable { + LOG(INFO) << "ArchiveManager::get_handle. P lambda. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; if (R.is_ok()) { + LOG(INFO) << "ArchiveManager::get_handle: R is valid" + << ". counter" << block_id.counter_; promise.set_value(R.move_as_ok()); } else { + LOG(INFO) << "ArchiveManager::get_handle: R is not valid" + << ". counter" << block_id.counter_; td::actor::send_closure(SelfId, &ArchiveManager::get_handle_cont, block_id, idx, std::move(promise)); } }); td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_handle, block_id, std::move(P)); } else { + LOG(INFO) << "ArchiveManager::get_handle: f is not valid" + << ". counter" << block_id.counter_; get_handle_cont(block_id, get_max_temp_file_desc_idx(), std::move(promise)); } } void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Promise promise) { + LOG(INFO) << "ArchiveManager::get_handle_cont. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = block_id.counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "ArchiveManager::get_handle_cont cost" << elapsed << "μs" + << ". counter" << counter; + })); + if (idx.is_empty()) { promise.set_error(td::Status::Error(ErrorCode::notready, "block handle not in db")); return; @@ -179,6 +211,11 @@ void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Pro } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, idx = get_prev_temp_file_desc_idx(idx), promise = std::move(promise)](td::Result R) mutable { + LOG(INFO) << "ArchiveManager::get_handle_cont. P lambda. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_ << " . R.is_ok:" << R.is_ok(); if (R.is_ok()) { td::actor::send_closure(SelfId, &ArchiveManager::get_handle_finish, R.move_as_ok(), std::move(promise)); } else { @@ -189,12 +226,37 @@ void ArchiveManager::get_handle_cont(BlockIdExt block_id, PackageId idx, td::Pro } void ArchiveManager::get_handle_finish(BlockHandle handle, td::Promise promise) { + const auto counter = handle->id().counter_; + LOG(INFO) << "ArchiveManager::get_handle_finish. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << counter; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "ArchiveManager::get_handle_finish cost" << elapsed << "μs" + << ". counter" << counter; + })); + auto f = get_file_desc_by_seqno(handle->id().shard_full(), handle->id().seqno(), false); if (!f) { promise.set_value(std::move(handle)); return; } auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise)](td::Result R) mutable { + const auto counter = handle->id().counter_; + LOG(INFO) << "ArchiveManager::get_handle_finish: P lambda. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << counter; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "ArchiveManager::get_handle_finish: P lambda cost" << elapsed << "μs" + << ". counter" << counter; + })); if (R.is_ok()) { promise.set_value(R.move_as_ok()); } else { @@ -342,19 +404,19 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", - std::move(path), std::move(data), std::move(P)) + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data), + std::move(P)) .release(); }; add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_state, + std::function write_state, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", - std::move(path), std::move(write_state), std::move(P)) + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), + std::move(write_state), std::move(P)) .release(); }; add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); @@ -624,8 +686,8 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, + archive_lru_.get(), statistics_); m.emplace(id, std::move(desc)); update_permanent_slices(); @@ -659,8 +721,8 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - new_desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, + archive_lru_.get(), statistics_); const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { update_desc(f, desc, shard, seqno, ts, lt); @@ -940,7 +1002,8 @@ void ArchiveManager::start_up() { void ArchiveManager::alarm() { alarm_timestamp() = td::Timestamp::in(60.0); auto stats = statistics_.to_string_and_reset(); - auto to_file_r = td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); + auto to_file_r = + td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); if (to_file_r.is_error()) { LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error(); return; @@ -1034,7 +1097,7 @@ void ArchiveManager::persistent_state_gc(std::pair last) { } if (res != 0) { delay_action([key, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); return; } @@ -1051,7 +1114,7 @@ void ArchiveManager::persistent_state_gc(std::pair last) { } if (!allow_delete) { delay_action([key, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); return; } @@ -1082,9 +1145,9 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); perm_states_.erase(it); } - delay_action([key, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, - td::Timestamp::in(1.0)); + delay_action( + [key, SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + td::Timestamp::in(1.0)); } PackageId ArchiveManager::get_temp_package_id() const { diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index d392431a9..d882fac95 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -25,13 +25,14 @@ #include "common/delay.h" #include "files-async.hpp" #include "db-utils.h" +#include "validator/impl/scope_guard.h" namespace ton { namespace validator { class PackageStatistics { - public: + public: void record_open(uint64_t count = 1) { open_count.fetch_add(count, std::memory_order_relaxed); } @@ -39,7 +40,7 @@ class PackageStatistics { void record_close(uint64_t count = 1) { close_count.fetch_add(count, std::memory_order_relaxed); } - + void record_read(double time, uint64_t bytes) { read_bytes.fetch_add(bytes, std::memory_order_relaxed); std::lock_guard guard(read_mutex); @@ -56,10 +57,10 @@ class PackageStatistics { std::stringstream ss; ss.setf(std::ios::fixed); ss.precision(6); - + ss << "ton.pack.open COUNT : " << open_count.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.close COUNT : " << close_count.exchange(0, std::memory_order_relaxed) << "\n"; - + ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; @@ -82,7 +83,7 @@ class PackageStatistics { return ss.str(); } - private: + private: std::atomic_uint64_t open_count{0}; std::atomic_uint64_t close_count{0}; PercentileStats read_time; @@ -118,7 +119,7 @@ void PackageWriter::append(std::string filename, td::BufferSlice data, return; } start = td::Timestamp::now(); - offset = p->append(std::move(filename), std::move(data), !async_mode_); + offset = p->append(std::move(filename), std::move(data), !async_mode_); end = td::Timestamp::now(); size = p->size(); } @@ -131,8 +132,12 @@ void PackageWriter::append(std::string filename, td::BufferSlice data, class PackageReader : public td::actor::Actor { public: PackageReader(std::shared_ptr package, td::uint64 offset, - td::Promise> promise, std::shared_ptr statistics) - : package_(std::move(package)), offset_(offset), promise_(std::move(promise)), statistics_(std::move(statistics)) { + td::Promise> promise, + std::shared_ptr statistics) + : package_(std::move(package)) + , offset_(offset) + , promise_(std::move(promise)) + , statistics_(std::move(statistics)) { } void start_up() override { auto start = td::Timestamp::now(); @@ -314,6 +319,18 @@ void ArchiveSlice::add_file_cont(size_t idx, FileReference ref_id, td::uint64 of } void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise promise) { + LOG(INFO) << "ArchiveSlice::get_handle. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = block_id.counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "ArchiveSlice::get_handle cost" << elapsed << "μs" + << ". counter" << counter; + })); + if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; @@ -386,7 +403,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P promise.set_value(std::move(R.move_as_ok().second)); } }); - td::actor::create_actor("reader", p->package, offset, std::move(P), statistics_.pack_statistics).release(); + td::actor::create_actor("reader", p->package, offset, std::move(P), statistics_.pack_statistics) + .release(); } void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, @@ -491,14 +509,14 @@ void ArchiveSlice::get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno return get_block_common( account_id, [seqno](ton_api::db_lt_desc_value &w) { - return seqno > static_cast(w.last_seqno_) - ? 1 - : seqno == static_cast(w.last_seqno_) ? 0 : -1; + return seqno > static_cast(w.last_seqno_) ? 1 + : seqno == static_cast(w.last_seqno_) ? 0 + : -1; }, [seqno](ton_api::db_lt_el_value &w) { - return seqno > static_cast(w.id_->seqno_) - ? 1 - : seqno == static_cast(w.id_->seqno_) ? 0 : -1; + return seqno > static_cast(w.id_->seqno_) ? 1 + : seqno == static_cast(w.id_->seqno_) ? 0 + : -1; }, true, std::move(promise)); } @@ -644,7 +662,7 @@ void ArchiveSlice::do_close() { packages_.clear(); } -template +template td::Promise ArchiveSlice::begin_async_query(td::Promise promise) { ++active_queries_; return [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 3071f565d..36e44b179 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -332,6 +332,11 @@ void RootDb::store_block_handle(BlockHandle handle, td::Promise promis } void RootDb::get_block_handle(BlockIdExt id, td::Promise promise) { + LOG(INFO) << "RootDb::get_block_handle. start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << id.counter_; td::actor::send_closure(archive_db_, &ArchiveManager::get_handle, id, std::move(promise)); } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 859289562..418254538 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -819,6 +819,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, LOG(INFO) << "perform_getAccountState cost " << elapsed << "μs" << ". counter" << c; })); + blkid.counter_ = c; if (blkid.id.workchain != masterchainId && blkid.id.workchain != workchain) { fatal_error("reference block for a getAccountState() must belong to the masterchain"); return; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3a77f2301..2f0afa6c4 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,7 +152,7 @@ class ValidatorManagerImpl : public ValidatorManager { void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, @@ -402,21 +402,25 @@ class ValidatorManagerImpl : public ValidatorManager { get_block_handle(block_id, false, promise.wrap([](BlockHandle &&handle) -> ConstBlockHandle { return handle; })); } void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + LOG(INFO) << "get_block_state_for_litequery: manager-disk. " + << "counter" << block_id.counter_; get_block_data_from_db_short(block_id, std::move(promise)); } void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + LOG(INFO) << "get_block_state_for_litequery: manager-disk. " + << "counter" << block_id.counter_; get_shard_state_from_db_short(block_id, std::move(promise)); } void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) override { + td::Promise promise) override { get_block_by_lt_from_db(account, lt, std::move(promise)); } void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override { + td::Promise promise) override { get_block_by_unix_time_from_db(account, ts, std::move(promise)); } void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override { + td::Promise promise) override { get_block_by_seqno_from_db(account, seqno, std::move(promise)); } void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index cf4d3799f..e8218310d 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -179,7 +179,7 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + std::function write_data, td::Promise promise) override { UNREACHABLE(); } @@ -464,21 +464,25 @@ class ValidatorManagerImpl : public ValidatorManager { get_block_handle(block_id, false, promise.wrap([](BlockHandle &&handle) -> ConstBlockHandle { return handle; })); } void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + LOG(INFO) << "get_block_data_for_litequery: manager-hardfork. " + << "counter" << block_id.counter_; get_block_data_from_db_short(block_id, std::move(promise)); } void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + LOG(INFO) << "get_block_state_for_litequery: manager-hardfork. " + << "counter" << block_id.counter_; get_shard_state_from_db_short(block_id, std::move(promise)); } void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) override { + td::Promise promise) override { get_block_by_lt_from_db(account, lt, std::move(promise)); } void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) override { + td::Promise promise) override { get_block_by_unix_time_from_db(account, ts, std::move(promise)); } void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) override { + td::Promise promise) override { get_block_by_seqno_from_db(account, seqno, std::move(promise)); } void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, diff --git a/validator/manager.cpp b/validator/manager.cpp index 9058be76a..677691e0e 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1188,7 +1188,7 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + std::function write_data, td::Promise promise) { td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, std::move(write_data), std::move(promise)); @@ -1404,6 +1404,8 @@ void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref sta } void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promise promise) { + LOG(INFO) << "get_block_handle. " + << "counter" << id.counter_; if (!id.is_valid()) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block id")); return; @@ -1412,6 +1414,8 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi // updates LRU position if found auto B = get_handle_from_lru(id); if (B) { + LOG(INFO) << "get_block_handle: hit lru. " + << "counter" << id.counter_; CHECK(B->id() == id); promise.set_value(std::move(B)); return; @@ -1422,6 +1426,8 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi auto handle = it->second.lock(); if (handle) { CHECK(handle->id() == id); + LOG(INFO) << "get_block_handle: hit handles_. " + << "counter" << id.counter_; promise.set_value(std::move(handle)); return; } else { @@ -1431,10 +1437,14 @@ void ValidatorManagerImpl::get_block_handle(BlockIdExt id, bool force, td::Promi auto it2 = wait_block_handle_.find(id); if (it2 != wait_block_handle_.end()) { + LOG(INFO) << "get_block_handle: hit wait_block_handle_. " + << "counter" << id.counter_; it2->second.waiting_.emplace_back(std::move(promise)); return; } + LOG(INFO) << "get_block_handle: not hit any cache. " + << "counter" << id.counter_; wait_block_handle_.emplace(id, WaitBlockHandle{}); wait_block_handle_[id].waiting_.emplace_back(std::move(promise)); @@ -1941,7 +1951,7 @@ void ValidatorManagerImpl::update_shards() { td::uint32 threshold = 9407194; bool force_group_id_upgrade = last_masterchain_seqno_ == threshold; auto legacy_opts_hash = opts.get_hash(); - if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() + if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); @@ -2032,7 +2042,6 @@ void ValidatorManagerImpl::update_shards() { auto legacy_val_group_id = get_validator_set_id(shard, val_set, legacy_opts_hash, key_seqno, opts); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); - auto it = validator_groups_.find(legacy_val_group_id); if (it != validator_groups_.end()) { new_validator_groups_.emplace(val_group_id, std::move(it->second)); @@ -2127,8 +2136,8 @@ void ValidatorManagerImpl::update_shards() { new_next_validator_groups_.emplace(val_group_id, std::move(it->second)); } else { new_next_validator_groups_.emplace( - val_group_id, - ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); + val_group_id, ValidatorGroupEntry{ + create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } @@ -2953,9 +2962,18 @@ void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndVa } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { + LOG(INFO) << "ValidatorManagerImpl::get_block_handle_for_litequery. " + << "counter" << block_id.counter_; get_block_handle(block_id, false, [SelfId = actor_id(this), block_id, promise = std::move(promise), allow_not_applied = opts_->nonfinal_ls_queries_enabled()](td::Result R) mutable { + LOG(INFO) << "ValidatorManagerImpl::get_block_handle_for_litequery: manager.cpp. " + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; + if (R.is_ok() && (allow_not_applied || R.ok()->is_applied())) { promise.set_value(R.move_as_ok()); } else { @@ -2966,33 +2984,66 @@ void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, t } void ValidatorManagerImpl::get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) { + LOG(INFO) << "get_block_data_for_litequery: manager.cpp. " + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; + if (candidates_buffer_.empty()) { - get_block_handle_for_litequery( - block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, std::move(handle), - std::move(promise)); - }); + LOG(INFO) << "candidates_buffer_ is empty. " + << "counter" << block_id.counter_; + get_block_handle_for_litequery(block_id, [manager = actor_id(this), promise = std::move(promise), + block_id](td::Result R) mutable { + LOG(INFO) << "get_block_data_for_litequery: manager.cpp. lambda 1. " + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + + .count() + << ". counter" << block_id.counter_; + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, std::move(handle), + std::move(promise)); + }); } else { + LOG(INFO) << "candidates_buffer_ is not empty. " + << "counter" << block_id.counter_; td::actor::send_closure( candidates_buffer_, &CandidatesBuffer::get_block_data, block_id, [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { + auto counter = block_id.counter_; + LOG(INFO) << "get_block_data_for_litequery: manager.cpp. lambda 2" + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << block_id.counter_; if (R.is_ok()) { promise.set_result(R.move_as_ok()); return; } - td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, - [manager, promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, - std::move(handle), std::move(promise)); - }); + td::actor::send_closure( + manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, + [manager, promise = std::move(promise), counter](td::Result R) mutable { + LOG(INFO) << "get_block_data_for_litequery: manager.cpp. lambda 3" + << "start time: " + << std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count() + << ". counter" << counter; + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, std::move(handle), + std::move(promise)); + }); }); } } void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) { + LOG(INFO) << "get_block_state_for_litequery: manager"; if (candidates_buffer_.empty()) { get_block_handle_for_litequery( block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -3008,18 +3059,18 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, promise.set_result(R.move_as_ok()); return; } - td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, - block_id, [manager, promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), - std::move(promise)); - }); + td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, + [manager, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, + std::move(handle), std::move(promise)); + }); }); } } void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, - td::Promise promise) { + td::Promise promise) { get_block_by_lt_from_db( account, lt, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_ok() && R.ok()->is_applied()) { @@ -3032,7 +3083,7 @@ void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull acc } void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, - td::Promise promise) { + td::Promise promise) { get_block_by_unix_time_from_db( account, ts, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_ok() && R.ok()->is_applied()) { @@ -3045,7 +3096,7 @@ void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixF } void ValidatorManagerImpl::get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, - td::Promise promise) { + td::Promise promise) { get_block_by_seqno_from_db( account, seqno, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { From 15e0933d68e73548935499cb9353c18e9c1d48ae Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 12 Oct 2024 10:47:48 +0800 Subject: [PATCH 007/101] fix build --- validator/impl/liteserver.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index bf5a30f54..7b6b33127 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -39,7 +39,6 @@ class LiteQuery : public td::actor::Actor { td::actor::ActorId manager_; td::actor::ActorId cache_; td::Timestamp timeout_; - string query_index_; td::Promise promise_; td::Promise, UnixTime, LogicalTime, std::unique_ptr>> From 956e26047d4b558fc81ee043437f62e2ade32e5e Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 12 Oct 2024 14:13:11 +0800 Subject: [PATCH 008/101] add bin perf tools --- build-docker.sh | 1 + perf.sh | 6 ++++++ run.sh | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 build-docker.sh create mode 100644 perf.sh create mode 100644 run.sh diff --git a/build-docker.sh b/build-docker.sh new file mode 100644 index 000000000..4970290af --- /dev/null +++ b/build-docker.sh @@ -0,0 +1 @@ +docker build -t ton -f ./Dockerfile . diff --git a/perf.sh b/perf.sh new file mode 100644 index 000000000..f7ce8913f --- /dev/null +++ b/perf.sh @@ -0,0 +1,6 @@ +sudo perf script > out.perf +git clone https://github.com/brendangregg/Flamegraph.git +perl Flamegraph/stackcollapse-perf.pl out.perf > out.folded +perl Flamegraph/flamegraph.pl out.folded > flamegraph.svg + +mv flamegraph.svg /data/zjg/transferfile \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100644 index 000000000..ef00c655b --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +rm flamegraph.svg out.folded out.perf perf.data ton.log +perf record -F 99 -a -g ./build/validator-engine/validator-engine -c /data/zjg/ton/data_ton/docker/volumes/ton-db/_data/config.json -C /data/zjg/ton/data_ton/docker/volumes/ton-db/_data/ton-global.config --db /data/zjg/ton/data_ton/docker/volumes/ton-db/_data --state-ttl 315360000 --archive-ttl 315360000 --threads 8 --verbosity 3 --block-ttl 315360000 > ton.log 2>&1 \ No newline at end of file From 0741fcd3b3bbbf364e28115015a36f550b36c0dd Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 12 Oct 2024 18:25:21 +0800 Subject: [PATCH 009/101] update --- validator/impl/liteserver.cpp | 20 +++++++++++++++++++- validator/manager.cpp | 5 ++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 604f81c4b..a1d9ac95d 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -643,7 +643,14 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { td::PerfWarningTimer timer{"okxdebug-request_mc_block_state", 0.01}; - LOG(INFO) << "okxdebug-request_mc_block_state"; + + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "got_block_state cost " << elapsed << "μs" + << ". counter" << counter; + })); + if (!blkid.is_masterchain() || !blkid.is_valid_full()) { return fatal_error("reference block must belong to the masterchain"); } @@ -659,6 +666,12 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); } else { + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "before got_mc_block_state cost " << elapsed << "μs" + << ". counter" << counter; + })); td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_state, blkid, res.move_as_ok()); } }); @@ -1220,6 +1233,11 @@ void LiteQuery::got_zero_state(BlockIdExt blkid, td::BufferSlice zerostate) { } void LiteQuery::check_pending() { + SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + const auto end{std::chrono::steady_clock::now()}; + const auto elapsed = std::chrono::duration_cast(end - start).count(); + LOG(INFO) << "check_pending cost " << elapsed << "μs" << ". counter" << counter; + })); CHECK(pending_ >= 0); if (!pending_) { if (!cont_set_) { diff --git a/validator/manager.cpp b/validator/manager.cpp index 677691e0e..3f21fe4f8 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2962,8 +2962,7 @@ void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndVa } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { - LOG(INFO) << "ValidatorManagerImpl::get_block_handle_for_litequery. " - << "counter" << block_id.counter_; + LOG(INFO) << "ValidatorManagerImpl::get_block_handle_for_litequery. " << "counter" << block_id.counter_; get_block_handle(block_id, false, [SelfId = actor_id(this), block_id, promise = std::move(promise), allow_not_applied = opts_->nonfinal_ls_queries_enabled()](td::Result R) mutable { @@ -3043,7 +3042,7 @@ void ValidatorManagerImpl::get_block_data_for_litequery(BlockIdExt block_id, td: void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) { - LOG(INFO) << "get_block_state_for_litequery: manager"; + LOG(INFO) << "get_block_state_for_litequery: manager" << "zjg, counter" << block_id.counter_; if (candidates_buffer_.empty()) { get_block_handle_for_litequery( block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { From fc4357ccef1db6d42287060e1eb9620415016c68 Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 12 Oct 2024 18:26:47 +0800 Subject: [PATCH 010/101] update --- perf.sh | 0 run.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 perf.sh mode change 100644 => 100755 run.sh diff --git a/perf.sh b/perf.sh old mode 100644 new mode 100755 diff --git a/run.sh b/run.sh old mode 100644 new mode 100755 From a7ed3148d708b5a183a7d5d3fcf2987999d37c89 Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 12 Oct 2024 18:45:00 +0800 Subject: [PATCH 011/101] update --- validator/impl/liteserver.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index a1d9ac95d..aa7295355 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -666,12 +666,12 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); } else { - SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { - const auto end{std::chrono::steady_clock::now()}; - const auto elapsed = std::chrono::duration_cast(end - start).count(); - LOG(INFO) << "before got_mc_block_state cost " << elapsed << "μs" - << ". counter" << counter; - })); + // SCOPE_EXIT(([start = std::chrono::steady_clock::now(), counter = counter_]() { + // const auto end{std::chrono::steady_clock::now()}; + // const auto elapsed = std::chrono::duration_cast(end - start).count(); + // LOG(INFO) << "before got_mc_block_state cost " << elapsed << "μs" + // << ". counter" << counter; + // })); td::actor::send_closure_later(Self, &LiteQuery::got_mc_block_state, blkid, res.move_as_ok()); } }); From ef48bad8623ffa82f6a03f33b1cf72aaf918cfda Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 15:07:23 +0800 Subject: [PATCH 012/101] update --- validator/db/rootdb.cpp | 7 +++++++ validator/manager.cpp | 14 ++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 36e44b179..d19d15e4e 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -249,22 +249,29 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { + auto now = std::chrono::system_clock::now(); + LOG(INFO) << "get_block_state: " << " b_count" << now << ", 1"; if (handle->inited_state_boc()) { + // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } + int query_count = + // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 3"; auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { + // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 4"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); + // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 5"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3f21fe4f8..30f8b4302 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3042,15 +3042,17 @@ void ValidatorManagerImpl::get_block_data_for_litequery(BlockIdExt block_id, td: void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) { - LOG(INFO) << "get_block_state_for_litequery: manager" << "zjg, counter" << block_id.counter_; + LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 1"; if (candidates_buffer_.empty()) { + LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 2"; get_block_handle_for_litequery( - block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, handle, std::move(R)); - td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), - std::move(promise)); - }); + block_id, [manager = actor_id(this), promise = std::move(promise), block_id](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), + std::move(promise)); + }); } else { + LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 3"; td::actor::send_closure( candidates_buffer_, &CandidatesBuffer::get_block_state, block_id, [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { From d4f60cc54707d37c65331dadf336ef0ee3c766e0 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 15:10:38 +0800 Subject: [PATCH 013/101] update --- validator/db/rootdb.cpp | 1 - validator/manager.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index d19d15e4e..a2d774147 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -257,7 +257,6 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> R) mutable { diff --git a/validator/manager.cpp b/validator/manager.cpp index 30f8b4302..75b34a7f0 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3046,7 +3046,7 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, if (candidates_buffer_.empty()) { LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 2"; get_block_handle_for_litequery( - block_id, [manager = actor_id(this), promise = std::move(promise), block_id](td::Result R) mutable { + block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, handle, std::move(R)); td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), std::move(promise)); From 074783f83c988e0dfd26009126825788b1a3309e Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 15:45:10 +0800 Subject: [PATCH 014/101] update --- validator/db/rootdb.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index a2d774147..9343865d9 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -252,25 +252,25 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promiseinited_state_boc()) { - // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 2"; + LOG(INFO) << "get_block_state: " << " b_count" << now << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 3"; + LOG(INFO) << "get_block_state: " << " b_count" << now << ", 3"; auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 4"; + LOG(INFO) << "get_block_state: " << " b_count" << now << ", 4"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); - // LOG(INFO) << "get_block_state: " << " b_count" << now << ", 5"; + LOG(INFO) << "get_block_state: " << " b_count" << now << ", 5"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } From 6c4cd60c999695c636f549db2d105ee64cde96f5 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 15:49:52 +0800 Subject: [PATCH 015/101] update --- validator/db/rootdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 9343865d9..aafeab5b8 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -259,7 +259,7 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> R) mutable { + td::PromiseCreator::lambda([handle, promise = std::move(promise), now](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { From 1e32df34c061733e6d41f21a8a61e39c223022f2 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 15:55:47 +0800 Subject: [PATCH 016/101] update --- validator/db/rootdb.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index aafeab5b8..8e6685d5c 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -249,28 +249,29 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { - auto now = std::chrono::system_clock::now(); - LOG(INFO) << "get_block_state: " << " b_count" << now << ", 1"; + auto now_time = std::chrono::system_clock::now(); + std::time_t now = std::chrono::system_clock::to_time_t(now_time); + LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 1"; if (handle->inited_state_boc()) { - LOG(INFO) << "get_block_state: " << " b_count" << now << ", 2"; + LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - LOG(INFO) << "get_block_state: " << " b_count" << now << ", 3"; + LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 3"; auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise), now](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - LOG(INFO) << "get_block_state: " << " b_count" << now << ", 4"; + LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 4"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); - LOG(INFO) << "get_block_state: " << " b_count" << now << ", 5"; + LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 5"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } From 29cf27c8e21c0d6ec36d6d10e22153835fb5413e Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 16:03:45 +0800 Subject: [PATCH 017/101] update --- validator/db/rootdb.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8e6685d5c..1f0a32c2e 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -249,29 +249,29 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { - auto now_time = std::chrono::system_clock::now(); - std::time_t now = std::chrono::system_clock::to_time_t(now_time); - LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 1"; + auto now = std::chrono::system_clock::now(); + auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); + LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 1"; if (handle->inited_state_boc()) { - LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 2"; + LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 3"; + LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 3"; auto P = - td::PromiseCreator::lambda([handle, promise = std::move(promise), now](td::Result> R) mutable { + td::PromiseCreator::lambda([handle, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 4"; + LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 4"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); - LOG(INFO) << "get_block_state: " << " b_count" << std::ctime(&now) << ", 5"; + LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 5"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } From ac8ce0ab595a430060315f5f5cd4deb919cda3b7 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 16:41:08 +0800 Subject: [PATCH 018/101] update --- validator/db/celldb.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 1701ae588..508454c9a 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -439,11 +439,17 @@ void CellDbIn::migrate_cells() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { + auto now = std::chrono::system_clock::now(); + auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); + LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 1"; if (!started_) { + LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 2"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { + LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 3"; auto P = td::PromiseCreator::lambda( - [cell_db_in = cell_db_.get(), hash, promise = std::move(promise)](td::Result> R) mutable { + [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { + LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 4"; if (R.is_error()) { td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); } else { @@ -451,6 +457,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise } }); boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); + LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 5"; } } From 535620c4ee3e35af737f7cfc86596497d71afc66 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 17:00:47 +0800 Subject: [PATCH 019/101] update --- validator/db/celldb.cpp | 10 +++++----- validator/db/rootdb.cpp | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 508454c9a..80336713f 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -441,23 +441,23 @@ void CellDbIn::migrate_cells() { void CellDb::load_cell(RootHash hash, td::Promise> promise) { auto now = std::chrono::system_clock::now(); auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); - LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 1"; + LOG(INFO) << "load_cell: " << timestamp_ns << ", 1"; if (!started_) { - LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 2"; + LOG(INFO) << "load_cell: " << timestamp_ns << ", 2"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { - LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 3"; + LOG(INFO) << "load_cell: " << timestamp_ns << ", 3"; auto P = td::PromiseCreator::lambda( [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { - LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 4"; + LOG(INFO) << "load_cell: " << timestamp_ns << ", 5"; if (R.is_error()) { td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); } else { promise.set_result(R.move_as_ok()); } }); + LOG(INFO) << "load_cell: " << timestamp_ns << ", 4"; boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); - LOG(INFO) << "load_cell: " << " b_count" << timestamp_ns << ", 5"; } } diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 1f0a32c2e..257f21440 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -251,27 +251,27 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { auto now = std::chrono::system_clock::now(); auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); - LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 1"; + LOG(INFO) << "get_block_state: " << timestamp_ns << ", 1"; if (handle->inited_state_boc()) { - LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 2"; + LOG(INFO) << "get_block_state: " << timestamp_ns << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 3"; + LOG(INFO) << "get_block_state: " << timestamp_ns << ", 3"; auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 4"; + LOG(INFO) << "get_block_state: " << timestamp_ns << ", 5"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); + LOG(INFO) << "get_block_state: " << timestamp_ns << ", 4"; td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); - LOG(INFO) << "get_block_state: " << " b_count" << timestamp_ns << ", 5"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } From 021926da11e254347b116d8dedced3cde44e6b53 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 17:09:59 +0800 Subject: [PATCH 020/101] update --- validator/db/celldb.cpp | 13 ++++++++----- validator/db/rootdb.cpp | 10 +++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 80336713f..a44a724e0 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -441,23 +441,26 @@ void CellDbIn::migrate_cells() { void CellDb::load_cell(RootHash hash, td::Promise> promise) { auto now = std::chrono::system_clock::now(); auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); - LOG(INFO) << "load_cell: " << timestamp_ns << ", 1"; + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 1"; if (!started_) { - LOG(INFO) << "load_cell: " << timestamp_ns << ", 2"; + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 2"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { - LOG(INFO) << "load_cell: " << timestamp_ns << ", 3"; + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 3"; auto P = td::PromiseCreator::lambda( [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { - LOG(INFO) << "load_cell: " << timestamp_ns << ", 5"; + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 5"; if (R.is_error()) { td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 6"; } else { promise.set_result(R.move_as_ok()); + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 7"; } }); - LOG(INFO) << "load_cell: " << timestamp_ns << ", 4"; + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 4"; boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); + LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 4-1"; } } diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 257f21440..4e6fe0fa3 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -251,26 +251,26 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { auto now = std::chrono::system_clock::now(); auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); - LOG(INFO) << "get_block_state: " << timestamp_ns << ", 1"; + LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 1"; if (handle->inited_state_boc()) { - LOG(INFO) << "get_block_state: " << timestamp_ns << ", 2"; + LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - LOG(INFO) << "get_block_state: " << timestamp_ns << ", 3"; + LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 3"; auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - LOG(INFO) << "get_block_state: " << timestamp_ns << ", 5"; + LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 5"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); - LOG(INFO) << "get_block_state: " << timestamp_ns << ", 4"; + LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 4"; td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); From 6c32828704426f94f6eec426d01937b524e477f2 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 17:39:48 +0800 Subject: [PATCH 021/101] update --- validator/db/rootdb.cpp | 19 +++++++++---------- validator/db/rootdb.hpp | 2 +- validator/impl/candidates-buffer.cpp | 2 +- validator/impl/candidates-buffer.hpp | 2 +- validator/interfaces/db.h | 2 +- validator/manager-disk.cpp | 6 +++--- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.cpp | 6 +++--- validator/manager-hardfork.hpp | 2 +- validator/manager-init.cpp | 2 +- validator/manager.cpp | 8 ++++---- validator/manager.hpp | 2 +- 12 files changed, 27 insertions(+), 28 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 4e6fe0fa3..aa356a65f 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -244,34 +244,33 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, }); td::actor::send_closure(cell_db_, &CellDb::store_cell, handle->id(), state->root_cell(), std::move(P)); } else { - get_block_state(handle, std::move(promise)); + get_block_state(handle, std::move(promise), 0); } } -void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { - auto now = std::chrono::system_clock::now(); - auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); - LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 1"; +void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { + LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 1"; if (handle->inited_state_boc()) { - LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 2"; + LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 3"; + LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 3"; auto P = - td::PromiseCreator::lambda([handle, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { + td::PromiseCreator::lambda([handle, promise = std::move(promise), counter_](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 5"; + LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 5"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); - LOG(INFO) << "timestamp get_block_state: " << timestamp_ns << ", 4"; + LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 4"; td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); + LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 4-1"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 061e9add8..02349c038 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -61,7 +61,7 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; - void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; + void get_block_state(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) override; void get_cell_db_reader(td::Promise> promise) override; void get_last_deleted_mc_state(td::Promise promise) override; diff --git a/validator/impl/candidates-buffer.cpp b/validator/impl/candidates-buffer.cpp index e24913b67..5ef1db867 100644 --- a/validator/impl/candidates-buffer.cpp +++ b/validator/impl/candidates-buffer.cpp @@ -85,7 +85,7 @@ void CandidatesBuffer::got_block_candidate(BlockIdExt id, td::Result> promise) { +void CandidatesBuffer::get_block_state(BlockIdExt id, td::Promise> promise, std::uint64_t counter_) { auto it = candidates_.find(id); if (it == candidates_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "unknown block candidate")); diff --git a/validator/impl/candidates-buffer.hpp b/validator/impl/candidates-buffer.hpp index 5db1e5160..480e99af4 100644 --- a/validator/impl/candidates-buffer.hpp +++ b/validator/impl/candidates-buffer.hpp @@ -31,7 +31,7 @@ class CandidatesBuffer : public td::actor::Actor { void add_new_candidate(BlockIdExt id, PublicKey source, FileHash collated_data_file_hash); void get_block_data(BlockIdExt id, td::Promise> promise); - void get_block_state(BlockIdExt id, td::Promise> promise); + void get_block_state(BlockIdExt id, td::Promise> promise, std::uint64_t counter_); private: td::actor::ActorId manager_; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index e0d88e4e7..3b7656342 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -49,7 +49,7 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; - virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; + virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void get_last_deleted_mc_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 75c83b04d..a0050d2bd 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -565,8 +565,8 @@ void ValidatorManagerImpl::get_block_data_from_db_short(BlockIdExt block_id, td: get_block_handle(block_id, false, std::move(P)); } -void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) { - td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise)); +void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { + td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise), counter_); } void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, @@ -580,7 +580,7 @@ void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, promise.set_error(R.move_as_error()); } else { auto handle = R.move_as_ok(); - td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise)); + td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise), 2); } }); get_block_handle(block_id, false, std::move(P)); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 2f0afa6c4..0fff13d0b 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -209,7 +209,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) override; - void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) override; void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index d1e10779e..61ce8ff06 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -391,8 +391,8 @@ void ValidatorManagerImpl::get_block_data_from_db_short(BlockIdExt block_id, td: get_block_handle(block_id, false, std::move(P)); } -void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) { - td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise)); +void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { + td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise), counter_); } void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, @@ -405,7 +405,7 @@ void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, promise.set_error(R.move_as_error()); } else { auto handle = R.move_as_ok(); - td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise)); + td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise), 4); } }); get_block_handle(block_id, false, std::move(P)); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index e8218310d..4156d0418 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -254,7 +254,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) override; - void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) override; void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index aa110380d..6cfe561c4 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -519,7 +519,7 @@ void ValidatorManagerMasterchainStarter::got_truncate_block_handle(BlockHandle h td::actor::send_closure(SelfId, &ValidatorManagerMasterchainStarter::got_truncate_state, td::Ref{R.move_as_ok()}); }); - td::actor::send_closure(db_, &Db::get_block_state, handle_, std::move(P)); + td::actor::send_closure(db_, &Db::get_block_state, handle_, std::move(P), 5); } void ValidatorManagerMasterchainStarter::got_truncate_state(td::Ref state) { diff --git a/validator/manager.cpp b/validator/manager.cpp index 75b34a7f0..cf1065580 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1011,8 +1011,8 @@ void ValidatorManagerImpl::get_block_data_from_db_short(BlockIdExt block_id, td: get_block_handle(block_id, false, std::move(P)); } -void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) { - td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise)); +void ValidatorManagerImpl::get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { + td::actor::send_closure(db_, &Db::get_block_state, handle, std::move(promise), counter_); } void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, @@ -1023,7 +1023,7 @@ void ValidatorManagerImpl::get_shard_state_from_db_short(BlockIdExt block_id, promise.set_error(R.move_as_error()); } else { auto handle = R.move_as_ok(); - td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise)); + td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(promise), 7); } }); get_block_handle(block_id, false, std::move(P)); @@ -1889,7 +1889,7 @@ void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) td::actor::send_closure(client, &ShardClient::force_update_shard_client_ex, std::move(handle), td::Ref{R.move_as_ok()}, std::move(P)); }); - td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(P)); + td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(P), 8); }); get_block_handle(b, true, std::move(P)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 388cb99a2..529088562 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -456,7 +456,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) override; - void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) override; + void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) override; void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; From c78bbeae7807ad0e7f4dbd04bb93cd6aca58fd06 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 17:50:52 +0800 Subject: [PATCH 022/101] update --- validator/validator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/validator.h b/validator/validator.h index afc32f3b9..e9a25efff 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -254,7 +254,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; - virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; + virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) = 0; virtual void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) = 0; From 9821e864236507fff070d994556d6ea8de4b083d Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 17:59:47 +0800 Subject: [PATCH 023/101] update --- validator/manager.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index cf1065580..4bb123804 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3045,17 +3045,19 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 1"; if (candidates_buffer_.empty()) { LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 2"; + std::uint64_t counter = block_id.counter_; get_block_handle_for_litequery( - block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + block_id, [manager = actor_id(this), promise = std::move(promise), counter](td::Result R) mutable { TRY_RESULT_PROMISE(promise, handle, std::move(R)); td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), - std::move(promise)); + std::move(promise), counter); }); } else { LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 3"; + std::uint64_t counter = block_id.counter_; td::actor::send_closure( candidates_buffer_, &CandidatesBuffer::get_block_state, block_id, - [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { + [manager = actor_id(this), promise = std::move(promise), block_id, counter](td::Result> R) mutable { if (R.is_ok()) { promise.set_result(R.move_as_ok()); return; @@ -3064,7 +3066,7 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, [manager, promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, handle, std::move(R)); td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, - std::move(handle), std::move(promise)); + std::move(handle), std::move(promise), counter); }); }); } From 27398791413b533fbb7984683a864ee46bfc8bef Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:01:30 +0800 Subject: [PATCH 024/101] update --- validator/manager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 4bb123804..cd6f31990 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3054,10 +3054,9 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, }); } else { LOG(INFO) << "get_block_state_for_litequery: manager" << " counter" << block_id.counter_ << ", 3"; - std::uint64_t counter = block_id.counter_; td::actor::send_closure( candidates_buffer_, &CandidatesBuffer::get_block_state, block_id, - [manager = actor_id(this), promise = std::move(promise), block_id, counter](td::Result> R) mutable { + [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { if (R.is_ok()) { promise.set_result(R.move_as_ok()); return; @@ -3066,7 +3065,7 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, [manager, promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, handle, std::move(R)); td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, - std::move(handle), std::move(promise), counter); + std::move(handle), std::move(promise), 0); }); }); } From a13adc9acfbe300f5929fee2cf7a6a7417c3bcb3 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:10:20 +0800 Subject: [PATCH 025/101] update --- validator-engine/validator-engine.cpp | 2 +- validator/downloaders/wait-block-state.cpp | 2 +- validator/full-node.cpp | 2 +- validator/import-db-slice.cpp | 2 +- validator/manager-init.cpp | 4 ++-- validator/shard-client.cpp | 4 ++-- validator/state-serializer.cpp | 4 ++-- validator/validate-broadcast.cpp | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 80ab387db..854202158 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3684,7 +3684,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardO } else { promise.set_value(res.move_as_ok()); } - }); + }, 0); }); } diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 0ae82beaa..97ec84d8d 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -76,7 +76,7 @@ void WaitBlockState::start() { td::actor::send_closure(SelfId, &WaitBlockState::got_state_from_db, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P), 0); } else if (handle_->id().id.seqno == 0 && next_static_file_attempt_.is_in_past()) { next_static_file_attempt_ = td::Timestamp::in(60.0); // id.file_hash contrains correct file hash of zero state diff --git a/validator/full-node.cpp b/validator/full-node.cpp index a72be3ff4..59196c04c 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -421,7 +421,7 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, - std::move(P)); + std::move(P), 0); } else { CHECK(handle->is_key_block()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index a93fb05be..943f05ed0 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -210,7 +210,7 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, td::Ref(R.move_as_ok())); }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P), 0); } void ArchiveImporter::got_new_materchain_state(td::Ref state) { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 6cfe561c4..ba2cf3b48 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -349,7 +349,7 @@ void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handl td::Ref{R.move_as_ok()}); }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P), 0); } void ValidatorManagerMasterchainStarter::got_init_block_state(td::Ref state) { @@ -393,7 +393,7 @@ void ValidatorManagerMasterchainStarter::got_gc_block_handle(BlockHandle handle) td::Ref{R.move_as_ok()}); }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, gc_handle_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, gc_handle_, std::move(P), 0); } void ValidatorManagerMasterchainStarter::got_gc_block_state(td::Ref state) { diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 8fad36122..11f86cd73 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -66,7 +66,7 @@ void ShardClient::got_init_handle_from_db(BlockHandle handle) { td::actor::send_closure(SelfId, &ShardClient::got_init_state_from_db, td::Ref{R.move_as_ok()}); }); td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_block_handle_, - std::move(P)); + std::move(P), 0); } void ShardClient::got_init_state_from_db(td::Ref state) { @@ -279,7 +279,7 @@ void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise{R.move_as_ok()}, std::move(promise)); }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, std::move(handle), std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, std::move(handle), std::move(P), 0); } void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref state, diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 516d8177f..59072de88 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -119,7 +119,7 @@ void AsyncStateSerializer::request_masterchain_state() { }); } }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P), 0); } void AsyncStateSerializer::request_shard_state(BlockIdExt shard) { @@ -373,7 +373,7 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { } }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P), 0); } void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, diff --git a/validator/validate-broadcast.cpp b/validator/validate-broadcast.cpp index 59cb1faca..ad6aa4b18 100644 --- a/validator/validate-broadcast.cpp +++ b/validator/validate-broadcast.cpp @@ -148,7 +148,7 @@ void ValidateBroadcast::got_key_block_handle(ConstBlockHandle handle) { td::actor::send_closure(SelfId, &ValidateBroadcast::got_zero_state, td::Ref{R.move_as_ok()}); } }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P), 0); } else { if (!handle->inited_proof() && !handle->inited_proof_link()) { abort_query(td::Status::Error(ErrorCode::notready, "reference key block proof not received")); From 8ca7ee2d9c84df5c21fb94d8e7fc3d11a85e6a7f Mon Sep 17 00:00:00 2001 From: oker Date: Mon, 14 Oct 2024 18:02:12 +0800 Subject: [PATCH 026/101] add log for celldb message queue --- validator/db/celldb.cpp | 46 +++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 1701ae588..b0222aae8 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -22,6 +22,7 @@ #include "td/db/RocksDb.h" #include "td/utils/filesystem.h" +#include "td/utils/logging.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" @@ -38,11 +39,13 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { void execute_async(std::function f) override { class Runner : public td::actor::Actor { public: - explicit Runner(std::function f) : f_(std::move(f)) {} + explicit Runner(std::function f) : f_(std::move(f)) { + } void start_up() override { f_(); stop(); } + private: std::function f_; }; @@ -52,6 +55,7 @@ class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { void execute_sync(std::function f) override { td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f)); } + private: td::actor::ActorId cell_db_; }; @@ -99,7 +103,6 @@ void CellDbIn::start_up() { db_options.use_direct_reads = opts_->get_celldb_direct_io(); cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); - boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); @@ -117,23 +120,25 @@ void CellDbIn::start_up() { if (opts_->get_celldb_preload_all()) { // Iterate whole DB in a separate thread - delay_action([snapshot = cell_db_->snapshot()]() { - LOG(WARNING) << "CellDb: pre-loading all keys"; - td::uint64 total = 0; - td::Timer timer; - auto S = snapshot->for_each([&](td::Slice, td::Slice) { - ++total; - if (total % 1000000 == 0) { - LOG(INFO) << "CellDb: iterated " << total << " keys"; - } - return td::Status::OK(); - }); - if (S.is_error()) { - LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); - } else { - LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; - } - }, td::Timestamp::now()); + delay_action( + [snapshot = cell_db_->snapshot()]() { + LOG(WARNING) << "CellDb: pre-loading all keys"; + td::uint64 total = 0; + td::Timer timer; + auto S = snapshot->for_each([&](td::Slice, td::Slice) { + ++total; + if (total % 1000000 == 0) { + LOG(INFO) << "CellDb: iterated " << total << " keys"; + } + return td::Status::OK(); + }); + if (S.is_error()) { + LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); + } else { + LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; + } + }, + td::Timestamp::now()); } } @@ -401,7 +406,7 @@ void CellDbIn::migrate_cells() { boc_->set_loader(std::make_unique(*loader)).ensure(); cell_db_->begin_write_batch().ensure(); td::uint32 checked = 0, migrated = 0; - for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128; ) { + for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128;) { ++checked; td::Bits256 hash = *it; it = cells_to_migrate_.erase(it); @@ -439,6 +444,7 @@ void CellDbIn::migrate_cells() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { + LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); if (!started_) { td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { From 42112b5f0e2ceef70fa163902fa029820d906127 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:16:18 +0800 Subject: [PATCH 027/101] update --- validator/downloaders/wait-block-state.cpp | 2 +- validator/manager.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 97ec84d8d..5a4622082 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -292,7 +292,7 @@ void WaitBlockState::force_read_from_db() { td::actor::send_closure(SelfId, &WaitBlockState::got_state_from_db, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P), 0); } void WaitBlockState::got_state_from_net(td::BufferSlice data) { diff --git a/validator/manager.cpp b/validator/manager.cpp index cd6f31990..3b211d7e4 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3067,7 +3067,7 @@ void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), std::move(promise), 0); }); - }); + }, 0); } } From 72be9f1baeebafa14c07530637b0db6834f4e585 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:27:03 +0800 Subject: [PATCH 028/101] update --- validator/db/rootdb.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index aa356a65f..570632cd7 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -249,28 +249,28 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { - LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 1"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 1"; if (handle->inited_state_boc()) { - LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 2"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 2"; if (handle->deleted_state_boc()) { promise.set_error(td::Status::Error(ErrorCode::error, "state already gc'd")); return; } - LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 3"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 3"; auto P = td::PromiseCreator::lambda([handle, promise = std::move(promise), counter_](td::Result> R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 5"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 5"; auto S = create_shard_state(handle->id(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); } }); - LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 4"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 4"; td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P)); - LOG(INFO) << "timestamp get_block_state: " << counter_ << ", 4-1"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 4-1"; } else { promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } From 1ed485f98b99099dc5e70e07b607f664e8652ab5 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:51:30 +0800 Subject: [PATCH 029/101] update --- validator/db/archive-mover.cpp | 2 +- validator/db/celldb.cpp | 22 ++++++++++------------ validator/db/celldb.hpp | 4 ++-- validator/db/rootdb.cpp | 4 +++- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/validator/db/archive-mover.cpp b/validator/db/archive-mover.cpp index aa89570d9..f3854f89c 100644 --- a/validator/db/archive-mover.cpp +++ b/validator/db/archive-mover.cpp @@ -416,7 +416,7 @@ void ArchiveMover::got_handle(BlockHandle handle) { S.ensure(); td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref{S.move_as_ok()}); }); - td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P)); + td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P), 0); } void ArchiveMover::got_state(td::Ref state) { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index a44a724e0..57f1b32dc 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -438,29 +438,27 @@ void CellDbIn::migrate_cells() { } } -void CellDb::load_cell(RootHash hash, td::Promise> promise) { - auto now = std::chrono::system_clock::now(); - auto timestamp_ns = std::chrono::duration_cast(now.time_since_epoch()).count(); - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 1"; +void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { + LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; if (!started_) { - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 2"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 3"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; auto P = td::PromiseCreator::lambda( - [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), timestamp_ns](td::Result> R) mutable { - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 5"; + [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { + LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; if (R.is_error()) { td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 6"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 6"; } else { promise.set_result(R.move_as_ok()); - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 7"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 7"; } }); - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 4"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 4"; boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); - LOG(INFO) << "timestamp load_cell: " << timestamp_ns << ", 4-1"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 4-1"; } } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 335d8a08e..df9a69102 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -58,7 +58,7 @@ class CellDbIn : public CellDbBase { public: using KeyHash = td::Bits256; - void load_cell(RootHash hash, td::Promise> promise); + void load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); void get_last_deleted_mc_state(td::Promise promise); @@ -162,7 +162,7 @@ class CellDbIn : public CellDbBase { class CellDb : public CellDbBase { public: - void load_cell(RootHash hash, td::Promise> promise); + void load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void update_snapshot(std::unique_ptr snapshot) { started_ = true; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 570632cd7..23599cb99 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -269,11 +269,13 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promisestate(), std::move(P)); + td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P), counter_); LOG(INFO) << "get_block_state, counter" << counter_ << ", 4-1"; } else { + LOG(INFO) << "get_block_state, counter" << counter_ << ", 6-0"; promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } + LOG(INFO) << "get_block_state, counter" << counter_ << ", 6"; } void RootDb::get_cell_db_reader(td::Promise> promise) { From f08a1e14bffb9bb9d4d0d3c4c3bc2adf13721c14 Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:52:50 +0800 Subject: [PATCH 030/101] update --- validator/db/rootdb.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 23599cb99..d5eb1b5b4 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -266,16 +266,17 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promiseid(), R.move_as_ok()); S.ensure(); promise.set_value(S.move_as_ok()); + LOG(INFO) << "get_block_state, counter" << counter_ << ", 6"; } }); LOG(INFO) << "get_block_state, counter" << counter_ << ", 4"; td::actor::send_closure(cell_db_, &CellDb::load_cell, handle->state(), std::move(P), counter_); LOG(INFO) << "get_block_state, counter" << counter_ << ", 4-1"; } else { - LOG(INFO) << "get_block_state, counter" << counter_ << ", 6-0"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 7-0"; promise.set_error(td::Status::Error(ErrorCode::notready, "state not in db")); } - LOG(INFO) << "get_block_state, counter" << counter_ << ", 6"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 7"; } void RootDb::get_cell_db_reader(td::Promise> promise) { From c3dbb0d657a93add0f98f2cff6d3daa3edae9e4a Mon Sep 17 00:00:00 2001 From: jianguo Date: Mon, 14 Oct 2024 18:56:41 +0800 Subject: [PATCH 031/101] update --- validator/db/celldb.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index df9a69102..1fa4d508f 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -58,7 +58,7 @@ class CellDbIn : public CellDbBase { public: using KeyHash = td::Bits256; - void load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_); + void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); void get_last_deleted_mc_state(td::Promise promise); From d852e43a18481ac9df892baa8640db987f6c2da3 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 10:24:17 +0800 Subject: [PATCH 032/101] update --- validator/db/celldb.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 57f1b32dc..3cbd924e5 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -452,6 +452,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); LOG(INFO) << " load_cell: counter" << counter_ << ", 6"; } else { + LOG(INFO) << " load_cell: counter" << counter_ << ", 7-0"; promise.set_result(R.move_as_ok()); LOG(INFO) << " load_cell: counter" << counter_ << ", 7"; } From 4ef6b9339d5dfe30dab79abc83f6ea7c85bf820b Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 10:43:03 +0800 Subject: [PATCH 033/101] update --- validator/db/rootdb.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index d5eb1b5b4..868736ede 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -264,9 +264,12 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promiseid(), R.move_as_ok()); + LOG(INFO) << "get_block_state, counter" << counter_ << ", 5-1"; S.ensure(); + LOG(INFO) << "get_block_state, counter" << counter_ << ", 5-2"; promise.set_value(S.move_as_ok()); - LOG(INFO) << "get_block_state, counter" << counter_ << ", 6"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 5-3"; + LOG(INFO) << "get_block_state, counter" << counter_ << ", 6"; } }); LOG(INFO) << "get_block_state, counter" << counter_ << ", 4"; From c2c6a5e9e192c1a0eec134462a05197979460242 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 10:55:13 +0800 Subject: [PATCH 034/101] update --- validator/impl/fabric.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index d69492393..1dad40f9d 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -85,10 +85,14 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff } td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { + LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 1"; auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); + LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 2"; if (res.is_error()) { + LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 3"; return res.move_as_error(); } else { + LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 4"; return td::Ref{res.move_as_ok()}; } } From e971692467b5a52f8e0ce709cdb859dd6f5eca76 Mon Sep 17 00:00:00 2001 From: oker Date: Tue, 15 Oct 2024 10:57:59 +0800 Subject: [PATCH 035/101] test1 --- tdactor/td/actor/common.h | 7 ++++--- tdactor/td/actor/core/ActorExecutor.cpp | 14 ++++++++------ tdactor/td/actor/core/CpuWorker.cpp | 2 ++ tdactor/td/actor/core/Scheduler.cpp | 3 ++- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index d38589747..2908bf236 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -324,7 +324,7 @@ void send_closure_impl(ActorRef actor_ref, ClosureT &&closure) { } template -void send_closure(ActorRef actor_ref, ArgsT &&... args) { +void send_closure(ActorRef actor_ref, ArgsT &&...args) { send_closure_impl(actor_ref, create_immediate_closure(std::forward(args)...)); } @@ -365,7 +365,7 @@ void send_closure_with_promise_later(ActorRef actor_ref, ClosureT &&closure, Pro } template -void send_closure_later(ActorRef actor_ref, ArgsT &&... args) { +void send_closure_later(ActorRef actor_ref, ArgsT &&...args) { send_closure_later_impl(actor_ref, create_delayed_closure(std::forward(args)...)); } @@ -396,11 +396,12 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); + LOG(INFO) << "yus register acotr " << actor_info_ptr->get_name() << "thread id " << state.get_scheduler_id().value(); core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } template -core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&... args) noexcept { +core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&...args) noexcept { auto *scheduler_context = core::SchedulerContext::get(); if (!options.has_scheduler()) { options.on_scheduler(scheduler_context->get_scheduler_id()); diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 267758d5e..e6f4e5275 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -127,13 +127,14 @@ void ActorExecutor::start() noexcept { } void ActorExecutor::finish() noexcept { - //LOG(ERROR) << "FINISH " << actor_info_.get_name() << " " << tag("own_lock", actor_locker_.own_lock()); + LOG(ERROR) << "yus actor executror FINISH " << actor_info_.get_name() << " " + << tag("own_lock", actor_locker_.own_lock()); if (!actor_locker_.own_lock()) { if (!pending_signals_.empty() && actor_locker_.add_signals(pending_signals_)) { flags_ = actor_locker_.flags(); - //LOG(ERROR) << "Own after finish " << actor_info_.get_name() << " " << format::as_binary(flags().raw()); + LOG(ERROR) << "yus Own after finish " << actor_info_.get_name() << " " << format::as_binary(flags().raw()); } else { - //LOG(ERROR) << "DO FINISH " << actor_info_.get_name() << " " << flags(); + LOG(ERROR) << "yus DO FINISH " << actor_info_.get_name() << " " << flags(); return; } } else { @@ -155,10 +156,10 @@ void ActorExecutor::finish() noexcept { signals.clear_signal(ActorSignals::Pop); flags().set_signals(signals); flags().set_in_queue(false); - //LOG(ERROR) << "clear in_queue " << format::as_binary(flags().raw()); + LOG(ERROR) << "yus clear in_queue " << format::as_binary(flags().raw()); } - //LOG(ERROR) << tag("in_queue", flags().is_in_queue()) << tag("has_signals", flags().has_signals()); + LOG(ERROR) << tag(" yus in_queue", flags().is_in_queue()) << tag("has_signals", flags().has_signals()); if (flags_.is_closed()) { // Writing to mailbox and closing actor may happen concurrently // We must ensure that all messages in mailbox will be deleted @@ -177,13 +178,14 @@ void ActorExecutor::finish() noexcept { } if (actor_locker_.try_unlock(flags())) { if (add_to_queue) { + LOG(INFO) << "yus add to queue " << flags().get_scheduler_id().value(); dispatcher_.add_to_queue(std::move(actor_info_ptr), flags().get_scheduler_id(), !flags().is_shared()); } break; } flags_ = actor_locker_.flags(); } - //LOG(ERROR) << "DO FINISH " << actor_info_.get_name() << " " << flags(); + LOG(ERROR) << "yus DO FINISH " << actor_info_.get_name() << " " << flags(); } bool ActorExecutor::flush_one_signal(ActorSignals &signals) { diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index d78660c6f..38a86e313 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -33,11 +33,13 @@ void CpuWorker::run() { MpmcWaiter::Slot slot; waiter_.init_slot(slot, thread_id); auto &debug = dispatcher.get_debug(); + LOG(INFO) << "yus thread_id" << thread_id; while (true) { SchedulerMessage message; if (try_pop(message, thread_id)) { waiter_.stop_wait(slot); if (!message) { + LOG(INFO) << "yus no message return" << " thread_id" << thread_id; return; } auto lock = debug.start(message->get_name()); diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 40f1e72cd..b1dd73c68 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -76,6 +76,7 @@ Scheduler::~Scheduler() { } void Scheduler::start() { + LOG(INFO) << "yus start scheduler"; for (size_t i = 0; i < cpu_threads_.size(); i++) { cpu_threads_[i] = td::thread([this, i] { this->run_in_context_impl(*this->info_->cpu_workers[i], [this, i] { @@ -156,7 +157,7 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler if (!scheduler_id.is_valid()) { scheduler_id = get_scheduler_id(); } - //LOG(ERROR) << "Add to queue: " << actor_info_ptr->get_name() << " " << scheduler_id.value(); + LOG(ERROR) << "Add to queue: " << actor_info_ptr->get_name() << " " << scheduler_id.value(); auto &info = scheduler_group()->schedulers.at(scheduler_id.value()); if (need_poll || !info.cpu_queue) { info.io_queue->writer_put(std::move(actor_info_ptr)); From 1309950559f5f013705dc2867dcf1b9c9ca6c802 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 11:36:01 +0800 Subject: [PATCH 036/101] update --- validator/db/archive-mover.cpp | 2 +- validator/db/rootdb.cpp | 4 ++-- validator/fabric.h | 2 +- validator/impl/fabric.cpp | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/validator/db/archive-mover.cpp b/validator/db/archive-mover.cpp index f3854f89c..26e00470b 100644 --- a/validator/db/archive-mover.cpp +++ b/validator/db/archive-mover.cpp @@ -412,7 +412,7 @@ void ArchiveMover::got_handle(BlockHandle handle) { auto P = td::PromiseCreator::lambda( [handle = handle_, SelfId = actor_id(this)](td::Result> R) mutable { R.ensure(); - auto S = create_shard_state(handle->id(), R.move_as_ok()); + auto S = create_shard_state(handle->id(), R.move_as_ok(), 0); S.ensure(); td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref{S.move_as_ok()}); }); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 868736ede..d41068465 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -230,7 +230,7 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, handle->set_state_root_hash(root_hash); handle->set_state_boc(); - auto S = create_shard_state(handle->id(), R.move_as_ok()); + auto S = create_shard_state(handle->id(), R.move_as_ok(), 0); S.ensure(); auto P = td::PromiseCreator::lambda( @@ -263,7 +263,7 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promiseid(), R.move_as_ok()); + auto S = create_shard_state(handle->id(), R.move_as_ok(), counter_); LOG(INFO) << "get_block_state, counter" << counter_ << ", 5-1"; S.ensure(); LOG(INFO) << "get_block_state, counter" << counter_ << ", 5-2"; diff --git a/validator/fabric.h b/validator/fabric.h index 949a6c9ff..ba4078c50 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -37,7 +37,7 @@ td::Result> create_proof(BlockIdExt masterchain_block_id, td::Buf td::Result> create_proof_link(BlockIdExt block_id, td::BufferSlice proof); td::Result> create_signature_set(td::BufferSlice sig_set); td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell, std::uint64_t counter_); td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 1dad40f9d..8b9df41a3 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -84,15 +84,15 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff } } -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { - LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 1"; +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell, std::uint64_t counter_) { + LOG(INFO) << "create_shard_state, counter" << counter_ << ", 1"; auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); - LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 2"; + LOG(INFO) << "create_shard_state, counter" << counter_ << ", 2"; if (res.is_error()) { - LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 3"; + LOG(INFO) << "create_shard_state, counter" << counter_ << ", 3"; return res.move_as_error(); } else { - LOG(INFO) << "create_shard_state, counter" << block_id.counter_ << ", 4"; + LOG(INFO) << "create_shard_state, counter" << counter_ << ", 4"; return td::Ref{res.move_as_ok()}; } } From e365ed2ffdd2a96cea05684ecbf06acb4f6a7d80 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 11:57:22 +0800 Subject: [PATCH 037/101] update --- validator/impl/fabric.cpp | 1 + validator/impl/shard.cpp | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 8b9df41a3..919abad0d 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -86,6 +86,7 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell, std::uint64_t counter_) { LOG(INFO) << "create_shard_state, counter" << counter_ << ", 1"; + block_id.counter_ = counter_; auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); LOG(INFO) << "create_shard_state, counter" << counter_ << ", 2"; if (res.is_error()) { diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 9c4245b6a..c7563ac95 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -62,19 +62,27 @@ ShardStateQ::ShardStateQ(const BlockIdExt& _id, Ref _root, td::BufferS } td::Result> ShardStateQ::fetch(const BlockIdExt& _id, td::BufferSlice _data, Ref _root) { + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 1"; if (_id.is_masterchain()) { + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 2"; auto res = MasterchainStateQ::fetch(_id, std::move(_data), std::move(_root)); if (res.is_error()) { + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 3"; return res.move_as_error(); } else { + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 4"; return Ref{res.move_as_ok()}; } } + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 5"; Ref res{true, _id, std::move(_root), std::move(_data)}; + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 6"; td::Status err = res.unique_write().init(); if (err.is_error()) { + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 7"; return err; } else { + LOG(INFO) << "fetch, counter" << _id.counter_ << ", 8"; return std::move(res); } } From 16a5ef4293143e8045afe6a3a046685d49c2401f Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 12:14:02 +0800 Subject: [PATCH 038/101] update --- validator/impl/shard.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index c7563ac95..bfc98ffd5 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -359,15 +359,22 @@ MasterchainStateQ* MasterchainStateQ::make_copy() const { td::Result> MasterchainStateQ::fetch(const BlockIdExt& _id, td::BufferSlice _data, Ref _root) { + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 1"; if (!ShardIdFull(_id).is_masterchain_ext()) { + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 2"; return td::Status::Error(-666, "invalid masterchain block/state id passed for creating a new masterchain state object"); } + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 3"; Ref res{true, _id, std::move(_root), std::move(_data)}; + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 4"; td::Status err = res.unique_write().mc_init(); + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 5"; if (err.is_error()) { + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 6"; return err; } else { + LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 7"; return std::move(res); } } From 9c66887a56db4f29e83e4718442b95028e803c55 Mon Sep 17 00:00:00 2001 From: oker Date: Tue, 15 Oct 2024 13:25:47 +0800 Subject: [PATCH 039/101] check worker id --- tdactor/td/actor/core/CpuWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index 38a86e313..6eef2ffef 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -33,7 +33,7 @@ void CpuWorker::run() { MpmcWaiter::Slot slot; waiter_.init_slot(slot, thread_id); auto &debug = dispatcher.get_debug(); - LOG(INFO) << "yus thread_id" << thread_id; + LOG(INFO) << "yus thread_id" << thread_id << "worker id " << this->id_; while (true) { SchedulerMessage message; if (try_pop(message, thread_id)) { From f7e11424aac665d44aff6102aee5eaa37b72c5a9 Mon Sep 17 00:00:00 2001 From: oker Date: Tue, 15 Oct 2024 13:25:32 +0800 Subject: [PATCH 040/101] check schedule --- tdactor/td/actor/core/Scheduler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index b1dd73c68..64cfa6838 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -67,6 +67,8 @@ Scheduler::Scheduler(std::shared_ptr scheduler_group_info, S scheduler_group_info_->iocp.init(); } #endif + LOG(INFO) << "yus scheduler create " << "id " << id.value() << "count " << cpu_threads_count << "scheduler_group_ " + << scheduler_group_info_->active_scheduler_count << " " << scheduler_group_info_->schedulers.size(); } Scheduler::~Scheduler() { From 48ba8b50bf2344d54c4951e2a4a1671926a144bc Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 14:20:02 +0800 Subject: [PATCH 041/101] update --- validator/impl/liteserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index aa7295355..3988fe45e 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -824,7 +824,7 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { return true; } -static std::atomic_uint64_t counter; +static std::atomic_uint64_t counter = 1000; void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, int mode) { td::PerfWarningTimer timer{"okxdebug-perform_getAccountState", 0.01}; From 76bbee02c572ca25788608c14c5c7349f4f6bd36 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 15:07:33 +0800 Subject: [PATCH 042/101] update --- validator/impl/shard.cpp | 10 +++++++--- validator/impl/shard.hpp | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index bfc98ffd5..a0f1d2031 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -368,7 +368,7 @@ td::Result> MasterchainStateQ::fetch(const BlockIdExt& _i LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 3"; Ref res{true, _id, std::move(_root), std::move(_data)}; LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 4"; - td::Status err = res.unique_write().mc_init(); + td::Status err = res.unique_write().mc_init(_id.counter_); LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 5"; if (err.is_error()) { LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 6"; @@ -379,12 +379,16 @@ td::Result> MasterchainStateQ::fetch(const BlockIdExt& _i } } -td::Status MasterchainStateQ::mc_init() { +td::Status MasterchainStateQ::mc_init(std::uint64_t counter_) { + LOG(INFO) << "mc_init, counter" << counter_ << ", 1"; auto err = init(); if (err.is_error()) { return err; } - return mc_reinit(); + LOG(INFO) << "mc_init, counter" << counter_ << ", 2"; + auto a = mc_reinit(); + LOG(INFO) << "mc_init, counter" << counter_ << ", 3"; + return a; } td::Status MasterchainStateQ::mc_reinit() { diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index d9a7dd655..c75113f8b 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -165,7 +165,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { std::shared_ptr config_; std::shared_ptr cur_validators_, next_validators_; MasterchainStateQ(const MasterchainStateQ& other) = default; - td::Status mc_init(); + td::Status mc_init(std::uint64_t counter_); td::Status mc_reinit(); Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno cc_seqno) const; From 3ac9aeaf4c8d229b44b929e0101b6766b4e7776b Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 15:12:36 +0800 Subject: [PATCH 043/101] update --- validator/impl/shard.cpp | 30 +++++++++++++++++++++++++----- validator/impl/shard.hpp | 2 +- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index a0f1d2031..98421cd09 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -87,8 +87,10 @@ td::Result> ShardStateQ::fetch(const BlockIdExt& _id, td::Buffe } } -td::Status ShardStateQ::init() { +td::Status ShardStateQ::init(std::uint64_t counter_) { + LOG(INFO) << "init, counter" << counter_ << ", 1"; if (root.is_null()) { + LOG(INFO) << "init, counter" << counter_ << ", 2"; if (data.empty()) { return td::Status::Error( -668, "cannot initialize shardchain state without either a root cell or a BufferSlice with serialized data"); @@ -96,50 +98,62 @@ td::Status ShardStateQ::init() { #if LAZY_STATE_DESERIALIZE vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; + LOG(INFO) << "init, counter" << counter_ << ", 3"; auto res = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(data.clone()), options); if (res.is_error()) { return res.move_as_error(); } + LOG(INFO) << "init, counter" << counter_ << ", 4"; auto boc = res.move_as_ok(); auto rc = boc->get_root_count(); if (rc.is_error()) { return rc.move_as_error(); } + LOG(INFO) << "init, counter" << counter_ << ", 5"; if (rc.move_as_ok() != 1) { return td::Status::Error(-668, "shardchain state BoC is invalid"); } auto res3 = boc->get_root_cell(0); bocs_.clear(); bocs_.push_back(std::move(boc)); + LOG(INFO) << "init, counter" << counter_ << ", 6"; #else auto res3 = vm::std_boc_deserialize(data.as_slice()); + LOG(INFO) << "init, counter" << counter_ << ", 7"; #endif if (res3.is_error()) { return res3.move_as_error(); } + LOG(INFO) << "init, counter" << counter_ << ", 8"; root = res3.move_as_ok(); if (root.is_null()) { return td::Status::Error(-668, "cannot extract root cell out of a shardchain state BoC"); } + LOG(INFO) << "init, counter" << counter_ << ", 9"; } + LOG(INFO) << "init, counter" << counter_ << ", 10"; rhash = root->get_hash().bits(); block::gen::ShardStateUnsplit::Record info; if (!tlb::unpack_cell(root, info)) { return td::Status::Error(-668, "shardchain state for block "s + blkid.id.to_str() + " does not contain a valid header"); } + LOG(INFO) << "init, counter" << counter_ << ", 11"; lt = info.gen_lt; utime = info.gen_utime; global_id_ = info.global_id; before_split_ = info.before_split; block::ShardId id{info.shard_id}; ton::BlockId hdr_id{ton::ShardIdFull(id), info.seq_no}; + LOG(INFO) << "init, counter" << counter_ << ", 12"; if (!id.is_valid() || get_shard() != ton::ShardIdFull(id) || get_seqno() != info.seq_no) { return td::Status::Error(-668, "header of unpacked shardchain state for block "s + blkid.id.to_str() + " contains BlockId " + hdr_id.to_str() + " different from the one originally required"); } + LOG(INFO) << "init, counter" << counter_ << ", 13"; if (info.r1.master_ref.write().fetch_long(1)) { + LOG(INFO) << "init, counter" << counter_ << ", 14"; BlockIdExt mc_id; if (!block::tlb::t_ExtBlkRef.unpack(info.r1.master_ref, mc_id, nullptr)) { return td::Status::Error(-668, "cannot unpack master_ref in shardchain state of "s + blkid.to_str()); @@ -148,6 +162,7 @@ td::Status ShardStateQ::init() { } else { master_ref = {}; } + LOG(INFO) << "init, counter" << counter_ << ", 15"; return td::Status::OK(); } @@ -381,41 +396,46 @@ td::Result> MasterchainStateQ::fetch(const BlockIdExt& _i td::Status MasterchainStateQ::mc_init(std::uint64_t counter_) { LOG(INFO) << "mc_init, counter" << counter_ << ", 1"; - auto err = init(); + auto err = init(counter_); if (err.is_error()) { return err; } LOG(INFO) << "mc_init, counter" << counter_ << ", 2"; - auto a = mc_reinit(); + auto a = mc_reinit(counter_); LOG(INFO) << "mc_init, counter" << counter_ << ", 3"; return a; } -td::Status MasterchainStateQ::mc_reinit() { +td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1"; auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); cur_validators_.reset(); next_validators_.reset(); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 2"; if (res.is_error()) { return res.move_as_error(); } config_ = res.move_as_ok(); CHECK(config_); CHECK(config_->set_block_id_ext(get_block_id())); - + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 3"; auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); cur_validators_ = std::move(validators); } + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4"; auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); next_validators_ = std::move(validators); } + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 5"; zerostate_id_ = config_->get_zerostate_id(); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 6"; return td::Status::OK(); } diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index c75113f8b..ada207520 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -50,7 +50,7 @@ class ShardStateQ : virtual public ShardState { ShardStateQ(ShardStateQ&& other) = default; public: - td::Status init(); + td::Status init(std::uint64_t counter_); ShardStateQ(const BlockIdExt& _id, td::BufferSlice _data); ShardStateQ(const BlockIdExt& _id, Ref _root, td::BufferSlice _data = {}); virtual ~ShardStateQ() = default; From eee049fd40e325d1b3b7121da6dd3afd13cf7201 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 15:37:12 +0800 Subject: [PATCH 044/101] update --- validator/impl/shard.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 98421cd09..33d945d8e 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -77,7 +77,7 @@ td::Result> ShardStateQ::fetch(const BlockIdExt& _id, td::Buffe LOG(INFO) << "fetch, counter" << _id.counter_ << ", 5"; Ref res{true, _id, std::move(_root), std::move(_data)}; LOG(INFO) << "fetch, counter" << _id.counter_ << ", 6"; - td::Status err = res.unique_write().init(); + td::Status err = res.unique_write().init(_id.counter_); if (err.is_error()) { LOG(INFO) << "fetch, counter" << _id.counter_ << ", 7"; return err; @@ -445,7 +445,7 @@ td::Status MasterchainStateQ::apply_block(BlockIdExt id, td::Ref bloc return err; } config_.reset(); - err = mc_reinit(); + err = mc_reinit(0); if (err.is_error()) { LOG(ERROR) << "cannot extract masterchain-specific state data from newly-computed state for block " << id.id.to_str() << " : " << err.to_string(); @@ -457,7 +457,7 @@ td::Status MasterchainStateQ::prepare() { if (config_) { return td::Status::OK(); } - return mc_reinit(); + return mc_reinit(0); } Ref MasterchainStateQ::compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, From cc6526d90d91d7a2b3db8b93d773fa617508f891 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 15:44:29 +0800 Subject: [PATCH 045/101] update --- validator/impl/shard.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index ada207520..e997d868e 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -166,7 +166,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { std::shared_ptr cur_validators_, next_validators_; MasterchainStateQ(const MasterchainStateQ& other) = default; td::Status mc_init(std::uint64_t counter_); - td::Status mc_reinit(); + td::Status mc_reinit(std::uint64_t counter_); Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno cc_seqno) const; }; From 20fbfae0b2662f1f9aa589f363321cd467da4414 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 16:04:47 +0800 Subject: [PATCH 046/101] update --- validator/impl/shard.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 33d945d8e..6ba4d6a13 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -411,8 +411,11 @@ td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-1"; cur_validators_.reset(); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-2"; next_validators_.reset(); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-3"; LOG(INFO) << "mc_reinit, counter" << counter_ << ", 2"; if (res.is_error()) { return res.move_as_error(); @@ -428,9 +431,13 @@ td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { } LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4"; auto nv_root = config_->get_config_param(37, 36); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-1"; if (nv_root.not_null()) { + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-2"; TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-3"; next_validators_ = std::move(validators); + LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-4"; } LOG(INFO) << "mc_reinit, counter" << counter_ << ", 5"; From cd5a871385ad075d21ef9c91bed6396ab9c7ec07 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 16:25:24 +0800 Subject: [PATCH 047/101] update --- crypto/block/check-proof.cpp | 2 +- crypto/block/create-state.cpp | 2 +- crypto/block/dump-block.cpp | 2 +- crypto/block/mc-config.cpp | 27 ++++++++++++++++++++------- crypto/block/mc-config.h | 4 ++-- lite-client/lite-client.cpp | 2 +- tonlib/tonlib/LastConfig.cpp | 2 +- tonlib/tonlib/TonlibClient.cpp | 2 +- validator/impl/collator.cpp | 4 ++-- validator/impl/liteserver.cpp | 10 +++++----- validator/impl/shard.cpp | 6 +++--- validator/impl/validate-query.cpp | 4 ++-- validator/impl/validator-set.cpp | 4 ++-- 13 files changed, 42 insertions(+), 29 deletions(-) diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 431a03fec..e7d24e265 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -492,7 +492,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { return td::Status::Error("BlockProofLink contains a state proof for "s + from.to_str() + " with incorrect root hash"); } - TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks)); + TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks, 0)); if (!config->check_old_mc_block_id(to, true)) { return td::Status::Error("cannot check that "s + to.to_str() + " is indeed a previous masterchain block of " + from.to_str() + " using the presented Merkle proof of masterchain state"); diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 348377e94..c10768bb0 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -417,7 +417,7 @@ bool store_config_params(vm::CellBuilder& cb) { // stores hash of initial masterchain validator set computed from configuration parameter 34 bool store_validator_list_hash(vm::CellBuilder& cb) { Ref vset_cell = config_dict.lookup_ref(td::BitArray<32>{34}); - auto res = block::Config::unpack_validator_set(std::move(vset_cell)); + auto res = block::Config::unpack_validator_set(std::move(vset_cell), 0); if (res.is_error()) { LOG(ERROR) << "cannot unpack current validator set: " << res.move_as_error().to_string(); return false; diff --git a/crypto/block/dump-block.cpp b/crypto/block/dump-block.cpp index e7a196775..2abc63f63 100644 --- a/crypto/block/dump-block.cpp +++ b/crypto/block/dump-block.cpp @@ -214,7 +214,7 @@ td::Status test_vset() { return td::Status::Error("no config parameter 34"); } std::cerr << "config param #34 obtained\n"; - TRY_RESULT(cur_validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(cur_validators, block::Config::unpack_validator_set(std::move(cv_root), 0)); // auto vconf = config->get_catchain_validators_config(); std::cerr << "validator set unpacked\n"; std::cerr << "unpacking ShardHashes\n"; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 6da0f034d..48f2c8d27 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -98,15 +98,19 @@ td::Result> ConfigInfo::extract_config(std::shared_p return td::Status::Error(-668, "Masterchain state BoC is invalid"); } TRY_RESULT(root, static_boc->get_root_cell(0)); - return extract_config(std::move(root), mode); + return extract_config(std::move(root), mode, 0); } -td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode) { +td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode, std::uint64_t counter_) { + LOG(INFO) << "extract_config, counter" << counter_ << ", 1"; if (mc_state_root.is_null()) { return td::Status::Error("configuration state root cell is null"); } + LOG(INFO) << "extract_config, counter" << counter_ << ", 2"; auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; + LOG(INFO) << "extract_config, counter" << counter_ << ", 3"; TRY_STATUS(config->unpack_wrapped()); + LOG(INFO) << "extract_config, counter" << counter_ << ", 4"; return std::move(config); } @@ -250,7 +254,7 @@ td::Status Config::unpack() { } config_dict = std::make_unique(config_root, 32); if (mode & needValidatorSet) { - auto vset_res = unpack_validator_set(get_config_param(35, 34)); + auto vset_res = unpack_validator_set(get_config_param(35, 34), 0); if (vset_res.is_error()) { return vset_res.move_as_error(); } @@ -294,13 +298,13 @@ td::Status Config::unpack() { td::Status Config::visit_validator_params() const { { // current validator set - TRY_RESULT(vset, unpack_validator_set(get_config_param(34))); + TRY_RESULT(vset, unpack_validator_set(get_config_param(34), 0)); } for (int i = 32; i < 38; i++) { // prev/current/next persistent and temporary validator sets auto vs = get_config_param(i); if (vs.not_null()) { - TRY_RESULT(vset, unpack_validator_set(std::move(vs))); + TRY_RESULT(vset, unpack_validator_set(std::move(vs), 0)); } } get_catchain_validators_config(); @@ -483,13 +487,16 @@ td::Result Config::unpack_workchain_list(Ref root) { return std::move(pair.first); } -td::Result> Config::unpack_validator_set(Ref vset_root) { +td::Result> Config::unpack_validator_set(Ref vset_root, std::uint64_t counter_) { + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 1"; if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 2"; gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 3"; gen::ValidatorSet::Record_validators rec0; if (!tlb::unpack_cell(std::move(vset_root), rec0)) { return td::Status::Error("validator set is invalid"); @@ -500,18 +507,22 @@ td::Result> Config::unpack_validator_set(Refprefetch_ref(); + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 5"; } else { return td::Status::Error("validator set cannot have zero total weight"); } vm::Dictionary dict{std::move(dict_root), 16}; td::BitArray<16> key_buffer; auto last = dict.get_minmax_key(key_buffer.bits(), 16, true); + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 6"; if (last.is_null() || (int)key_buffer.to_ulong() != rec.total - 1) { return td::Status::Error( "maximal index in a validator set dictionary must be one less than the total number of validators"); } + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7"; auto ptr = std::make_unique(rec.utime_since, rec.utime_until, rec.total, rec.main); for (int i = 0; i < rec.total; i++) { key_buffer.store_ulong(i); @@ -542,9 +553,11 @@ td::Result> Config::unpack_validator_set(Reflist.emplace_back(sig_pubkey.pubkey, descr.weight, ptr->total_weight, descr.adnl_addr); ptr->total_weight += descr.weight; } + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 8"; if (rec.total_weight && rec.total_weight != ptr->total_weight) { return td::Status::Error("validator set declares incorrect total weight"); } + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 9"; return std::move(ptr); } @@ -1922,7 +1935,7 @@ std::vector Config::do_compute_validator_set(const block::C } std::vector Config::compute_total_validator_set(int next) const { - auto res = unpack_validator_set(get_config_param(next < 0 ? 32 : (next ? 36 : 34))); + auto res = unpack_validator_set(get_config_param(next < 0 ? 32 : (next ? 36 : 34)), 0); if (res.is_error()) { return {}; } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 25d94ec7e..3237c822b 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -617,7 +617,7 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root); + static td::Result> unpack_validator_set(Ref valset_root, std::uint64_t counter_); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; @@ -762,7 +762,7 @@ class ConfigInfo : public Config, public ShardConfig { td::Result> get_prev_blocks_info() const; static td::Result> extract_config(std::shared_ptr static_boc, int mode = 0); - static td::Result> extract_config(Ref mc_state_root, int mode = 0); + static td::Result> extract_config(Ref mc_state_root, int mode = 0, std::uint64_t counter_); private: ConfigInfo(Ref mc_state_root, int _mode = 0); diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 64383730b..37612fc09 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -3520,7 +3520,7 @@ td::Status TestNode::ValidatorLoadInfo::unpack_vset() { } vset_hash = vset_root->get_hash().bits(); TRY_RESULT_PREFIX_ASSIGN( - vset, block::Config::unpack_validator_set(vset_root), + vset, block::Config::unpack_validator_set(vset_root, 0), PSLICE() << "cannot unpack validator set from configuration parameter 34 of block " << blk_id.to_str() << " :"); valid_since = vset->utime_since; vset_map = vset->compute_validator_map(); diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index e972d84e0..7ab0dd587 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -94,7 +94,7 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrstate_proof_.as_slice(), raw_config->config_proof_.as_slice())); TRY_RESULT(config, block::ConfigInfo::extract_config( - std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); + std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities, 0)); for (auto i : params_) { VLOG(last_config) << "ConfigParam(" << i << ") = "; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index f62b9ae47..977e2c7fe 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1966,7 +1966,7 @@ class RunEmulator : public TonlibQueryActor { } try { - auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111); + auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111, 0); if (r_config.is_error()) { check(r_config.move_as_error()); return; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2cf77bfa3..376a0d46c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -682,7 +682,7 @@ bool Collator::unpack_last_mc_state() { block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | - (is_masterchain() ? block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc : 0)); + (is_masterchain() ? block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc : 0), 0); if (res.is_error()) { td::Status err = res.move_as_error(); LOG(ERROR) << "cannot extract configuration from most recent masterchain state: " << err.to_string(); @@ -4446,7 +4446,7 @@ bool Collator::create_mc_state_extra() { if (cur_vset_cell.is_null()) { cur_vset_cell = cfg_dict_new.lookup_ref(td::BitArray<32>{34}); } - auto res = block::Config::unpack_validator_set(std::move(cur_vset_cell)); + auto res = block::Config::unpack_validator_set(std::move(cur_vset_cell), 0); if (res.is_error()) { auto err = res.move_as_error(); LOG(ERROR) << "cannot unpack current validator set: " << err.to_string(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 3988fe45e..4f2518776 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -979,7 +979,7 @@ void LiteQuery::continue_getLibraries(Ref mc_s mc_state_ = Ref(std::move(mc_state)); CHECK(mc_state_.not_null()); - auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries, 0); if (rconfig.is_error()) { fatal_error("cannot extract library list block configuration from masterchain state"); return; @@ -1358,7 +1358,7 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, BlockIdExt& blkid, A bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid) { vm::MerkleProofBuilder mpb{std::move(state_root)}; - auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks); + auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks, 0); if (rconfig.is_error()) { return fatal_error( "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s + @@ -1453,7 +1453,7 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); if (mode_ & 0x80000000) { - auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF, 0); if (config.is_error()) { fatal_error(config.move_as_error()); return; @@ -1637,7 +1637,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice // **** INIT VM **** auto r_config = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | - block::ConfigInfo::needPrevBlocks); + block::ConfigInfo::needPrevBlocks, 0); if (r_config.is_error()) { fatal_error(r_config.move_as_error()); return; @@ -2030,7 +2030,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) } cfg = res.move_as_ok(); } else { - auto res = block::ConfigInfo::extract_config(mpb.root(), mode); + auto res = block::ConfigInfo::extract_config(mpb.root(), mode, 0); if (res.is_error()) { fatal_error(res.move_as_error()); return; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 6ba4d6a13..21c46da46 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -410,7 +410,7 @@ td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1"; auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); + block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks, counter_); LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-1"; cur_validators_.reset(); LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-2"; @@ -426,7 +426,7 @@ td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { LOG(INFO) << "mc_reinit, counter" << counter_ << ", 3"; auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), 0)); cur_validators_ = std::move(validators); } LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4"; @@ -434,7 +434,7 @@ td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-1"; if (nv_root.not_null()) { LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-2"; - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), counter_)); LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-3"; next_validators_ = std::move(validators); LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-4"; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 003b7f9f7..320c1d6e0 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -837,7 +837,7 @@ bool ValidateQuery::try_unpack_mc_state() { block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | - (is_masterchain() ? block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc : 0)); + (is_masterchain() ? block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc : 0), 0); if (res.is_error()) { return fatal_error(-666, "cannot extract configuration from reference masterchain state "s + mc_blkid_.to_str() + " : " + res.move_as_error().to_string()); @@ -1317,7 +1317,7 @@ bool ValidateQuery::compute_next_state() { state_root_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needAccountsRoot | - block::ConfigInfo::needSpecialSmc | block::ConfigInfo::needCapabilities); + block::ConfigInfo::needSpecialSmc | block::ConfigInfo::needCapabilities, 0); if (r_config_info.is_error()) { return reject_query("cannot extract configuration from new masterchain state "s + mc_blkid_.to_str() + " : " + r_config_info.error().to_string()); diff --git a/validator/impl/validator-set.cpp b/validator/impl/validator-set.cpp index 629337cf8..a3e21f67f 100644 --- a/validator/impl/validator-set.cpp +++ b/validator/impl/validator-set.cpp @@ -137,12 +137,12 @@ td::Status ValidatorSetCompute::init(const block::Config *config) { config_ = config; auto cv_root = config_->get_config_param(34); if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), 0)); cur_validators_ = std::move(validators); } auto nv_root = config_->get_config_param(36); if (nv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), 0)); next_validators_ = std::move(validators); } return td::Status::OK(); From dedf429d194d97c00b779063319627a150053fc9 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 16:35:35 +0800 Subject: [PATCH 048/101] update --- crypto/block/mc-config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 3237c822b..642e002b4 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -762,7 +762,7 @@ class ConfigInfo : public Config, public ShardConfig { td::Result> get_prev_blocks_info() const; static td::Result> extract_config(std::shared_ptr static_boc, int mode = 0); - static td::Result> extract_config(Ref mc_state_root, int mode = 0, std::uint64_t counter_); + static td::Result> extract_config(Ref mc_state_root, int mode = 0, std::uint64_t counter_ = 0); private: ConfigInfo(Ref mc_state_root, int _mode = 0); From ea20f4d3d218bd80e98977918d6956c2284fe030 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 17:03:12 +0800 Subject: [PATCH 049/101] update --- crypto/block/mc-config.cpp | 30 ++++++++++++++++++++++++++---- crypto/block/mc-config.h | 4 ++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 48f2c8d27..7158d5aad 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -109,7 +109,7 @@ td::Result> ConfigInfo::extract_config(Ref LOG(INFO) << "extract_config, counter" << counter_ << ", 2"; auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; LOG(INFO) << "extract_config, counter" << counter_ << ", 3"; - TRY_STATUS(config->unpack_wrapped()); + TRY_STATUS(config->unpack_wrapped(counter_)); LOG(INFO) << "extract_config, counter" << counter_ << ", 4"; return std::move(config); } @@ -119,9 +119,9 @@ ConfigInfo::ConfigInfo(Ref mc_state_root, int _mode) : Config(_mode), block_id.file_hash.set_zero(); } -td::Status ConfigInfo::unpack_wrapped() { +td::Status ConfigInfo::unpack_wrapped(std::uint64_t counter_) { try { - return unpack(); + return unpack(counter_); } catch (vm::VmError& err) { return td::Status::Error(PSLICE() << "error unpacking block state header and configuration: " << err.get_msg()); } catch (vm::VmVirtError& err) { @@ -130,95 +130,117 @@ td::Status ConfigInfo::unpack_wrapped() { } } -td::Status ConfigInfo::unpack() { +td::Status ConfigInfo::unpack(std::uint64_t counter_) { + LOG(INFO) << "unpack, counter" << counter_ << ", 1"; gen::ShardStateUnsplit::Record root_info; if (!tlb::unpack_cell(state_root, root_info) || !root_info.global_id) { return td::Status::Error("configuration state root cannot be deserialized"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 2"; global_id_ = root_info.global_id; block::ShardId shard_id{root_info.shard_id}; block_id.id = ton::BlockId{ton::ShardIdFull(shard_id), (unsigned)root_info.seq_no}; + LOG(INFO) << "unpack, counter" << counter_ << ", 3"; block_id.root_hash.set_zero(); block_id.file_hash.set_zero(); vert_seqno = root_info.vert_seq_no; utime = root_info.gen_utime; lt = root_info.gen_lt; min_ref_mc_seqno_ = root_info.min_ref_mc_seqno; + LOG(INFO) << "unpack, counter" << counter_ << ", 4"; if (!root_info.custom->size_refs()) { return td::Status::Error("state does not have a `custom` field with masterchain configuration"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 5"; if (mode & needLibraries) { lib_root_ = root_info.r1.libraries->prefetch_ref(); libraries_dict_ = std::make_unique(lib_root_, 256); } + LOG(INFO) << "unpack, counter" << counter_ << ", 6"; if (mode & needAccountsRoot) { accounts_root = vm::load_cell_slice_ref(root_info.accounts); LOG(DEBUG) << "requested accounts dictionary"; accounts_dict = std::make_unique(accounts_root, 256, block::tlb::aug_ShardAccounts); LOG(DEBUG) << "accounts dictionary created"; } + LOG(INFO) << "unpack, counter" << counter_ << ", 7"; state_extra_root_ = root_info.custom->prefetch_ref(); if (!is_masterchain()) { if (mode & (needShardHashes | needValidatorSet | needSpecialSmc | needPrevBlocks | needWorkchainInfo)) { return td::Status::Error("cannot extract masterchain-specific configuration data from a non-masterchain state"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 8"; cleanup(); + LOG(INFO) << "unpack, counter" << counter_ << ", 9"; return td::Status::OK(); } + LOG(INFO) << "unpack, counter" << counter_ << ", 10"; gen::McStateExtra::Record extra_info; if (!tlb::unpack_cell(state_extra_root_, extra_info)) { vm::load_cell_slice(state_extra_root_).print_rec(std::cerr); block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_); return td::Status::Error("state extra information is invalid"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 11"; gen::ValidatorInfo::Record validator_info; if (!tlb::csr_unpack(extra_info.r1.validator_info, validator_info)) { return td::Status::Error("validator_info in state extra information is invalid"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 12"; cc_seqno_ = validator_info.catchain_seqno; nx_cc_updated = validator_info.nx_cc_updated; if ((mode & needShardHashes) && !ShardConfig::unpack(extra_info.shard_hashes)) { return td::Status::Error("cannot unpack Shard configuration"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 13"; is_key_state_ = extra_info.r1.after_key_block; if (extra_info.r1.last_key_block->size() > 1) { + LOG(INFO) << "unpack, counter" << counter_ << ", 14"; auto& cs = extra_info.r1.last_key_block.write(); block::gen::ExtBlkRef::Record ext_ref; if (!(cs.advance(1) && tlb::unpack_exact(cs, ext_ref))) { return td::Status::Error("cannot unpack last_key_block from masterchain state"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 15"; last_key_block_.id = ton::BlockId{ton::masterchainId, ton::shardIdAll, ext_ref.seq_no}; last_key_block_.root_hash = ext_ref.root_hash; last_key_block_.file_hash = ext_ref.file_hash; last_key_block_lt_ = ext_ref.end_lt; + LOG(INFO) << "unpack, counter" << counter_ << ", 16"; } else { last_key_block_.invalidate(); last_key_block_.id.seqno = 0; last_key_block_lt_ = 0; + LOG(INFO) << "unpack, counter" << counter_ << ", 17"; } // unpack configuration TRY_STATUS(Config::unpack_wrapped(std::move(extra_info.config))); // unpack previous masterchain block collection std::unique_ptr prev_blocks_dict = std::make_unique(extra_info.r1.prev_blocks, 32, block::tlb::aug_OldMcBlocksInfo); + LOG(INFO) << "unpack, counter" << counter_ << ", 18"; if (block_id.id.seqno) { block::gen::ExtBlkRef::Record extref = {}; auto ref = prev_blocks_dict->lookup(td::BitArray<32>::zero()); if (!(ref.not_null() && ref.write().advance(1) && tlb::csr_unpack(ref, extref) && !extref.seq_no)) { return td::Status::Error("OldMcBlocks in masterchain state does not contain a valid zero state reference"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 19"; zerostate_id_.root_hash = extref.root_hash; zerostate_id_.file_hash = extref.file_hash; } else { zerostate_id_.root_hash.set_zero(); zerostate_id_.file_hash.set_zero(); } + LOG(INFO) << "unpack, counter" << counter_ << ", 20"; zerostate_id_.workchain = ton::masterchainId; if (mode & needPrevBlocks) { prev_blocks_dict_ = std::move(prev_blocks_dict); } + LOG(INFO) << "unpack, counter" << counter_ << ", 21"; // ... cleanup(); + LOG(INFO) << "unpack, counter" << counter_ << ", 22"; return td::Status::OK(); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 642e002b4..bfd698bd5 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -766,8 +766,8 @@ class ConfigInfo : public Config, public ShardConfig { private: ConfigInfo(Ref mc_state_root, int _mode = 0); - td::Status unpack_wrapped(); - td::Status unpack(); + td::Status unpack_wrapped(std::uint64_t counter_ = 0); + td::Status unpack(std::uint64_t counter_ = 0); void reset_mc_hash(); void cleanup(); }; From 090e2e3138f6825260d1efa294dd4676440685cf Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 17:14:23 +0800 Subject: [PATCH 050/101] update --- crypto/block/mc-config.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 7158d5aad..32fe4236c 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -214,7 +214,9 @@ td::Status ConfigInfo::unpack(std::uint64_t counter_) { LOG(INFO) << "unpack, counter" << counter_ << ", 17"; } // unpack configuration + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1"; TRY_STATUS(Config::unpack_wrapped(std::move(extra_info.config))); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-2"; // unpack previous masterchain block collection std::unique_ptr prev_blocks_dict = std::make_unique(extra_info.r1.prev_blocks, 32, block::tlb::aug_OldMcBlocksInfo); From 466be4e00aab0fe076dd6c1d88129d6dde1ea26e Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 17:39:44 +0800 Subject: [PATCH 051/101] update --- crypto/block/mc-config.cpp | 24 ++++++++++++++++++------ crypto/block/mc-config.h | 6 +++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 32fe4236c..8cb109707 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -215,7 +215,7 @@ td::Status ConfigInfo::unpack(std::uint64_t counter_) { } // unpack configuration LOG(INFO) << "unpack, counter" << counter_ << ", 16-1"; - TRY_STATUS(Config::unpack_wrapped(std::move(extra_info.config))); + TRY_STATUS(Config::unpack_wrapped(std::move(extra_info.config), counter_)); LOG(INFO) << "unpack, counter" << counter_ << ", 16-2"; // unpack previous masterchain block collection std::unique_ptr prev_blocks_dict = @@ -246,9 +246,10 @@ td::Status ConfigInfo::unpack(std::uint64_t counter_) { return td::Status::OK(); } -td::Status Config::unpack_wrapped(Ref config_csr) { +td::Status Config::unpack_wrapped(Ref config_csr, std::uint64_t counter_) { try { - return unpack(std::move(config_csr)); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-1"; + return unpack(std::move(config_csr), counter_); } catch (vm::VmError err) { return td::Status::Error(PSLICE() << "error unpacking masterchain configuration: " << err.get_msg()); } @@ -262,17 +263,21 @@ td::Status Config::unpack_wrapped() { } } -td::Status Config::unpack(Ref config_cs) { +td::Status Config::unpack(Ref config_cs, std::uint64_t counter_) { + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-2"; gen::ConfigParams::Record config_params; if (!tlb::csr_unpack(std::move(config_cs), config_params)) { return td::Status::Error("cannot unpack ConfigParams"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-3"; config_addr = config_params.config_addr; config_root = std::move(config_params.config); - return unpack(); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-4"; + return unpack(counter_); } -td::Status Config::unpack() { +td::Status Config::unpack(std::uint64_t counter_) { + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5"; if (config_root.is_null()) { return td::Status::Error("configuration root not set"); } @@ -283,6 +288,7 @@ td::Status Config::unpack() { return vset_res.move_as_error(); } cur_validators_ = vset_res.move_as_ok(); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-6"; } if (mode & needSpecialSmc) { LOG(DEBUG) << "needSpecialSmc flag set"; @@ -293,17 +299,21 @@ td::Status Config::unpack() { special_smc_dict = std::make_unique(vm::load_cell_slice_ref(std::move(param)), 256); LOG(DEBUG) << "smc dictionary created"; } + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-7"; } if (mode & needWorkchainInfo) { TRY_RESULT(pair, unpack_workchain_list_ext(get_config_param(12))); workchains_ = std::move(pair.first); workchains_dict_ = std::move(pair.second); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-8"; } if (mode & needCapabilities) { + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-10"; auto cell = get_config_param(8); if (cell.is_null()) { version_ = 0; capabilities_ = 0; + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-11"; } else { block::gen::GlobalVersion::Record gv; if (!tlb::unpack_cell(std::move(cell), gv)) { @@ -313,9 +323,11 @@ td::Status Config::unpack() { } version_ = gv.version; capabilities_ = gv.capabilities; + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-12"; } } // ... + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-13"; return td::Status::OK(); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index bfd698bd5..51a37d26d 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -674,10 +674,10 @@ class Config { config_addr.set_zero(); } Config(Ref config_root, const td::Bits256& config_addr = td::Bits256::zero(), int _mode = 0); - td::Status unpack_wrapped(Ref config_csr); - td::Status unpack(Ref config_csr); + td::Status unpack_wrapped(Ref config_csr, std::uint64_t counter_ = 0); + td::Status unpack(Ref config_csr, std::uint64_t counter_ = 0)); td::Status unpack_wrapped(); - td::Status unpack(); + td::Status unpack(std::uint64_t counter_ = 0); }; class ConfigInfo : public Config, public ShardConfig { From 351098ecd8e8cc3d742b9686f2ece61b4573da93 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 17:42:37 +0800 Subject: [PATCH 052/101] update --- crypto/block/mc-config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 51a37d26d..918f70643 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -675,7 +675,7 @@ class Config { } Config(Ref config_root, const td::Bits256& config_addr = td::Bits256::zero(), int _mode = 0); td::Status unpack_wrapped(Ref config_csr, std::uint64_t counter_ = 0); - td::Status unpack(Ref config_csr, std::uint64_t counter_ = 0)); + td::Status unpack(Ref config_csr, std::uint64_t counter_ = 0); td::Status unpack_wrapped(); td::Status unpack(std::uint64_t counter_ = 0); }; From 8f2c5648bc8fe70bd56df08b9c0f8e54517ea4bc Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 17:54:00 +0800 Subject: [PATCH 053/101] update --- crypto/block/mc-config.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 8cb109707..683e9d623 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -281,12 +281,17 @@ td::Status Config::unpack(std::uint64_t counter_) { if (config_root.is_null()) { return td::Status::Error("configuration root not set"); } + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-1"; config_dict = std::make_unique(config_root, 32); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-2"; if (mode & needValidatorSet) { + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3"; auto vset_res = unpack_validator_set(get_config_param(35, 34), 0); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-4"; if (vset_res.is_error()) { return vset_res.move_as_error(); } + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-5"; cur_validators_ = vset_res.move_as_ok(); LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-6"; } From c12eea544d881fa81a0a6f76c8a5bf4a8a2b4a08 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 18:07:31 +0800 Subject: [PATCH 054/101] update --- crypto/block/mc-config.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 683e9d623..97b7de718 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -286,7 +286,9 @@ td::Status Config::unpack(std::uint64_t counter_) { LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-2"; if (mode & needValidatorSet) { LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3"; - auto vset_res = unpack_validator_set(get_config_param(35, 34), 0); + auto c = get_config_param(35, 34); + LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3-1"; + auto vset_res = unpack_validator_set(c, 0); LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-4"; if (vset_res.is_error()) { return vset_res.move_as_error(); From b2269f2950ac3ea7c57e5e2d8f9bac78bd7429ca Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 18:14:54 +0800 Subject: [PATCH 055/101] update --- crypto/block/mc-config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 97b7de718..04981fd7d 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -288,7 +288,7 @@ td::Status Config::unpack(std::uint64_t counter_) { LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3"; auto c = get_config_param(35, 34); LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3-1"; - auto vset_res = unpack_validator_set(c, 0); + auto vset_res = unpack_validator_set(c, counter_); LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-4"; if (vset_res.is_error()) { return vset_res.move_as_error(); From 51bbceeedf2f37c329a61333c5835732a67161ea Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 15 Oct 2024 18:28:55 +0800 Subject: [PATCH 056/101] update --- crypto/block/mc-config.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 04981fd7d..f44e29e9f 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -565,16 +565,19 @@ td::Result> Config::unpack_validator_set(Refempty_ext())) { @@ -582,19 +585,23 @@ td::Result> Config::unpack_validator_set(Ref ~(ptr->total_weight)) { return td::Status::Error("total weight of all validators in validator set exceeds 2^64"); } + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-6"; ptr->list.emplace_back(sig_pubkey.pubkey, descr.weight, ptr->total_weight, descr.adnl_addr); ptr->total_weight += descr.weight; + LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-7"; } LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 8"; if (rec.total_weight && rec.total_weight != ptr->total_weight) { From e332c4c7308c38a2a1475dc7eeb3e668de990330 Mon Sep 17 00:00:00 2001 From: jianguo Date: Wed, 16 Oct 2024 10:18:02 +0800 Subject: [PATCH 057/101] update --- crypto/block/mc-config.cpp | 32 +++++++++++++++--------------- validator/impl/shard.cpp | 40 +++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index f44e29e9f..62df322a3 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -531,15 +531,15 @@ td::Result Config::unpack_workchain_list(Ref root) { } td::Result> Config::unpack_validator_set(Ref vset_root, std::uint64_t counter_) { - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 1"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 1"; if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 2"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 2"; gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 3"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 3"; gen::ValidatorSet::Record_validators rec0; if (!tlb::unpack_cell(std::move(vset_root), rec0)) { return td::Status::Error("validator set is invalid"); @@ -550,34 +550,34 @@ td::Result> Config::unpack_validator_set(Refprefetch_ref(); - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 5"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 5"; } else { return td::Status::Error("validator set cannot have zero total weight"); } vm::Dictionary dict{std::move(dict_root), 16}; td::BitArray<16> key_buffer; auto last = dict.get_minmax_key(key_buffer.bits(), 16, true); - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 6"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 6"; if (last.is_null() || (int)key_buffer.to_ulong() != rec.total - 1) { return td::Status::Error( "maximal index in a validator set dictionary must be one less than the total number of validators"); } - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7" << " ,total=" << rec.total; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7" << " ,total=" << rec.total; auto ptr = std::make_unique(rec.utime_since, rec.utime_until, rec.total, rec.main); for (int i = 0; i < rec.total; i++) { - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-1"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-1"; key_buffer.store_ulong(i); auto descr_cs = dict.lookup(key_buffer.bits(), 16); if (descr_cs.is_null()) { return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); } - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-2"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-2"; gen::ValidatorDescr::Record_validator_addr descr; if (!tlb::csr_unpack(descr_cs, descr)) { - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-3"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-3"; descr.adnl_addr.set_zero(); if (!(gen::t_ValidatorDescr.unpack_validator(descr_cs.write(), descr.public_key, descr.weight) && descr_cs->empty_ext())) { @@ -585,29 +585,29 @@ td::Result> Config::unpack_validator_set(Ref ~(ptr->total_weight)) { return td::Status::Error("total weight of all validators in validator set exceeds 2^64"); } - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-6"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-6"; ptr->list.emplace_back(sig_pubkey.pubkey, descr.weight, ptr->total_weight, descr.adnl_addr); ptr->total_weight += descr.weight; - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-7"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 7-7"; } - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 8"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 8"; if (rec.total_weight && rec.total_weight != ptr->total_weight) { return td::Status::Error("validator set declares incorrect total weight"); } - LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 9"; + // LOG(INFO) << "unpack_validator_set, counter" << counter_ << ", 9"; return std::move(ptr); } diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 21c46da46..4fc624107 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -374,22 +374,22 @@ MasterchainStateQ* MasterchainStateQ::make_copy() const { td::Result> MasterchainStateQ::fetch(const BlockIdExt& _id, td::BufferSlice _data, Ref _root) { - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 1"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 1"; if (!ShardIdFull(_id).is_masterchain_ext()) { - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 2"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 2"; return td::Status::Error(-666, "invalid masterchain block/state id passed for creating a new masterchain state object"); } - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 3"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 3"; Ref res{true, _id, std::move(_root), std::move(_data)}; - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 4"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 4"; td::Status err = res.unique_write().mc_init(_id.counter_); - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 5"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 5"; if (err.is_error()) { - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 6"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 6"; return err; } else { - LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 7"; + // LOG(INFO) << "Mfetch, counter" << _id.counter_ << ", 7"; return std::move(res); } } @@ -407,42 +407,42 @@ td::Status MasterchainStateQ::mc_init(std::uint64_t counter_) { } td::Status MasterchainStateQ::mc_reinit(std::uint64_t counter_) { - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1"; auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks, counter_); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-1"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-1"; cur_validators_.reset(); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-2"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-2"; next_validators_.reset(); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-3"; - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 2"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 1-3"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 2"; if (res.is_error()) { return res.move_as_error(); } config_ = res.move_as_ok(); CHECK(config_); CHECK(config_->set_block_id_ext(get_block_id())); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 3"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 3"; auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), 0)); cur_validators_ = std::move(validators); } - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4"; auto nv_root = config_->get_config_param(37, 36); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-1"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-1"; if (nv_root.not_null()) { - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-2"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-2"; TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), counter_)); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-3"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-3"; next_validators_ = std::move(validators); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-4"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 4-4"; } - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 5"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 5"; zerostate_id_ = config_->get_zerostate_id(); - LOG(INFO) << "mc_reinit, counter" << counter_ << ", 6"; + // LOG(INFO) << "mc_reinit, counter" << counter_ << ", 6"; return td::Status::OK(); } From 8093638fb571da17de8ee3e5b524881034ff00be Mon Sep 17 00:00:00 2001 From: jianguo Date: Wed, 16 Oct 2024 10:29:24 +0800 Subject: [PATCH 058/101] update --- crypto/block/mc-config.cpp | 92 +++++++++++++++++++------------------- validator/db/celldb.cpp | 18 ++++---- validator/impl/shard.cpp | 34 +++++++------- 3 files changed, 72 insertions(+), 72 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 62df322a3..e485d8107 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -102,15 +102,15 @@ td::Result> ConfigInfo::extract_config(std::shared_p } td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode, std::uint64_t counter_) { - LOG(INFO) << "extract_config, counter" << counter_ << ", 1"; + // LOG(INFO) << "extract_config, counter" << counter_ << ", 1"; if (mc_state_root.is_null()) { return td::Status::Error("configuration state root cell is null"); } - LOG(INFO) << "extract_config, counter" << counter_ << ", 2"; + // LOG(INFO) << "extract_config, counter" << counter_ << ", 2"; auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; - LOG(INFO) << "extract_config, counter" << counter_ << ", 3"; + // LOG(INFO) << "extract_config, counter" << counter_ << ", 3"; TRY_STATUS(config->unpack_wrapped(counter_)); - LOG(INFO) << "extract_config, counter" << counter_ << ", 4"; + // LOG(INFO) << "extract_config, counter" << counter_ << ", 4"; return std::move(config); } @@ -131,124 +131,124 @@ td::Status ConfigInfo::unpack_wrapped(std::uint64_t counter_) { } td::Status ConfigInfo::unpack(std::uint64_t counter_) { - LOG(INFO) << "unpack, counter" << counter_ << ", 1"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 1"; gen::ShardStateUnsplit::Record root_info; if (!tlb::unpack_cell(state_root, root_info) || !root_info.global_id) { return td::Status::Error("configuration state root cannot be deserialized"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 2"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 2"; global_id_ = root_info.global_id; block::ShardId shard_id{root_info.shard_id}; block_id.id = ton::BlockId{ton::ShardIdFull(shard_id), (unsigned)root_info.seq_no}; - LOG(INFO) << "unpack, counter" << counter_ << ", 3"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 3"; block_id.root_hash.set_zero(); block_id.file_hash.set_zero(); vert_seqno = root_info.vert_seq_no; utime = root_info.gen_utime; lt = root_info.gen_lt; min_ref_mc_seqno_ = root_info.min_ref_mc_seqno; - LOG(INFO) << "unpack, counter" << counter_ << ", 4"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 4"; if (!root_info.custom->size_refs()) { return td::Status::Error("state does not have a `custom` field with masterchain configuration"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 5"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 5"; if (mode & needLibraries) { lib_root_ = root_info.r1.libraries->prefetch_ref(); libraries_dict_ = std::make_unique(lib_root_, 256); } - LOG(INFO) << "unpack, counter" << counter_ << ", 6"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 6"; if (mode & needAccountsRoot) { accounts_root = vm::load_cell_slice_ref(root_info.accounts); LOG(DEBUG) << "requested accounts dictionary"; accounts_dict = std::make_unique(accounts_root, 256, block::tlb::aug_ShardAccounts); LOG(DEBUG) << "accounts dictionary created"; } - LOG(INFO) << "unpack, counter" << counter_ << ", 7"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 7"; state_extra_root_ = root_info.custom->prefetch_ref(); if (!is_masterchain()) { if (mode & (needShardHashes | needValidatorSet | needSpecialSmc | needPrevBlocks | needWorkchainInfo)) { return td::Status::Error("cannot extract masterchain-specific configuration data from a non-masterchain state"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 8"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 8"; cleanup(); - LOG(INFO) << "unpack, counter" << counter_ << ", 9"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 9"; return td::Status::OK(); } - LOG(INFO) << "unpack, counter" << counter_ << ", 10"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 10"; gen::McStateExtra::Record extra_info; if (!tlb::unpack_cell(state_extra_root_, extra_info)) { vm::load_cell_slice(state_extra_root_).print_rec(std::cerr); block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_); return td::Status::Error("state extra information is invalid"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 11"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 11"; gen::ValidatorInfo::Record validator_info; if (!tlb::csr_unpack(extra_info.r1.validator_info, validator_info)) { return td::Status::Error("validator_info in state extra information is invalid"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 12"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 12"; cc_seqno_ = validator_info.catchain_seqno; nx_cc_updated = validator_info.nx_cc_updated; if ((mode & needShardHashes) && !ShardConfig::unpack(extra_info.shard_hashes)) { return td::Status::Error("cannot unpack Shard configuration"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 13"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 13"; is_key_state_ = extra_info.r1.after_key_block; if (extra_info.r1.last_key_block->size() > 1) { - LOG(INFO) << "unpack, counter" << counter_ << ", 14"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 14"; auto& cs = extra_info.r1.last_key_block.write(); block::gen::ExtBlkRef::Record ext_ref; if (!(cs.advance(1) && tlb::unpack_exact(cs, ext_ref))) { return td::Status::Error("cannot unpack last_key_block from masterchain state"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 15"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 15"; last_key_block_.id = ton::BlockId{ton::masterchainId, ton::shardIdAll, ext_ref.seq_no}; last_key_block_.root_hash = ext_ref.root_hash; last_key_block_.file_hash = ext_ref.file_hash; last_key_block_lt_ = ext_ref.end_lt; - LOG(INFO) << "unpack, counter" << counter_ << ", 16"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16"; } else { last_key_block_.invalidate(); last_key_block_.id.seqno = 0; last_key_block_lt_ = 0; - LOG(INFO) << "unpack, counter" << counter_ << ", 17"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 17"; } // unpack configuration - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1"; TRY_STATUS(Config::unpack_wrapped(std::move(extra_info.config), counter_)); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-2"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-2"; // unpack previous masterchain block collection std::unique_ptr prev_blocks_dict = std::make_unique(extra_info.r1.prev_blocks, 32, block::tlb::aug_OldMcBlocksInfo); - LOG(INFO) << "unpack, counter" << counter_ << ", 18"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 18"; if (block_id.id.seqno) { block::gen::ExtBlkRef::Record extref = {}; auto ref = prev_blocks_dict->lookup(td::BitArray<32>::zero()); if (!(ref.not_null() && ref.write().advance(1) && tlb::csr_unpack(ref, extref) && !extref.seq_no)) { return td::Status::Error("OldMcBlocks in masterchain state does not contain a valid zero state reference"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 19"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 19"; zerostate_id_.root_hash = extref.root_hash; zerostate_id_.file_hash = extref.file_hash; } else { zerostate_id_.root_hash.set_zero(); zerostate_id_.file_hash.set_zero(); } - LOG(INFO) << "unpack, counter" << counter_ << ", 20"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 20"; zerostate_id_.workchain = ton::masterchainId; if (mode & needPrevBlocks) { prev_blocks_dict_ = std::move(prev_blocks_dict); } - LOG(INFO) << "unpack, counter" << counter_ << ", 21"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 21"; // ... cleanup(); - LOG(INFO) << "unpack, counter" << counter_ << ", 22"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 22"; return td::Status::OK(); } td::Status Config::unpack_wrapped(Ref config_csr, std::uint64_t counter_) { try { - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-1"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-1"; return unpack(std::move(config_csr), counter_); } catch (vm::VmError err) { return td::Status::Error(PSLICE() << "error unpacking masterchain configuration: " << err.get_msg()); @@ -264,38 +264,38 @@ td::Status Config::unpack_wrapped() { } td::Status Config::unpack(Ref config_cs, std::uint64_t counter_) { - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-2"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-2"; gen::ConfigParams::Record config_params; if (!tlb::csr_unpack(std::move(config_cs), config_params)) { return td::Status::Error("cannot unpack ConfigParams"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-3"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-3"; config_addr = config_params.config_addr; config_root = std::move(config_params.config); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-4"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-4"; return unpack(counter_); } td::Status Config::unpack(std::uint64_t counter_) { - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5"; if (config_root.is_null()) { return td::Status::Error("configuration root not set"); } - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-1"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-1"; config_dict = std::make_unique(config_root, 32); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-2"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-2"; if (mode & needValidatorSet) { - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3"; auto c = get_config_param(35, 34); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3-1"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-3-1"; auto vset_res = unpack_validator_set(c, counter_); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-4"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-4"; if (vset_res.is_error()) { return vset_res.move_as_error(); } - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-5"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-5-5"; cur_validators_ = vset_res.move_as_ok(); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-6"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-6"; } if (mode & needSpecialSmc) { LOG(DEBUG) << "needSpecialSmc flag set"; @@ -306,21 +306,21 @@ td::Status Config::unpack(std::uint64_t counter_) { special_smc_dict = std::make_unique(vm::load_cell_slice_ref(std::move(param)), 256); LOG(DEBUG) << "smc dictionary created"; } - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-7"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-7"; } if (mode & needWorkchainInfo) { TRY_RESULT(pair, unpack_workchain_list_ext(get_config_param(12))); workchains_ = std::move(pair.first); workchains_dict_ = std::move(pair.second); - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-8"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-8"; } if (mode & needCapabilities) { - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-10"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-10"; auto cell = get_config_param(8); if (cell.is_null()) { version_ = 0; capabilities_ = 0; - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-11"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-11"; } else { block::gen::GlobalVersion::Record gv; if (!tlb::unpack_cell(std::move(cell), gv)) { @@ -330,11 +330,11 @@ td::Status Config::unpack(std::uint64_t counter_) { } version_ = gv.version; capabilities_ = gv.capabilities; - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-12"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-12"; } } // ... - LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-13"; + // LOG(INFO) << "unpack, counter" << counter_ << ", 16-1-13"; return td::Status::OK(); } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 3cbd924e5..43036ceed 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -439,27 +439,27 @@ void CellDbIn::migrate_cells() { } void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { - LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; if (!started_) { - LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { - LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; auto P = td::PromiseCreator::lambda( [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { - LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; if (R.is_error()) { td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); - LOG(INFO) << " load_cell: counter" << counter_ << ", 6"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 6"; } else { - LOG(INFO) << " load_cell: counter" << counter_ << ", 7-0"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 7-0"; promise.set_result(R.move_as_ok()); - LOG(INFO) << " load_cell: counter" << counter_ << ", 7"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 7"; } }); - LOG(INFO) << " load_cell: counter" << counter_ << ", 4"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 4"; boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); - LOG(INFO) << " load_cell: counter" << counter_ << ", 4-1"; + // LOG(INFO) << " load_cell: counter" << counter_ << ", 4-1"; } } diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 4fc624107..e15b8cc0b 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -88,9 +88,9 @@ td::Result> ShardStateQ::fetch(const BlockIdExt& _id, td::Buffe } td::Status ShardStateQ::init(std::uint64_t counter_) { - LOG(INFO) << "init, counter" << counter_ << ", 1"; + // LOG(INFO) << "init, counter" << counter_ << ", 1"; if (root.is_null()) { - LOG(INFO) << "init, counter" << counter_ << ", 2"; + // LOG(INFO) << "init, counter" << counter_ << ", 2"; if (data.empty()) { return td::Status::Error( -668, "cannot initialize shardchain state without either a root cell or a BufferSlice with serialized data"); @@ -98,25 +98,25 @@ td::Status ShardStateQ::init(std::uint64_t counter_) { #if LAZY_STATE_DESERIALIZE vm::StaticBagOfCellsDbLazy::Options options; options.check_crc32c = true; - LOG(INFO) << "init, counter" << counter_ << ", 3"; + // LOG(INFO) << "init, counter" << counter_ << ", 3"; auto res = vm::StaticBagOfCellsDbLazy::create(td::BufferSliceBlobView::create(data.clone()), options); if (res.is_error()) { return res.move_as_error(); } - LOG(INFO) << "init, counter" << counter_ << ", 4"; + // LOG(INFO) << "init, counter" << counter_ << ", 4"; auto boc = res.move_as_ok(); auto rc = boc->get_root_count(); if (rc.is_error()) { return rc.move_as_error(); } - LOG(INFO) << "init, counter" << counter_ << ", 5"; + // LOG(INFO) << "init, counter" << counter_ << ", 5"; if (rc.move_as_ok() != 1) { return td::Status::Error(-668, "shardchain state BoC is invalid"); } auto res3 = boc->get_root_cell(0); bocs_.clear(); bocs_.push_back(std::move(boc)); - LOG(INFO) << "init, counter" << counter_ << ", 6"; + // LOG(INFO) << "init, counter" << counter_ << ", 6"; #else auto res3 = vm::std_boc_deserialize(data.as_slice()); LOG(INFO) << "init, counter" << counter_ << ", 7"; @@ -124,36 +124,36 @@ td::Status ShardStateQ::init(std::uint64_t counter_) { if (res3.is_error()) { return res3.move_as_error(); } - LOG(INFO) << "init, counter" << counter_ << ", 8"; + // LOG(INFO) << "init, counter" << counter_ << ", 8"; root = res3.move_as_ok(); if (root.is_null()) { return td::Status::Error(-668, "cannot extract root cell out of a shardchain state BoC"); } - LOG(INFO) << "init, counter" << counter_ << ", 9"; + // LOG(INFO) << "init, counter" << counter_ << ", 9"; } - LOG(INFO) << "init, counter" << counter_ << ", 10"; + // LOG(INFO) << "init, counter" << counter_ << ", 10"; rhash = root->get_hash().bits(); block::gen::ShardStateUnsplit::Record info; if (!tlb::unpack_cell(root, info)) { return td::Status::Error(-668, "shardchain state for block "s + blkid.id.to_str() + " does not contain a valid header"); } - LOG(INFO) << "init, counter" << counter_ << ", 11"; + // LOG(INFO) << "init, counter" << counter_ << ", 11"; lt = info.gen_lt; utime = info.gen_utime; global_id_ = info.global_id; before_split_ = info.before_split; block::ShardId id{info.shard_id}; ton::BlockId hdr_id{ton::ShardIdFull(id), info.seq_no}; - LOG(INFO) << "init, counter" << counter_ << ", 12"; + // LOG(INFO) << "init, counter" << counter_ << ", 12"; if (!id.is_valid() || get_shard() != ton::ShardIdFull(id) || get_seqno() != info.seq_no) { return td::Status::Error(-668, "header of unpacked shardchain state for block "s + blkid.id.to_str() + " contains BlockId " + hdr_id.to_str() + " different from the one originally required"); } - LOG(INFO) << "init, counter" << counter_ << ", 13"; + // LOG(INFO) << "init, counter" << counter_ << ", 13"; if (info.r1.master_ref.write().fetch_long(1)) { - LOG(INFO) << "init, counter" << counter_ << ", 14"; + // LOG(INFO) << "init, counter" << counter_ << ", 14"; BlockIdExt mc_id; if (!block::tlb::t_ExtBlkRef.unpack(info.r1.master_ref, mc_id, nullptr)) { return td::Status::Error(-668, "cannot unpack master_ref in shardchain state of "s + blkid.to_str()); @@ -162,7 +162,7 @@ td::Status ShardStateQ::init(std::uint64_t counter_) { } else { master_ref = {}; } - LOG(INFO) << "init, counter" << counter_ << ", 15"; + // LOG(INFO) << "init, counter" << counter_ << ", 15"; return td::Status::OK(); } @@ -395,14 +395,14 @@ td::Result> MasterchainStateQ::fetch(const BlockIdExt& _i } td::Status MasterchainStateQ::mc_init(std::uint64_t counter_) { - LOG(INFO) << "mc_init, counter" << counter_ << ", 1"; + // LOG(INFO) << "mc_init, counter" << counter_ << ", 1"; auto err = init(counter_); if (err.is_error()) { return err; } - LOG(INFO) << "mc_init, counter" << counter_ << ", 2"; + // LOG(INFO) << "mc_init, counter" << counter_ << ", 2"; auto a = mc_reinit(counter_); - LOG(INFO) << "mc_init, counter" << counter_ << ", 3"; + // LOG(INFO) << "mc_init, counter" << counter_ << ", 3"; return a; } From 45a9f629cd22f156e58a4316fdb203c600a408f9 Mon Sep 17 00:00:00 2001 From: oker Date: Wed, 16 Oct 2024 10:33:50 +0800 Subject: [PATCH 059/101] check actor and thread idle --- tdactor/td/actor/common.h | 6 +++++- tdactor/td/actor/core/ActorExecutor.cpp | 8 -------- tdactor/td/actor/core/CpuWorker.cpp | 5 ++++- tdactor/td/actor/core/Scheduler.cpp | 20 +++++++++++++++----- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index 2908bf236..17444239c 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -135,7 +135,9 @@ class Scheduler { while (scheduler->run(10)) { } }); - thread.set_name(PSLICE() << "#" << it << ":io"); + auto name = PSLICE() << "#" << it << ":io"; + LOG(INFO) << "yus scheduler start " << name << "scheduler size " << schedulers_.size(); + thread.set_name(name); thread.detach(); } } @@ -188,11 +190,13 @@ class Scheduler { bool skip_timeouts_{false}; void init() { + LOG(INFO) << " yus " << "create actor scheduler size " << infos_.size(); CHECK(infos_.size() < 256); CHECK(!group_info_); group_info_ = std::make_shared(infos_.size()); td::uint8 id = 0; for (const auto &info : infos_) { + LOG(INFO) << " yus NodeInfo " << id << "cpu thread " << info.cpu_threads_; schedulers_.emplace_back( td::make_unique(group_info_, core::SchedulerId{id}, info.cpu_threads_, skip_timeouts_)); id++; diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index e6f4e5275..285e63674 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -127,14 +127,10 @@ void ActorExecutor::start() noexcept { } void ActorExecutor::finish() noexcept { - LOG(ERROR) << "yus actor executror FINISH " << actor_info_.get_name() << " " - << tag("own_lock", actor_locker_.own_lock()); if (!actor_locker_.own_lock()) { if (!pending_signals_.empty() && actor_locker_.add_signals(pending_signals_)) { flags_ = actor_locker_.flags(); - LOG(ERROR) << "yus Own after finish " << actor_info_.get_name() << " " << format::as_binary(flags().raw()); } else { - LOG(ERROR) << "yus DO FINISH " << actor_info_.get_name() << " " << flags(); return; } } else { @@ -156,10 +152,8 @@ void ActorExecutor::finish() noexcept { signals.clear_signal(ActorSignals::Pop); flags().set_signals(signals); flags().set_in_queue(false); - LOG(ERROR) << "yus clear in_queue " << format::as_binary(flags().raw()); } - LOG(ERROR) << tag(" yus in_queue", flags().is_in_queue()) << tag("has_signals", flags().has_signals()); if (flags_.is_closed()) { // Writing to mailbox and closing actor may happen concurrently // We must ensure that all messages in mailbox will be deleted @@ -178,14 +172,12 @@ void ActorExecutor::finish() noexcept { } if (actor_locker_.try_unlock(flags())) { if (add_to_queue) { - LOG(INFO) << "yus add to queue " << flags().get_scheduler_id().value(); dispatcher_.add_to_queue(std::move(actor_info_ptr), flags().get_scheduler_id(), !flags().is_shared()); } break; } flags_ = actor_locker_.flags(); } - LOG(ERROR) << "yus DO FINISH " << actor_info_.get_name() << " " << flags(); } bool ActorExecutor::flush_one_signal(ActorSignals &signals) { diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index 6eef2ffef..eeef47f18 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -29,6 +29,7 @@ namespace core { void CpuWorker::run() { auto thread_id = get_thread_id(); auto &dispatcher = *SchedulerContext::get(); + LOG(INFO) << "yus start work " << thread_id; MpmcWaiter::Slot slot; waiter_.init_slot(slot, thread_id); @@ -39,12 +40,14 @@ void CpuWorker::run() { if (try_pop(message, thread_id)) { waiter_.stop_wait(slot); if (!message) { - LOG(INFO) << "yus no message return" << " thread_id" << thread_id; + LOG(INFO) << "yus no message return" << " thread_id " << thread_id; return; } auto lock = debug.start(message->get_name()); + LOG(INFO) << "yus actor " << message->get_name() << " mailbox number " << message->mailbox().reader().calc_size(); ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); } else { + LOG(INFO) << "yus no message, IDLE"; waiter_.wait(slot); } } diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 64cfa6838..b29c00194 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -78,14 +78,16 @@ Scheduler::~Scheduler() { } void Scheduler::start() { - LOG(INFO) << "yus start scheduler"; + LOG(INFO) << "yus start scheduler, cpu size " << cpu_threads_.size(); for (size_t i = 0; i < cpu_threads_.size(); i++) { cpu_threads_[i] = td::thread([this, i] { this->run_in_context_impl(*this->info_->cpu_workers[i], [this, i] { CpuWorker(*info_->cpu_queue, *info_->cpu_queue_waiter, i, info_->cpu_local_queue).run(); }); }); - cpu_threads_[i].set_name(PSLICE() << "#" << info_->id.value() << ":cpu#" << i); + auto name = PSLICE() << "#" << info_->id.value() << ":cpu#" << i; + LOG(INFO) << "yus create " << "cpu_workers " << name; + cpu_threads_[i].set_name(name); } #if TD_PORT_WINDOWS // FIXME: use scheduler_id @@ -159,7 +161,8 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler if (!scheduler_id.is_valid()) { scheduler_id = get_scheduler_id(); } - LOG(ERROR) << "Add to queue: " << actor_info_ptr->get_name() << " " << scheduler_id.value(); + auto name = actor_info_ptr->get_name(); + auto &info = scheduler_group()->schedulers.at(scheduler_id.value()); if (need_poll || !info.cpu_queue) { info.io_queue->writer_put(std::move(actor_info_ptr)); @@ -168,13 +171,20 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler // may push local CHECK(actor_info_ptr); auto raw = actor_info_ptr.release(); - auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push( - raw, [&](auto value) { info.cpu_queue->push(value, get_thread_id()); }); + auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push(raw, [&](auto value) { + info.cpu_queue->push(value, get_thread_id()); + LOG(ERROR) << "yus add to queue overflow: " << name << " sche id " << scheduler_id.value() << " worker id " + << cpu_worker_id_.value() << " thread id " << get_thread_id(); + }); if (should_notify) { info.cpu_queue_waiter->notify(); } + LOG(ERROR) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " + << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " + << get_thread_id(); return; } + LOG(ERROR) << "yus add to global queue: " << scheduler_id.value() << "thread id " << get_thread_id(); info.cpu_queue->push(actor_info_ptr.release(), get_thread_id()); info.cpu_queue_waiter->notify(); } From 8938df1eaf452e21c62210e22d31882aaa9373eb Mon Sep 17 00:00:00 2001 From: jianguo Date: Wed, 16 Oct 2024 10:49:59 +0800 Subject: [PATCH 060/101] update --- validator-engine/validator-engine.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 854202158..e24e4ea08 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4206,7 +4206,6 @@ int main(int argc, char *argv[]) { v = 127; } threads = v; - threads = 127; return td::Status::OK(); }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); From e30a04eafc4202e519a7932910185ff38c7482fe Mon Sep 17 00:00:00 2001 From: oker Date: Wed, 16 Oct 2024 11:53:41 +0800 Subject: [PATCH 061/101] change thread num to 4 --- validator-engine/validator-engine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 80ab387db..4af2a9556 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4206,7 +4206,7 @@ int main(int argc, char *argv[]) { v = 127; } threads = v; - threads = 127; + threads = 4; return td::Status::OK(); }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); From bf5b9d48e6d8f48f1ff0901b35ae2f985f139a72 Mon Sep 17 00:00:00 2001 From: jianguo Date: Wed, 16 Oct 2024 11:59:50 +0800 Subject: [PATCH 062/101] recover thread --- validator-engine/validator-engine.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 4af2a9556..219154f12 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4206,7 +4206,6 @@ int main(int argc, char *argv[]) { v = 127; } threads = v; - threads = 4; return td::Status::OK(); }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); From 923c31d17634152c39286ad770eb14fa59c11420 Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 11:11:32 +0800 Subject: [PATCH 063/101] uypdate --- validator-engine/validator-engine.cpp | 1 + validator/impl/liteserver.cpp | 69 +++++++++++++++++++++++++-- validator/manager.cpp | 3 +- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index e24e4ea08..86ffbc1e5 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3911,6 +3911,7 @@ void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNode if (!started_) { return; } + LOG(INFO) << "process_control_query, 1"; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::run_ext_query, std::move(data), std::move(promise)); return; diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 4f2518776..9bfe8617f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -45,6 +45,9 @@ #include "td/actor/MultiPromise.h" #include "collator-impl.h" #include "scope_guard.h" +#include +#include +#include #define LOG_TIME_EST(hint) \ SCOPE_EXIT([start = std::chrono::steady_clock::now(), hint]() { \ @@ -66,6 +69,7 @@ int global_count = 0; void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise) { + LOG(INFO) << "fatal_error 104"; td::actor::create_actor("litequery", std::move(data), std::move(manager), std::move(cache), std::move(promise)) .release(); @@ -75,12 +79,14 @@ void LiteQuery::fetch_account_state( WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, td::Promise, UnixTime, LogicalTime, std::unique_ptr>> promise) { + LOG(INFO) << "fatal_error 103"; td::actor::create_actor("litequery", wc, acc_addr, std::move(manager), std::move(promise)).release(); } LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise) : query_(std::move(data)), manager_(std::move(manager)), cache_(std::move(cache)), promise_(std::move(promise)) { + LOG(INFO) << "fatal_error 101," << counter_; timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } @@ -89,6 +95,7 @@ LiteQuery::LiteQuery( td::Promise, UnixTime, LogicalTime, std::unique_ptr>> promise) : manager_(std::move(manager)), acc_state_promise_(std::move(promise)), acc_workchain_(wc), acc_addr_(acc_addr) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); + LOG(INFO) << "fatal_error 102," << counter_; } void LiteQuery::abort_query(td::Status reason) { @@ -110,6 +117,7 @@ void LiteQuery::abort_query_ext(td::Status reason, std::string comment) { } bool LiteQuery::fatal_error(td::Status error) { + LOG(INFO) << "fatal_error 1," << counter_; abort_query(std::move(error)); return false; } @@ -142,8 +150,9 @@ bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) { void LiteQuery::start_up() { alarm_timestamp() = timeout_; - + LOG(INFO) << "fatal_error 112," << counter_; if (acc_state_promise_) { + LOG(INFO) << "fatal_error 110," << counter_; td::actor::send_closure_later(actor_id(this), &LiteQuery::perform_fetchAccountState); return; } @@ -151,7 +160,9 @@ void LiteQuery::start_up() { auto F = fetch_tl_object(query_, true); if (F.is_error()) { td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, 0); // unknown + LOG(INFO) << "fatal_error 2," << counter_; abort_query(F.move_as_error()); + // std::this_thread::sleep_for(std::chrono::seconds(10000)); return; } query_obj_ = F.move_as_ok(); @@ -164,6 +175,7 @@ void LiteQuery::start_up() { if (R.is_ok()) { td::actor::send_closure(SelfId, &LiteQuery::perform); } else { + LOG(INFO) << "fatal_error 3,"; td::actor::send_closure(SelfId, &LiteQuery::abort_query, R.move_as_error_prefix("cannot send external message : ")); } @@ -207,8 +219,11 @@ void LiteQuery::perform() { td::overloaded( [&](lite_api::liteServer_getTime& q) { this->perform_getTime(); }, [&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); }, - [&](lite_api::liteServer_getMasterchainInfo& q) { this->perform_getMasterchainInfo(-1); }, + [&](lite_api::liteServer_getMasterchainInfo& q) { + LOG(INFO) << "fatal_error 120"; + this->perform_getMasterchainInfo(-1); }, [&](lite_api::liteServer_getMasterchainInfoExt& q) { + LOG(INFO) << "fatal_error 121"; this->perform_getMasterchainInfo(q.mode_ & 0x7fffffff); }, [&](lite_api::liteServer_getBlock& q) { this->perform_getBlock(ton::create_block_id(q.id_)); }, @@ -296,7 +311,10 @@ void LiteQuery::perform() { [&](lite_api::liteServer_getBlockOutMsgQueueSize& q) { this->perform_getBlockOutMsgQueueSize(q.mode_, create_block_id(q.id_)); }, - [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); + [&](auto& obj) { + LOG(INFO) << "fatal_error 4," << counter_; + this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); + })); } void LiteQuery::perform_getTime() { @@ -323,6 +341,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 5"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); @@ -367,6 +386,7 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { td::actor::send_closure(manager_, &ValidatorManager::get_block_data_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 6,"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, @@ -393,6 +413,7 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { td::actor::send_closure(manager_, &ValidatorManager::get_block_data_for_litequery, blkid, [Self = actor_id(this), blkid, mode](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 7,"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, mode, @@ -508,6 +529,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { td::actor::send_closure(manager_, &ValidatorManager::get_block_state_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 8," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, @@ -518,6 +540,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, [Self = actor_id(this), blkid](td::Result res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 9," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, @@ -533,6 +556,7 @@ void LiteQuery::continue_getState(BlockIdExt blkid, Refserialize(); if (res.is_error()) { + LOG(INFO) << "fatal_error 10," << counter_; abort_query(res.move_as_error()); return; } @@ -562,6 +586,7 @@ void LiteQuery::perform_sendMessage(td::BufferSlice data) { if (res.is_error()) { // Don't cache errors td::actor::send_closure(cache, &LiteServerCache::drop_send_message_from_cache, cache_key); + LOG(INFO) << "fatal_error 11"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot apply external message to current state : "s)); } else { @@ -596,6 +621,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { manager_, &ValidatorManager::get_block_data_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 12," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); } else { @@ -631,6 +657,7 @@ bool LiteQuery::request_mc_proof(BlockIdExt blkid, int mode) { manager, &ValidatorManager::get_block_proof_from_db_short, blkid, [Self, blkid, mode](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 13," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof for "s + blkid.to_str() + " : ")); } else { @@ -663,6 +690,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { manager_, &ValidatorManager::get_block_state_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 14," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); } else { @@ -707,6 +735,7 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { td::actor::send_closure(manager_, &ValidatorManager::get_block_state_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 15," << blkid.counter_; td::actor::send_closure( Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); @@ -731,6 +760,7 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { td::actor::send_closure(manager_, &ValidatorManager::get_block_data_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 16," << blkid.counter_; td::actor::send_closure( Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); @@ -764,6 +794,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { manager, &ValidatorManager::get_block_proof_link_from_db_short, blkid, [Self, blkid](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 17," << blkid.counter_; td::actor::send_closure( Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); @@ -775,6 +806,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { } else { get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 18," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } @@ -782,6 +814,7 @@ bool LiteQuery::request_proof_link(BlockIdExt blkid) { manager, &ValidatorManager::get_block_proof_link_from_db, R.move_as_ok(), [=](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 19," << blkid.counter_; td::actor::send_closure( Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load proof link for "s + blkid.to_str() + " : ")); @@ -808,12 +841,14 @@ bool LiteQuery::request_zero_state(BlockIdExt blkid) { ++pending_; get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 20," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } td::actor::send_closure_later( manager, &ValidatorManager::get_zero_state, blkid, [=](td::Result res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 21," << blkid.counter_; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error_prefix("cannot load zerostate of "s + blkid.to_str() + " : ")); } else { @@ -879,6 +914,7 @@ void LiteQuery::perform_getAccountState(BlockIdExt blkid, WorkchainId workchain, manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, [Self = actor_id(this)](td::Result, BlockIdExt>> res) -> void { if (res.is_error()) { + LOG(INFO) << "fatal_error 22"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); @@ -905,6 +941,7 @@ void LiteQuery::continue_getAccountState_0(Ref } void LiteQuery::perform_fetchAccountState() { + LOG(INFO) << "fatal_error 111," << counter_; perform_getMasterchainInfo(-1); } @@ -962,6 +999,7 @@ void LiteQuery::perform_getLibraries(std::vector library_list) { [Self = actor_id(this), library_list](td::Result, BlockIdExt>> res) -> void { if (res.is_error()) { + LOG(INFO) << "fatal_error 23"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); @@ -1990,6 +2028,7 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< load_prevKeyBlock(blkid, [this, blkid, mode, param_list = std::move(param_list)]( td::Result>> res) mutable { if (res.is_error()) { + LOG(INFO) << "fatal_error 24"; this->abort_query(res.move_as_error()); } else { this->base_blk_id_ = res.ok().first; @@ -2187,15 +2226,18 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, auto P = td::PromiseCreator::lambda([Self = actor_id(this), mc_blkid, manager = manager_, mode, pfx](td::Result res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 25"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); return; } auto handle = res.move_as_ok(); if (!handle->inited_masterchain_ref_block()) { + LOG(INFO) << "fatal_error 26"; td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); return; } if (handle->masterchain_ref_block() > mc_blkid.seqno()) { + LOG(INFO) << "fatal_error 27"; td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("specified mc block is older than block's masterchain ref")); return; @@ -2205,6 +2247,7 @@ void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, manager, &ValidatorManager::get_block_data_from_db, handle, [Self, mc_ref_blkid = handle->masterchain_ref_block(), mc_blkid, pfx, mode](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 28"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, @@ -2264,12 +2307,14 @@ void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref R) mutable { if (R.is_error()) { + LOG(INFO) << "fatal_error 29"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), [Self, masterchain_ref_seqno](td::Result> res) mutable { if (res.is_error()) { + LOG(INFO) << "fatal_error 30"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); return; } @@ -2312,12 +2357,14 @@ void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref pr manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno, [manager = manager_, Self = actor_id(this)](td::Result R) mutable { if (R.is_error()) { + LOG(INFO) << "fatal_error 31"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), [Self](td::Result> res) mutable { if (res.is_error()) { + LOG(INFO) << "fatal_error 32"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); return; } @@ -2405,6 +2452,7 @@ void LiteQuery::continue_lookupBlockWithProof_buildProofLinks( manager_, &ValidatorManager::get_block_data_from_db_short, prev_id, [Self = actor_id(this), result = std::move(result)](td::Result> R) mutable { if (R.is_error()) { + LOG(INFO) << "fatal_error 33"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, R.move_as_ok(), @@ -2489,6 +2537,7 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni auto P = td::PromiseCreator::lambda( [Self = actor_id(this), manager = manager_, mode = (mode >> 4)](td::Result res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 34"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto handle = res.move_as_ok(); @@ -2496,6 +2545,7 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), mode](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 35"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, @@ -2850,6 +2900,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, td::actor::send_closure_later(manager_, &ValidatorManager::get_shard_state_from_db_short, bblk, [Self = actor_id(this), from, to, bblk, mode](td::Result> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 36"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockProof, from, @@ -2862,6 +2913,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, [Self = actor_id(this), from, to, mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 37"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); @@ -2875,6 +2927,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, [Self = actor_id(this), from, mode](td::Result, BlockIdExt>> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 38"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto pair = res.move_as_ok(); @@ -2886,6 +2939,7 @@ void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_shard_client_state, false, [Self = actor_id(this), from, mode](td::Result res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 39"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::perform_getBlockProof, from, @@ -3355,11 +3409,13 @@ void LiteQuery::perform_getShardBlockProof(BlockIdExt blkid) { blk_id_ = blkid; get_block_handle_checked(blkid, [manager = manager_, Self = actor_id(this)](td::Result R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 40"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } ConstBlockHandle handle = R.move_as_ok(); if (!handle->inited_masterchain_ref_block()) { + LOG(INFO) << "fatal_error 41"; td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); return; } @@ -3368,12 +3424,14 @@ void LiteQuery::perform_getShardBlockProof(BlockIdExt blkid) { manager, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, handle->masterchain_ref_block(), [Self, manager](td::Result R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 42"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { ConstBlockHandle handle = R.move_as_ok(); td::actor::send_closure_later( manager, &ValidatorManager::get_block_data_from_db, handle, [Self](td::Result> R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 43"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getShardBlockProof, R.move_as_ok(), @@ -3461,6 +3519,7 @@ void LiteQuery::continue_getShardBlockProof(Ref cur_block, manager_, &ValidatorManager::get_block_data_from_db_short, prev_id, [Self = actor_id(this), result = std::move(result)](td::Result> R) mutable { if (R.is_error()) { + LOG(INFO) << "fatal_error 44"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getShardBlockProof, R.move_as_ok(), @@ -3475,6 +3534,7 @@ void LiteQuery::perform_getOutMsgQueueSizes(td::optional shard) { manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, [Self = actor_id(this), shard](td::Result, BlockIdExt>> res) { if (res.is_error()) { + LOG(INFO) << "fatal_error 45"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::continue_getOutMsgQueueSizes, shard, res.ok().first); @@ -3506,6 +3566,7 @@ void LiteQuery::continue_getOutMsgQueueSizes(td::optional shard, Re } ig.add_promise([Self = actor_id(this), res](td::Result R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 46"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); return; } @@ -3588,6 +3649,7 @@ void LiteQuery::perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blk manager_, &ValidatorManager::get_block_candidate_for_litequery, PublicKey{pubkeys::Ed25519{source}}, blkid, collated_data_hash, [Self = actor_id(this)](td::Result R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 47"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { BlockCandidate cand = R.move_as_ok(); @@ -3614,6 +3676,7 @@ void LiteQuery::perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard) manager_, &ValidatorManager::get_validator_groups_info_for_litequery, maybe_shard, [Self = actor_id(this)](td::Result> R) { if (R.is_error()) { + LOG(INFO) << "fatal_error 48"; td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); } else { td::actor::send_closure_later(Self, &LiteQuery::finish_query, serialize_tl_object(R.move_as_ok(), true), diff --git a/validator/manager.cpp b/validator/manager.cpp index 3b211d7e4..a87139a66 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -581,6 +581,7 @@ void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { } void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) override { + LOG(INFO) << "add_ext_server_id, 1"; td::actor::send_closure(id_, &ValidatorManagerImpl::run_ext_query, std::move(data), std::move(promise)); } @@ -646,7 +647,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise(data, true); if (E.is_error()) { run_liteserver_query(std::move(data), actor_id(this), lite_server_cache_.get(), std::move(P)); From bfdb2fd78fdd6195d5ee86fe9f83d93fe9fee4cd Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 11:21:02 +0800 Subject: [PATCH 064/101] update --- validator-engine/validator-engine.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 86ffbc1e5..a1347d816 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1954,6 +1954,7 @@ void ValidatorEngine::start_full_node() { } void ValidatorEngine::started_full_node() { + LOG(INFO) << "started_full_node, 1"; start_lite_server(); } @@ -1966,6 +1967,7 @@ void ValidatorEngine::add_lite_server(ton::PublicKeyHash id, td::uint16 port) { } void ValidatorEngine::start_lite_server() { + LOG(INFO) << "start_lite_server, 1"; for (auto &s : config_.liteservers) { add_lite_server(s.second, static_cast(s.first)); } @@ -2192,7 +2194,7 @@ void ValidatorEngine::try_add_liteserver(ton::PublicKeyHash id, td::int32 port, promise.set_value(td::Unit()); return; } - + LOG(INFO) << "try_add_liteserver, 1"; add_lite_server(id, static_cast(port)); write_config(std::move(promise)); @@ -2888,6 +2890,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_changeFul void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addLiteserver &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + LOG(INFO) << "run_control_query, 1"; if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; From d232974e394746d8dc67537ea55dcae862e17d68 Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 11:33:45 +0800 Subject: [PATCH 065/101] update --- adnl/adnl-local-id.cpp | 1 + overlay/overlay.cpp | 1 + overlay/overlay.hpp | 1 + 3 files changed, 3 insertions(+) diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index d72fc7bcc..132467793 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -87,6 +87,7 @@ void AdnlLocalId::deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::P for (auto &cb : cb_) { auto f = cb.first; if (f.length() <= s.length() && s.as_slice().substr(0, f.length()) == f) { + LOG(INFO) << "process_query, 2"; cb.second->receive_query(src, short_id_, std::move(s), std::move(promise)); return; } diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 43b3b7e5a..3a47ef656 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -154,6 +154,7 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, if (R.is_error()) { // allow custom query to be here + LOG(INFO) << "process_query, 3"; callback_->receive_query(src, overlay_id_, std::move(data), std::move(promise)); return; } diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 8fb3d91d7..1bc4242d9 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -273,6 +273,7 @@ class OverlayImpl : public Overlay { private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { + LOG(INFO) << "process_query, 1"; callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } From 86e27562beb45fbae7c46b1ff90782e8d82ff7b0 Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 11:42:29 +0800 Subject: [PATCH 066/101] update --- adnl/adnl-local-id.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index 132467793..2b8bc0860 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -87,9 +87,9 @@ void AdnlLocalId::deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::P for (auto &cb : cb_) { auto f = cb.first; if (f.length() <= s.length() && s.as_slice().substr(0, f.length()) == f) { - LOG(INFO) << "process_query, 2"; - cb.second->receive_query(src, short_id_, std::move(s), std::move(promise)); - return; + LOG(INFO) << "process_query, 2 over"; + //cb.second->receive_query(src, short_id_, std::move(s), std::move(promise)); + //return; } } VLOG(ADNL_INFO) << this << ": dropping IN message from " << src From 029f4ae96edb721fcce35307b307c00ca85e1e2e Mon Sep 17 00:00:00 2001 From: oker Date: Thu, 17 Oct 2024 10:47:03 +0800 Subject: [PATCH 067/101] more log but exception --- tdactor/td/actor/core/ActorExecutor.cpp | 7 ++++++- tdactor/td/actor/core/CpuWorker.cpp | 10 +++++++++- tdactor/td/actor/core/Scheduler.cpp | 6 ++++-- validator/db/celldb.cpp | 3 ++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 285e63674..6ce05ea47 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -19,6 +19,7 @@ #include "td/actor/core/ActorExecutor.h" #include "td/utils/ScopeGuard.h" +#include "td/utils/logging.h" namespace td { namespace actor { @@ -116,11 +117,13 @@ void ActorExecutor::start() noexcept { while (flush_one_signal(signals)) { if (actor_execute_context_.has_immediate_flags()) { + LOG(INFO) << " yus " << actor_info_.get_name() << " has_immediate_flags return "; return; } } while (flush_one_message()) { if (actor_execute_context_.has_immediate_flags()) { + LOG(INFO) << " yus " << actor_info_.get_name() << " has_immediate_flags return "; return; } } @@ -172,6 +175,7 @@ void ActorExecutor::finish() noexcept { } if (actor_locker_.try_unlock(flags())) { if (add_to_queue) { + LOG(INFO) << "yus " << actor_info_ptr->get_name() << " finishing "; dispatcher_.add_to_queue(std::move(actor_info_ptr), flags().get_scheduler_id(), !flags().is_shared()); } break; @@ -227,7 +231,8 @@ bool ActorExecutor::flush_one_signal(ActorSignals &signals) { bool ActorExecutor::flush_one_message() { auto message = actor_info_.mailbox().reader().read(); - //LOG(ERROR) << "flush one message " << !!message << " " << actor_info_.get_name(); + // LOG(ERROR) << "flush one message " << !!message << " " << actor_info_.get_name(); + LOG(INFO) << "yus " << "flush one message " << actor_info_.actor_ptr() << " name " << actor_info_.get_name(); if (!message) { pending_signals_.clear_signal(ActorSignals::Message); return false; diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index eeef47f18..6e9080415 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -22,6 +22,7 @@ #include "td/actor/core/SchedulerContext.h" #include "td/actor/core/Scheduler.h" // FIXME: afer LocalQueue is in a separate file +#include "td/utils/logging.h" namespace td { namespace actor { @@ -44,7 +45,8 @@ void CpuWorker::run() { return; } auto lock = debug.start(message->get_name()); - LOG(INFO) << "yus actor " << message->get_name() << " mailbox number " << message->mailbox().reader().calc_size(); + LOG(INFO) << "yus actor " << message->get_name() << " " << message->actor_ptr() << " mailbox number " + << message->mailbox().reader().calc_size(); ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); } else { LOG(INFO) << "yus no message, IDLE"; @@ -57,6 +59,7 @@ bool CpuWorker::try_pop_local(SchedulerMessage &message) { SchedulerMessage::Raw *raw_message; if (local_queues_[id_].try_pop(raw_message)) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); + LOG(INFO) << "yus " << message->get_name() << "from local queue_" << &local_queues_[id_] << message->actor_ptr(); return true; } return false; @@ -66,6 +69,7 @@ bool CpuWorker::try_pop_global(SchedulerMessage &message, size_t thread_id) { SchedulerMessage::Raw *raw_message; if (queue_.try_pop(raw_message, thread_id)) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); + LOG(INFO) << "yus " << message->get_name() << "from global queue_ " << message->actor_ptr(); return true; } return false; @@ -83,11 +87,15 @@ bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { } } + LOG(INFO) << " yus " << thread_id << " try to steal"; for (size_t i = 1; i < local_queues_.size(); i++) { size_t pos = (i + id_) % local_queues_.size(); SchedulerMessage::Raw *raw_message; if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); + + LOG(INFO) << " yus " << thread_id << " steal from " << i + id_ << " " << message->actor_ptr() << " name " + << message->get_name(); return true; } } diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index b29c00194..8295d904b 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -20,6 +20,7 @@ #include "td/actor/core/CpuWorker.h" #include "td/actor/core/IoWorker.h" +#include "td/utils/logging.h" namespace td { namespace actor { @@ -179,8 +180,8 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler if (should_notify) { info.cpu_queue_waiter->notify(); } - LOG(ERROR) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " - << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " + LOG(ERROR) << "yus add to queue: " << name << actor_info_ptr->actor_ptr() << " sche id " << scheduler_id.value() + << " " << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " << get_thread_id(); return; } @@ -219,6 +220,7 @@ void Scheduler::ContextImpl::set_alarm_timestamp(const ActorInfoPtr &actor_info_ // 2. Update timeout only when it has increased // 3. Use signal-like logic to combile multiple timeout updates into one if (!has_heap()) { + LOG(INFO) << " yus " << "set alarm " << actor_info_ptr->get_name() << " " << actor_info_ptr->actor_ptr(); add_to_queue(actor_info_ptr, {}, true); return; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index b0222aae8..bf7008878 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -444,7 +444,8 @@ void CellDbIn::migrate_cells() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); + LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << " " + << this->get_actor_info_ptr()->actor_ptr(); if (!started_) { td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { From 1ed08cb3c2b043f1e55b87d30facc404697297fd Mon Sep 17 00:00:00 2001 From: oker Date: Thu, 17 Oct 2024 11:41:32 +0800 Subject: [PATCH 068/101] more info about local queue --- tdactor/td/actor/core/ActorExecutor.cpp | 2 +- tdactor/td/actor/core/CpuWorker.cpp | 9 ++++----- tdactor/td/actor/core/Scheduler.cpp | 6 +++--- tdutils/td/utils/StealingQueue.h | 3 +++ validator/db/celldb.cpp | 3 +-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 6ce05ea47..415fe4f0a 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -232,7 +232,7 @@ bool ActorExecutor::flush_one_signal(ActorSignals &signals) { bool ActorExecutor::flush_one_message() { auto message = actor_info_.mailbox().reader().read(); // LOG(ERROR) << "flush one message " << !!message << " " << actor_info_.get_name(); - LOG(INFO) << "yus " << "flush one message " << actor_info_.actor_ptr() << " name " << actor_info_.get_name(); + LOG(INFO) << "yus " << "flush one message " << " name " << actor_info_.get_name(); if (!message) { pending_signals_.clear_signal(ActorSignals::Message); return false; diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index 6e9080415..e0a3a0974 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -45,7 +45,7 @@ void CpuWorker::run() { return; } auto lock = debug.start(message->get_name()); - LOG(INFO) << "yus actor " << message->get_name() << " " << message->actor_ptr() << " mailbox number " + LOG(INFO) << "yus actor " << message->get_name() << " " << " mailbox number " << message->mailbox().reader().calc_size(); ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); } else { @@ -59,7 +59,7 @@ bool CpuWorker::try_pop_local(SchedulerMessage &message) { SchedulerMessage::Raw *raw_message; if (local_queues_[id_].try_pop(raw_message)) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << "yus " << message->get_name() << "from local queue_" << &local_queues_[id_] << message->actor_ptr(); + LOG(INFO) << "yus " << message->get_name() << "from local queue_" << &local_queues_[id_]; return true; } return false; @@ -69,7 +69,7 @@ bool CpuWorker::try_pop_global(SchedulerMessage &message, size_t thread_id) { SchedulerMessage::Raw *raw_message; if (queue_.try_pop(raw_message, thread_id)) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << "yus " << message->get_name() << "from global queue_ " << message->actor_ptr(); + LOG(INFO) << "yus " << message->get_name() << "from global queue_ "; return true; } return false; @@ -94,8 +94,7 @@ bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << " yus " << thread_id << " steal from " << i + id_ << " " << message->actor_ptr() << " name " - << message->get_name(); + LOG(INFO) << " yus " << thread_id << " steal from " << pos << " " << " name " << message->get_name(); return true; } } diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 8295d904b..0a3c5e8c2 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -180,8 +180,8 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler if (should_notify) { info.cpu_queue_waiter->notify(); } - LOG(ERROR) << "yus add to queue: " << name << actor_info_ptr->actor_ptr() << " sche id " << scheduler_id.value() - << " " << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " + LOG(ERROR) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " + << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " << get_thread_id(); return; } @@ -220,7 +220,7 @@ void Scheduler::ContextImpl::set_alarm_timestamp(const ActorInfoPtr &actor_info_ // 2. Update timeout only when it has increased // 3. Use signal-like logic to combile multiple timeout updates into one if (!has_heap()) { - LOG(INFO) << " yus " << "set alarm " << actor_info_ptr->get_name() << " " << actor_info_ptr->actor_ptr(); + LOG(INFO) << " yus " << "set alarm " << actor_info_ptr->get_name() << " "; add_to_queue(actor_info_ptr, {}, true); return; } diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h index 20a39dc06..8fa3b198f 100644 --- a/tdutils/td/utils/StealingQueue.h +++ b/tdutils/td/utils/StealingQueue.h @@ -20,6 +20,7 @@ #include "td/utils/Status.h" #include "td/utils/Span.h" +#include "td/utils/logging.h" namespace td { template class StealingQueue { @@ -89,6 +90,8 @@ class StealingQueue { n -= n / 2; n = td::min(n, static_cast(head + N - tail)); if (n == 0) { + LOG(INFO) << "yus steal failed" << " other head " << other_head << " other tail " << other_tail << " N " << N + << " own head " << head << " own tail " << tail; return false; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index bf7008878..5ba8cc2f2 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -444,8 +444,7 @@ void CellDbIn::migrate_cells() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << " " - << this->get_actor_info_ptr()->actor_ptr(); + LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << " "; if (!started_) { td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { From fc4bd5d74c70ed00018bc296cbece2501e935ccb Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 14:12:12 +0800 Subject: [PATCH 069/101] update --- adnl/adnl-local-id.cpp | 4 ++-- validator/manager.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index 2b8bc0860..b2e33b950 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -88,8 +88,8 @@ void AdnlLocalId::deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::P auto f = cb.first; if (f.length() <= s.length() && s.as_slice().substr(0, f.length()) == f) { LOG(INFO) << "process_query, 2 over"; - //cb.second->receive_query(src, short_id_, std::move(s), std::move(promise)); - //return; + cb.second->receive_query(src, short_id_, std::move(s), std::move(promise)); + return; } } VLOG(ADNL_INFO) << this << ": dropping IN message from " << src diff --git a/validator/manager.cpp b/validator/manager.cpp index a87139a66..40118e8b6 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -581,7 +581,7 @@ void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { } void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) override { - LOG(INFO) << "add_ext_server_id, 1"; + LOG(INFO) << "run_ext_query, 1-1"; td::actor::send_closure(id_, &ValidatorManagerImpl::run_ext_query, std::move(data), std::move(promise)); } From b59a03f755d7b9c3598ffffe8501424dfeff78f7 Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 14:16:41 +0800 Subject: [PATCH 070/101] update --- adnl/adnl-ext-server.cpp | 2 ++ adnl/adnl-peer.cpp | 1 + rldp/rldp.cpp | 1 + rldp2/rldp.cpp | 1 + 4 files changed, 5 insertions(+) diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index 162a53afb..4684fdf08 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -37,6 +37,8 @@ td::Status AdnlInboundConnection::process_packet(td::BufferSlice data) { td::actor::send_closure(SelfId, &AdnlInboundConnection::send, serialize_tl_object(B, true)); } }); + + LOG(INFO) << "process_packet, 1"; td::actor::send_closure(peer_table_, &AdnlPeerTable::deliver_query, remote_id_, local_id_, std::move(f->query_), std::move(P)); return td::Status::OK(); diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 2b0077a8e..dfccdfbce 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -569,6 +569,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess } } }); + LOG(INFO) << "process_message, 1"; td::actor::send_closure(local_actor_, &AdnlLocalId::deliver_query, peer_id_short_, message.data(), std::move(P)); } diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 402740e19..57f3d7366 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -208,6 +208,7 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; + LOG(INFO) << "process_message, 1"; td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver_query, source, local_id, std::move(message.data_), std::move(P)); } diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 95780b237..7f2a80a2f 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -189,6 +189,7 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; + LOG(INFO) << "process_message, 3"; td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver_query, source, local_id, std::move(message.data_), std::move(P)); } From 6c32f1b378e38851c1f6182d8936ac85f2123f9a Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 14:34:57 +0800 Subject: [PATCH 071/101] updatge --- adnl/adnl-ext-connection.cpp | 2 +- adnl/adnl-local-id.cpp | 4 +++- validator/impl/liteserver.cpp | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/adnl/adnl-ext-connection.cpp b/adnl/adnl-ext-connection.cpp index 06926f4d1..1278adcfb 100644 --- a/adnl/adnl-ext-connection.cpp +++ b/adnl/adnl-ext-connection.cpp @@ -183,7 +183,7 @@ td::Status AdnlExtConnection::receive_packet(td::BufferSlice data) { if (processed) { return td::Status::OK(); } - + LOG(INFO) << "receive_packet, 1"; return process_packet(std::move(data)); } diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index b2e33b950..3ca45aa6c 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -85,9 +85,11 @@ void AdnlLocalId::deliver(AdnlNodeIdShort src, td::BufferSlice data) { void AdnlLocalId::deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { auto s = std::move(data); for (auto &cb : cb_) { + int a = 0; auto f = cb.first; if (f.length() <= s.length() && s.as_slice().substr(0, f.length()) == f) { - LOG(INFO) << "process_query, 2 over"; + LOG(INFO) << "process_query, 2 over" << ", a" << a; + a = a +1; cb.second->receive_query(src, short_id_, std::move(s), std::move(promise)); return; } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 9bfe8617f..5e52b8c38 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -221,7 +221,8 @@ void LiteQuery::perform() { [&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); }, [&](lite_api::liteServer_getMasterchainInfo& q) { LOG(INFO) << "fatal_error 120"; - this->perform_getMasterchainInfo(-1); }, + //this->perform_getMasterchainInfo(-1); + }, [&](lite_api::liteServer_getMasterchainInfoExt& q) { LOG(INFO) << "fatal_error 121"; this->perform_getMasterchainInfo(q.mode_ & 0x7fffffff); @@ -232,10 +233,12 @@ void LiteQuery::perform() { }, [&](lite_api::liteServer_getState& q) { this->perform_getState(ton::create_block_id(q.id_)); }, [&](lite_api::liteServer_getAccountState& q) { + LOG(INFO) << "perform_getAccountState before 1"; this->perform_getAccountState(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, 0); }, [&](lite_api::liteServer_getAccountStatePrunned& q) { + LOG(INFO) << "perform_getAccountState before 2"; this->perform_getAccountState(ton::create_block_id(q.id_), static_cast(q.account_->workchain_), q.account_->id_, 0x40000000); }, @@ -354,6 +357,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { void LiteQuery::gotMasterchainInfoForAccountState(Ref mc_state, BlockIdExt blkid, int mode) { + LOG(INFO) << "perform_getAccountState before 3"; perform_getAccountState(blkid, acc_workchain_, acc_addr_, 0x80000000); } @@ -983,6 +987,7 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St fatal_error("virtualization error while deserializing parameter list: "s + vme.get_msg()); return; } + LOG(INFO) << "perform_getAccountState before 4"; perform_getAccountState(blkid, workchain, addr, mode | 0x10000); } From c409dca4a830dd9007744956e9bc7c4b85890a8b Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 14:49:42 +0800 Subject: [PATCH 072/101] update --- validator/impl/liteserver.cpp | 11 ++++++++++- validator/manager.cpp | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 5e52b8c38..575b9383f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -135,14 +135,19 @@ void LiteQuery::alarm() { } bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) { + LOG(INFO) << "finish_query 1"; if (use_cache_ && !skip_cache_update) { + LOG(INFO) << "finish_query 2"; td::actor::send_closure(cache_, &LiteServerCache::update, cache_key_, result.clone()); } + LOG(INFO) << "finish_query 3"; if (promise_) { + LOG(INFO) << "finish_query 4"; promise_.set_result(std::move(result)); stop(); return true; } else { + LOG(INFO) << "finish_query 5"; stop(); return false; } @@ -221,7 +226,7 @@ void LiteQuery::perform() { [&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); }, [&](lite_api::liteServer_getMasterchainInfo& q) { LOG(INFO) << "fatal_error 120"; - //this->perform_getMasterchainInfo(-1); + this->perform_getMasterchainInfo(-1); }, [&](lite_api::liteServer_getMasterchainInfoExt& q) { LOG(INFO) << "fatal_error 121"; @@ -340,9 +345,11 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { fatal_error("unsupported getMasterchainInfo mode"); return; } + LOG(INFO) << "perform_getMasterchainInfo 1"; td::actor::send_closure_later(manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, [Self = actor_id(this), return_state = bool(acc_state_promise_), mode](td::Result, BlockIdExt>> res) { + LOG(INFO) << "perform_getMasterchainInfo 2"; if (res.is_error()) { LOG(INFO) << "fatal_error 5"; td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); @@ -350,6 +357,7 @@ void LiteQuery::perform_getMasterchainInfo(int mode) { auto pair = res.move_as_ok(); auto func = return_state ? &LiteQuery::gotMasterchainInfoForAccountState : &LiteQuery::continue_getMasterchainInfo; + LOG(INFO) << "perform_getMasterchainInfo 3"; td::actor::send_closure_later(Self, func, std::move(pair.first), pair.second, mode); } }); @@ -369,6 +377,7 @@ void LiteQuery::continue_getMasterchainInfo(Refget_zerostate_id(); auto zs_tl = create_tl_object(zerostate_id.workchain, zerostate_id.root_hash, zerostate_id.file_hash); diff --git a/validator/manager.cpp b/validator/manager.cpp index 40118e8b6..4f1a7838c 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1493,9 +1493,12 @@ void ValidatorManagerImpl::get_top_masterchain_state(td::Promise ValidatorManagerImpl::do_get_last_liteserver_state() { + LOG(INFO) << "do_get_last_liteserver_state 1"; if (last_masterchain_state_.is_null()) { + LOG(INFO) << "do_get_last_liteserver_state 2"; return {}; } + LOG(INFO) << "do_get_last_liteserver_state 3"; if (last_liteserver_state_.is_null()) { last_liteserver_state_ = last_masterchain_state_; return last_liteserver_state_; @@ -1533,6 +1536,7 @@ void ValidatorManagerImpl::get_top_masterchain_state_block( void ValidatorManagerImpl::get_last_liteserver_state_block( td::Promise, BlockIdExt>> promise) { + LOG(INFO) << "get_last_liteserver_state_block 1"; auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ton::ErrorCode::notready, "not started")); From 7afe8271f9c46c1f01fae5092c0b8de2d4b9ad4c Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 17:35:14 +0800 Subject: [PATCH 073/101] update --- validator/impl/liteserver.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 575b9383f..09e34e07b 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -225,8 +225,11 @@ void LiteQuery::perform() { [&](lite_api::liteServer_getTime& q) { this->perform_getTime(); }, [&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); }, [&](lite_api::liteServer_getMasterchainInfo& q) { - LOG(INFO) << "fatal_error 120"; - this->perform_getMasterchainInfo(-1); + static int a = 1; + LOG(INFO) << "fatal_error 120" << a++; + if (a <= 10) { + this->perform_getMasterchainInfo(-1); + } }, [&](lite_api::liteServer_getMasterchainInfoExt& q) { LOG(INFO) << "fatal_error 121"; From 540243c1d64e063c8abe475030ed7214fd5f3330 Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 17:48:18 +0800 Subject: [PATCH 074/101] update --- validator/impl/liteserver.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 09e34e07b..575b9383f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -225,11 +225,8 @@ void LiteQuery::perform() { [&](lite_api::liteServer_getTime& q) { this->perform_getTime(); }, [&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); }, [&](lite_api::liteServer_getMasterchainInfo& q) { - static int a = 1; - LOG(INFO) << "fatal_error 120" << a++; - if (a <= 10) { - this->perform_getMasterchainInfo(-1); - } + LOG(INFO) << "fatal_error 120"; + this->perform_getMasterchainInfo(-1); }, [&](lite_api::liteServer_getMasterchainInfoExt& q) { LOG(INFO) << "fatal_error 121"; From b3c2ecd403db8e784f78be66768c712bd84f540f Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 18:08:20 +0800 Subject: [PATCH 075/101] update --- adnl/adnl-ext-server.cpp | 2 +- validator/manager.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index 4684fdf08..014dd2fc8 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -38,7 +38,7 @@ td::Status AdnlInboundConnection::process_packet(td::BufferSlice data) { } }); - LOG(INFO) << "process_packet, 1"; + LOG(INFO) << "process_packet, 1" << ", f->query_ len:" << f->query_.size(); td::actor::send_closure(peer_table_, &AdnlPeerTable::deliver_query, remote_id_, local_id_, std::move(f->query_), std::move(P)); return td::Status::OK(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 4f1a7838c..268084cef 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -647,7 +647,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promisequery_ len:" << data.size(); auto E = fetch_tl_prefix(data, true); if (E.is_error()) { run_liteserver_query(std::move(data), actor_id(this), lite_server_cache_.get(), std::move(P)); From c907092ef9c0a97bf86496dfe620f2bd24a03467 Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 18:17:20 +0800 Subject: [PATCH 076/101] update --- validator/impl/liteserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 575b9383f..dd26687a3 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -165,7 +165,7 @@ void LiteQuery::start_up() { auto F = fetch_tl_object(query_, true); if (F.is_error()) { td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, 0); // unknown - LOG(INFO) << "fatal_error 2," << counter_; + LOG(INFO) << "fatal_error 2," << counter_ << ", f->query_ len:" << query_.size(); abort_query(F.move_as_error()); // std::this_thread::sleep_for(std::chrono::seconds(10000)); return; From bcacb689dbd2f48d7bec152e243cd01b51ab2adb Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 18:26:59 +0800 Subject: [PATCH 077/101] update --- validator/impl/fabric.cpp | 1 + validator/impl/liteserver.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 919abad0d..21946452e 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -251,6 +251,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise) { + LOG(INFO) << "run_liteserver_query, 1" << "f->query_ len:" << data.size(); LiteQuery::run_query(std::move(data), std::move(manager), std::move(cache), std::move(promise)); } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index dd26687a3..9616fcfaa 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -69,7 +69,7 @@ int global_count = 0; void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise) { - LOG(INFO) << "fatal_error 104"; + LOG(INFO) << "fatal_error 104" << ", f->query_ len:" << data.size(); td::actor::create_actor("litequery", std::move(data), std::move(manager), std::move(cache), std::move(promise)) .release(); From adfccc07a90595fbcb4c525f926b04f5392aec7a Mon Sep 17 00:00:00 2001 From: jianguo Date: Thu, 17 Oct 2024 18:47:24 +0800 Subject: [PATCH 078/101] update --- validator/manager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validator/manager.cpp b/validator/manager.cpp index 268084cef..7dc712822 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -650,10 +650,12 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promisequery_ len:" << data.size(); auto E = fetch_tl_prefix(data, true); if (E.is_error()) { + LOG(INFO) << "run_ext_query, 2" << "f->query_ len:" << data.size(); run_liteserver_query(std::move(data), actor_id(this), lite_server_cache_.get(), std::move(P)); } else { auto e = E.move_as_ok(); if (static_cast(e->seqno_) <= min_confirmed_masterchain_seqno_) { + LOG(INFO) << "run_ext_query, 3" << "f->query_ len:" << data.size(); run_liteserver_query(std::move(data), actor_id(this), lite_server_cache_.get(), std::move(P)); } else { auto t = e->timeout_ms_ < 10000 ? e->timeout_ms_ * 0.001 : 10.0; @@ -664,6 +666,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promisequery_ len:" << data.size(); run_liteserver_query(std::move(data), SelfId, cache, std::move(promise)); }); wait_shard_client_state(e->seqno_, td::Timestamp::in(t), std::move(Q)); From e14737b86146af3bba2224bd08fb41fe5e131263 Mon Sep 17 00:00:00 2001 From: oker Date: Fri, 18 Oct 2024 08:59:54 +0800 Subject: [PATCH 079/101] more log for load cell async --- tdactor/td/actor/common.h | 7 +------ tdactor/td/actor/core/CpuWorker.cpp | 3 ++- validator/db/celldb.cpp | 1 + 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index 17444239c..a325e9663 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -135,9 +135,7 @@ class Scheduler { while (scheduler->run(10)) { } }); - auto name = PSLICE() << "#" << it << ":io"; - LOG(INFO) << "yus scheduler start " << name << "scheduler size " << schedulers_.size(); - thread.set_name(name); + thread.set_name(PSLICE() << "#" << it << ":io"); thread.detach(); } } @@ -190,13 +188,11 @@ class Scheduler { bool skip_timeouts_{false}; void init() { - LOG(INFO) << " yus " << "create actor scheduler size " << infos_.size(); CHECK(infos_.size() < 256); CHECK(!group_info_); group_info_ = std::make_shared(infos_.size()); td::uint8 id = 0; for (const auto &info : infos_) { - LOG(INFO) << " yus NodeInfo " << id << "cpu thread " << info.cpu_threads_; schedulers_.emplace_back( td::make_unique(group_info_, core::SchedulerId{id}, info.cpu_threads_, skip_timeouts_)); id++; @@ -400,7 +396,6 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); - LOG(INFO) << "yus register acotr " << actor_info_ptr->get_name() << "thread id " << state.get_scheduler_id().value(); core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index e0a3a0974..a8bd74245 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -94,7 +94,8 @@ bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << " yus " << thread_id << " steal from " << pos << " " << " name " << message->get_name(); + LOG(INFO) << " yus " << thread_id << "local queue id " << id_ << " steal from " << pos << " " << " name " + << message->get_name(); return true; } } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 5ba8cc2f2..926927a49 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -456,6 +456,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise promise.set_result(R.move_as_ok()); } }); + LOG(INFO) << "yus before local cell async"; boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); } } From ec42edd317b0e1f2532158403958904e8aa20164 Mon Sep 17 00:00:00 2001 From: oker Date: Fri, 18 Oct 2024 09:34:19 +0800 Subject: [PATCH 080/101] async 2 --- crypto/vm/db/DynamicBagOfCellsDb.cpp | 69 +++++++++++++++++----------- validator/db/celldb.cpp | 3 ++ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 1aa4e0f56..21a40d896 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -88,37 +88,50 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) override { + LOG(INFO) << "yus load cell async 1"; auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { + LOG(INFO) << "yus load cell async 2"; TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell()); promise.set_result(loaded_cell.data_cell); return; } + LOG(INFO) << "yus load cell async 3"; SimpleExtCellCreator ext_cell_creator(cell_db_reader_); auto promise_ptr = std::make_shared>>(std::move(promise)); - executor->execute_async( - [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, - ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { - TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); - if (res.status != CellLoader::LoadResult::Ok) { - promise->set_error(td::Status::Error("cell not found")); - return; - } - Ref cell = res.cell(); - executor->execute_sync([hash, db, res = std::move(res), - ext_cell_creator = std::move(ext_cell_creator)]() mutable { - db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { - db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); - }); - for (auto &ext_cell : ext_cell_creator.get_created_cells()) { - auto ext_cell_hash = ext_cell->get_hash(); - db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) { - db->update_cell_info_created_ext(info, std::move(ext_cell)); - }); - } - }); - promise->set_result(std::move(cell)); + LOG(INFO) << "yus load cell async 4"; + executor->execute_async([executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, + ext_cell_creator = std::move(ext_cell_creator), + promise = std::move(promise_ptr)]() mutable { + LOG(INFO) << "yus load cell async 4 - 1"; + TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); + LOG(INFO) << "yus load cell async 4 - 2"; + if (res.status != CellLoader::LoadResult::Ok) { + LOG(INFO) << "yus load cell async 4 - 3"; + promise->set_error(td::Status::Error("cell not found")); + return; + } + LOG(INFO) << "yus load cell async 4 - 4"; + Ref cell = res.cell(); + executor->execute_sync([hash, db, res = std::move(res), + ext_cell_creator = std::move(ext_cell_creator)]() mutable { + LOG(INFO) << "yus load cell async 4 - 5"; + db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { + db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); }); + LOG(INFO) << "yus load cell async 4 - 6"; + for (auto &ext_cell : ext_cell_creator.get_created_cells()) { + auto ext_cell_hash = ext_cell->get_hash(); + db->hash_table_.apply(ext_cell_hash.as_slice(), + [&](CellInfo &info) { db->update_cell_info_created_ext(info, std::move(ext_cell)); }); + } + LOG(INFO) << "yus load cell async 4 - 7"; + }); + promise->set_result(std::move(cell)); + LOG(INFO) << "yus load cell async 4 - 8"; + }); + + LOG(INFO) << "yus load cell async 5"; } CellInfo &get_cell_info_force(td::Slice hash) { return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); @@ -214,7 +227,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat celldb_compress_depth_ = value; } - vm::ExtCellCreator& as_ext_cell_creator() override { + vm::ExtCellCreator &as_ext_cell_creator() override { return *this; } @@ -234,8 +247,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat class SimpleExtCellCreator : public ExtCellCreator { public: - explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : - cell_db_reader_(std::move(cell_db_reader)) {} + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) + : cell_db_reader_(std::move(cell_db_reader)) { + } td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, @@ -244,7 +258,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(ext_cell); } - std::vector>& get_created_cells() { + std::vector> &get_created_cells() { return created_cells_; } @@ -347,8 +361,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } bool not_in_db = false; - for_each( - info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); + for_each(info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); if (not_in_db) { CHECK(!info.in_db); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 926927a49..03635e231 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -143,6 +143,8 @@ void CellDbIn::start_up() { } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { + LOG(INFO) << " yus celldbin " << this->get_name() << " " + << this->get_actor_info_ptr()->mailbox().reader().calc_size(); boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise)); } @@ -446,6 +448,7 @@ void CellDbIn::migrate_cells() { void CellDb::load_cell(RootHash hash, td::Promise> promise) { LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << " "; if (!started_) { + LOG(INFO) << "directly send_closure to celldbin"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { auto P = td::PromiseCreator::lambda( From e92f64d4eacd24caa3f5c5ab3f7f170d9ecfa6e9 Mon Sep 17 00:00:00 2001 From: oker Date: Fri, 18 Oct 2024 09:34:24 +0800 Subject: [PATCH 081/101] Revert "more log for load cell async" This reverts commit e14737b86146af3bba2224bd08fb41fe5e131263. --- tdactor/td/actor/common.h | 7 ++++++- tdactor/td/actor/core/CpuWorker.cpp | 3 +-- validator/db/celldb.cpp | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index a325e9663..17444239c 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -135,7 +135,9 @@ class Scheduler { while (scheduler->run(10)) { } }); - thread.set_name(PSLICE() << "#" << it << ":io"); + auto name = PSLICE() << "#" << it << ":io"; + LOG(INFO) << "yus scheduler start " << name << "scheduler size " << schedulers_.size(); + thread.set_name(name); thread.detach(); } } @@ -188,11 +190,13 @@ class Scheduler { bool skip_timeouts_{false}; void init() { + LOG(INFO) << " yus " << "create actor scheduler size " << infos_.size(); CHECK(infos_.size() < 256); CHECK(!group_info_); group_info_ = std::make_shared(infos_.size()); td::uint8 id = 0; for (const auto &info : infos_) { + LOG(INFO) << " yus NodeInfo " << id << "cpu thread " << info.cpu_threads_; schedulers_.emplace_back( td::make_unique(group_info_, core::SchedulerId{id}, info.cpu_threads_, skip_timeouts_)); id++; @@ -396,6 +400,7 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); + LOG(INFO) << "yus register acotr " << actor_info_ptr->get_name() << "thread id " << state.get_scheduler_id().value(); core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index a8bd74245..e0a3a0974 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -94,8 +94,7 @@ bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << " yus " << thread_id << "local queue id " << id_ << " steal from " << pos << " " << " name " - << message->get_name(); + LOG(INFO) << " yus " << thread_id << " steal from " << pos << " " << " name " << message->get_name(); return true; } } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 03635e231..b0243de3b 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -459,7 +459,6 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise promise.set_result(R.move_as_ok()); } }); - LOG(INFO) << "yus before local cell async"; boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); } } From db8a4256fdb90cf0e88f4d9ee37880ac2c05d0eb Mon Sep 17 00:00:00 2001 From: oker Date: Fri, 18 Oct 2024 10:19:35 +0800 Subject: [PATCH 082/101] async 3 --- crypto/vm/db/DynamicBagOfCellsDb.cpp | 20 ++++++++++---------- tdactor/td/actor/core/CpuWorker.cpp | 3 ++- validator/db/celldb.cpp | 1 + 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 21a40d896..b53c4fd43 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -99,39 +99,39 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat LOG(INFO) << "yus load cell async 3"; SimpleExtCellCreator ext_cell_creator(cell_db_reader_); auto promise_ptr = std::make_shared>>(std::move(promise)); - LOG(INFO) << "yus load cell async 4"; + LOG(INFO) << "yus load cell async 4" << &ext_cell_creator; executor->execute_async([executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { - LOG(INFO) << "yus load cell async 4 - 1"; + LOG(INFO) << "yus load cell async 4 - 1 " << &ext_cell_creator; TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); - LOG(INFO) << "yus load cell async 4 - 2"; + LOG(INFO) << "yus load cell async 4 - 2 " << &ext_cell_creator; if (res.status != CellLoader::LoadResult::Ok) { - LOG(INFO) << "yus load cell async 4 - 3"; + LOG(INFO) << "yus load cell async 4 - 3 " << &ext_cell_creator; promise->set_error(td::Status::Error("cell not found")); return; } - LOG(INFO) << "yus load cell async 4 - 4"; + LOG(INFO) << "yus load cell async 4 - 4 " << &ext_cell_creator; Ref cell = res.cell(); executor->execute_sync([hash, db, res = std::move(res), ext_cell_creator = std::move(ext_cell_creator)]() mutable { - LOG(INFO) << "yus load cell async 4 - 5"; + LOG(INFO) << "yus load cell async 4 - 5 " << &ext_cell_creator; db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); }); - LOG(INFO) << "yus load cell async 4 - 6"; + LOG(INFO) << "yus load cell async 4 - 6 " << &ext_cell_creator; for (auto &ext_cell : ext_cell_creator.get_created_cells()) { auto ext_cell_hash = ext_cell->get_hash(); db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) { db->update_cell_info_created_ext(info, std::move(ext_cell)); }); } - LOG(INFO) << "yus load cell async 4 - 7"; + LOG(INFO) << "yus load cell async 4 - 7 " << &ext_cell_creator; }); promise->set_result(std::move(cell)); - LOG(INFO) << "yus load cell async 4 - 8"; + LOG(INFO) << "yus load cell async 4 - 8 " << &ext_cell_creator; }); - LOG(INFO) << "yus load cell async 5"; + LOG(INFO) << "yus load cell async 5 " << &ext_cell_creator; } CellInfo &get_cell_info_force(td::Slice hash) { return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index e0a3a0974..a8bd74245 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -94,7 +94,8 @@ bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << " yus " << thread_id << " steal from " << pos << " " << " name " << message->get_name(); + LOG(INFO) << " yus " << thread_id << "local queue id " << id_ << " steal from " << pos << " " << " name " + << message->get_name(); return true; } } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index b0243de3b..ea8b43375 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -454,6 +454,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise auto P = td::PromiseCreator::lambda( [cell_db_in = cell_db_.get(), hash, promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { + LOG(INFO) << "yus err then send to cell db in"; td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); } else { promise.set_result(R.move_as_ok()); From f0c4d84c1868c79bb0425c29b0be446f514a7291 Mon Sep 17 00:00:00 2001 From: oker Date: Fri, 18 Oct 2024 12:33:30 +0800 Subject: [PATCH 083/101] async log --- crypto/vm/db/DynamicBagOfCellsDb.cpp | 21 +++++++++++---------- tdactor/td/actor/core/ActorExecutor.cpp | 7 ++++--- tdactor/td/actor/core/ActorExecutor.h | 2 ++ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index b53c4fd43..eb8d2f5b1 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/db/DynamicBagOfCellsDb.h" +#include "td/utils/int_types.h" #include "vm/db/CellStorage.h" #include "vm/db/CellHashTable.h" @@ -99,39 +100,39 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat LOG(INFO) << "yus load cell async 3"; SimpleExtCellCreator ext_cell_creator(cell_db_reader_); auto promise_ptr = std::make_shared>>(std::move(promise)); - LOG(INFO) << "yus load cell async 4" << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4" << &(ext_cell_creator.get_created_cells()); executor->execute_async([executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { - LOG(INFO) << "yus load cell async 4 - 1 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 1 " << &(ext_cell_creator.get_created_cells()) << " " << hash; TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); - LOG(INFO) << "yus load cell async 4 - 2 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 2 " << &(ext_cell_creator.get_created_cells()) << " " << hash; if (res.status != CellLoader::LoadResult::Ok) { - LOG(INFO) << "yus load cell async 4 - 3 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 3 " << &(ext_cell_creator.get_created_cells()) << " " << hash; promise->set_error(td::Status::Error("cell not found")); return; } - LOG(INFO) << "yus load cell async 4 - 4 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 4 " << &(ext_cell_creator.get_created_cells()) << " " << hash; Ref cell = res.cell(); executor->execute_sync([hash, db, res = std::move(res), ext_cell_creator = std::move(ext_cell_creator)]() mutable { - LOG(INFO) << "yus load cell async 4 - 5 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 5 " << &(ext_cell_creator.get_created_cells()) << " " << hash; db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); }); - LOG(INFO) << "yus load cell async 4 - 6 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 6 " << &(ext_cell_creator.get_created_cells()) << " " << hash; for (auto &ext_cell : ext_cell_creator.get_created_cells()) { auto ext_cell_hash = ext_cell->get_hash(); db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) { db->update_cell_info_created_ext(info, std::move(ext_cell)); }); } - LOG(INFO) << "yus load cell async 4 - 7 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 7 " << &(ext_cell_creator.get_created_cells()) << " " << hash; }); promise->set_result(std::move(cell)); - LOG(INFO) << "yus load cell async 4 - 8 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 4 - 8 " << &(ext_cell_creator.get_created_cells()) << " " << hash; }); - LOG(INFO) << "yus load cell async 5 " << &ext_cell_creator; + LOG(INFO) << "yus load cell async 5 " << &(ext_cell_creator.get_created_cells()); } CellInfo &get_cell_info_force(td::Slice hash) { return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 415fe4f0a..98f797144 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -77,7 +77,7 @@ void ActorExecutor::send(ActorSignals signals) { } void ActorExecutor::start() noexcept { - //LOG(ERROR) << "START " << actor_info_.get_name() << " " << tag("from_queue", options_.from_queue); + LOG(ERROR) << "yus START " << actor_info_.get_name() << " " << tag("from_queue", options_.from_queue); if (is_closed()) { return; } @@ -117,13 +117,13 @@ void ActorExecutor::start() noexcept { while (flush_one_signal(signals)) { if (actor_execute_context_.has_immediate_flags()) { - LOG(INFO) << " yus " << actor_info_.get_name() << " has_immediate_flags return "; + LOG(ERROR) << " yus " << actor_info_.get_name() << " has_immediate_flags return "; return; } } while (flush_one_message()) { if (actor_execute_context_.has_immediate_flags()) { - LOG(INFO) << " yus " << actor_info_.get_name() << " has_immediate_flags return "; + LOG(ERROR) << " yus " << actor_info_.get_name() << " has_immediate_flags return " << &actor_execute_context_; return; } } @@ -234,6 +234,7 @@ bool ActorExecutor::flush_one_message() { // LOG(ERROR) << "flush one message " << !!message << " " << actor_info_.get_name(); LOG(INFO) << "yus " << "flush one message " << " name " << actor_info_.get_name(); if (!message) { + LOG(INFO) << "yus " << " name " << actor_info_.get_name() << " no message return"; pending_signals_.clear_signal(ActorSignals::Message); return false; } diff --git a/tdactor/td/actor/core/ActorExecutor.h b/tdactor/td/actor/core/ActorExecutor.h index 366cb754d..edd2a2568 100644 --- a/tdactor/td/actor/core/ActorExecutor.h +++ b/tdactor/td/actor/core/ActorExecutor.h @@ -27,6 +27,7 @@ #include "td/actor/core/SchedulerContext.h" #include "td/utils/format.h" +#include "td/utils/logging.h" namespace td { namespace actor { @@ -56,6 +57,7 @@ class ActorExecutor { : actor_info_(actor_info), dispatcher_(dispatcher), options_(options) { old_log_tag_ = LOG_TAG2; LOG_TAG2 = actor_info.get_name().c_str(); + LOG(INFO) << "yus " << LOG_TAG2; start(); } ActorExecutor(const ActorExecutor &) = delete; From 04aaadc7b850538484340de55558e81275ee8193 Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 18 Oct 2024 17:38:28 +0800 Subject: [PATCH 084/101] update --- validator/db/celldb.cpp | 32 ++++++++++++++++++++++++-------- validator/db/celldb.hpp | 1 + 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 3fb04dbf8..3b50ca82a 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -26,6 +26,8 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include +#include namespace ton { @@ -443,24 +445,38 @@ void CellDbIn::migrate_cells() { } } +int getRandom(){ + std::random_device rd; + std::mt19937 gen(rd()); // Mersenne Twister 19937 伪随机数生成器 + + // 定义范围 0 到 999 + std::uniform_int_distribution<> distr(0, 999); + + // 生成一个随机数 + int random_number = distr(gen); + + return random_number; +} + void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; + int ranNum = getRandom(); LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); if (!started_) { - // LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; - td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; + td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); } else { - // LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; auto P = td::PromiseCreator::lambda( - [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { - // LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; + [cell_db_in = cell_db_read_[ranNum].get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { + LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; if (R.is_error()) { td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); - // LOG(INFO) << " load_cell: counter" << counter_ << ", 6"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 6"; } else { - // LOG(INFO) << " load_cell: counter" << counter_ << ", 7-0"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 7-0"; promise.set_result(R.move_as_ok()); - // LOG(INFO) << " load_cell: counter" << counter_ << ", 7"; + LOG(INFO) << " load_cell: counter" << counter_ << ", 7"; } }); // LOG(INFO) << " load_cell: counter" << counter_ << ", 4"; diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 1fa4d508f..25c37059c 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -183,6 +183,7 @@ class CellDb : public CellDbBase { td::Ref opts_; td::actor::ActorOwn cell_db_; + td::actor::ActorOwn cell_db_read_[1000]; std::unique_ptr boc_; bool started_ = false; From 365f3c60091c69877c235b6aec8f39d312dd1594 Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 18 Oct 2024 17:46:58 +0800 Subject: [PATCH 085/101] update --- validator/db/celldb.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 3b50ca82a..02e667ce8 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -502,6 +502,10 @@ void CellDb::start_up() { boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_); + for (int i = 0; i < 1000; i++) { + cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_); + } + on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", cell_db_.get())), compress_depth = opts_->get_celldb_compress_depth()](const vm::CellLoader::LoadResult& res) { From 410661f84741e78545e26dbdab2430b848c7d41f Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 18 Oct 2024 18:17:49 +0800 Subject: [PATCH 086/101] update --- validator/db/celldb.cpp | 29 +++++++++++++++++++++++++---- validator/db/celldb.hpp | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 02e667ce8..418cc0121 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -71,8 +71,12 @@ void CellDbBase::execute_sync(std::function f) { } CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, - td::Ref opts) + td::Ref opts, std::shared_ptr cell_db) : root_db_(root_db), parent_(parent), path_(std::move(path)), opts_(opts) { + if (cell_db != NULL) { + LOG(INFO) << "zjg: using provided KeyValue"; + cell_db_ = cell_db; + } } void CellDbIn::start_up() { @@ -103,7 +107,10 @@ void CellDbIn::start_up() { LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); - cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + if (cell_db_ == NULL){ + LOG(INFO) << "zjg: using new KeyValue"; + cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + } boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); @@ -501,9 +508,23 @@ void CellDb::start_up() { CellDbBase::start_up(); boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); - cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_); + + td::RocksDbOptions db_options; + auto statistics_ = td::RocksDb::create_statistics(); + if (!opts_->get_disable_rocksdb_stats()) { + db_options.snapshot_statistics = std::make_shared(); + } + db_options.statistics = statistics_; + if (opts_->get_celldb_cache_size()) { + db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + } + db_options.use_direct_reads = opts_->get_celldb_direct_io(); + auto rock_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + + cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rock_db); for (int i = 0; i < 1000; i++) { - cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_); + cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rock_db); } on_load_callback_ = [actor = std::make_shared>( diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 25c37059c..70220049b 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -68,7 +68,7 @@ class CellDbIn : public CellDbBase { void flush_db_stats(); CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, - td::Ref opts); + td::Ref opts, std::shared_ptr cell_db); void start_up() override; void alarm() override; From 9fe16a0907a583cbcd06cc4729c477ff6152f78f Mon Sep 17 00:00:00 2001 From: jianguo Date: Fri, 18 Oct 2024 18:26:54 +0800 Subject: [PATCH 087/101] update --- validator/db/celldb.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 418cc0121..6bed383e6 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -454,21 +454,16 @@ void CellDbIn::migrate_cells() { int getRandom(){ std::random_device rd; - std::mt19937 gen(rd()); // Mersenne Twister 19937 伪随机数生成器 - - // 定义范围 0 到 999 + std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(0, 999); - - // 生成一个随机数 int random_number = distr(gen); - return random_number; } void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; int ranNum = getRandom(); - LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); + LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); From 016ce4a043a0ff22419fe299390d54f7ff5b275d Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 19 Oct 2024 09:20:46 +0800 Subject: [PATCH 088/101] update --- validator/manager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.hpp b/validator/manager.hpp index 529088562..8110e1316 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -678,7 +678,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::unique_ptr callback_; td::actor::ActorOwn db_; - bool started_ = true; + bool started_ = false; bool allow_validate_ = false; private: From 22bd19f69ae7383898ba1a8ce27415562b28a203 Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 19 Oct 2024 09:39:34 +0800 Subject: [PATCH 089/101] update --- tdactor/td/actor/core/Scheduler.cpp | 6 +++--- validator/db/celldb.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index b29c00194..581ae7a82 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -173,18 +173,18 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler auto raw = actor_info_ptr.release(); auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push(raw, [&](auto value) { info.cpu_queue->push(value, get_thread_id()); - LOG(ERROR) << "yus add to queue overflow: " << name << " sche id " << scheduler_id.value() << " worker id " + LOG(INFO) << "yus add to queue overflow: " << name << " sche id " << scheduler_id.value() << " worker id " << cpu_worker_id_.value() << " thread id " << get_thread_id(); }); if (should_notify) { info.cpu_queue_waiter->notify(); } - LOG(ERROR) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " + LOG(INFO) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " << get_thread_id(); return; } - LOG(ERROR) << "yus add to global queue: " << scheduler_id.value() << "thread id " << get_thread_id(); + LOG(INFO) << "yus add to global queue: " << scheduler_id.value() << "thread id " << get_thread_id(); info.cpu_queue->push(actor_info_ptr.release(), get_thread_id()); info.cpu_queue_waiter->notify(); } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 6bed383e6..23a57c11d 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -463,7 +463,7 @@ int getRandom(){ void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; int ranNum = getRandom(); - LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); From c9a55b45e1474b2008183fa6f7efe9dd3e3f7dbc Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 19 Oct 2024 09:44:20 +0800 Subject: [PATCH 090/101] update --- validator/db/celldb.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 23a57c11d..f3f707bb9 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -463,7 +463,11 @@ int getRandom(){ void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; int ranNum = getRandom(); - LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + static int64_t ranCount = 0; + ranCount++; + if (ranCount % 1000 == 0) { + LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + } if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); From b1ecd51bf4441f85158a9e0d6912fd23d22ade39 Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 19 Oct 2024 10:00:36 +0800 Subject: [PATCH 091/101] update --- validator/db/celldb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index f3f707bb9..c7cb40cb9 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -466,7 +466,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise static int64_t ranCount = 0; ranCount++; if (ranCount % 1000 == 0) { - LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; } if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; From 8a74db3df286ab8720514c8c99a1879720749148 Mon Sep 17 00:00:00 2001 From: jianguo Date: Sat, 19 Oct 2024 10:00:58 +0800 Subject: [PATCH 092/101] update --- validator/db/celldb.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index c7cb40cb9..b75e3e583 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -467,6 +467,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise ranCount++; if (ranCount % 1000 == 0) { LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + ranCount = 0; } if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; From 7e81128d0f6df97d7062b0eff54376d484ef1048 Mon Sep 17 00:00:00 2001 From: oker Date: Mon, 21 Oct 2024 14:10:05 +0800 Subject: [PATCH 093/101] change log from info to debug --- crypto/vm/db/DynamicBagOfCellsDb.cpp | 26 ++++++++++++------------- tdactor/td/actor/common.h | 8 ++++---- tdactor/td/actor/core/ActorExecutor.cpp | 12 ++++++------ tdactor/td/actor/core/ActorExecutor.h | 2 +- tdactor/td/actor/core/CpuWorker.cpp | 18 ++++++++--------- tdactor/td/actor/core/Scheduler.cpp | 14 ++++++------- tdutils/td/utils/StealingQueue.h | 2 +- validator/db/celldb.cpp | 8 ++++---- 8 files changed, 45 insertions(+), 45 deletions(-) diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index eb8d2f5b1..db809a86f 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -89,50 +89,50 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) override { - LOG(INFO) << "yus load cell async 1"; + LOG(DEBUG) << "yus load cell async 1"; auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { - LOG(INFO) << "yus load cell async 2"; + LOG(DEBUG) << "yus load cell async 2"; TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell()); promise.set_result(loaded_cell.data_cell); return; } - LOG(INFO) << "yus load cell async 3"; + LOG(DEBUG) << "yus load cell async 3"; SimpleExtCellCreator ext_cell_creator(cell_db_reader_); auto promise_ptr = std::make_shared>>(std::move(promise)); - LOG(INFO) << "yus load cell async 4" << &(ext_cell_creator.get_created_cells()); + LOG(DEBUG) << "yus load cell async 4" << &(ext_cell_creator.get_created_cells()); executor->execute_async([executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { - LOG(INFO) << "yus load cell async 4 - 1 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 1 " << &(ext_cell_creator.get_created_cells()) << " " << hash; TRY_RESULT_PROMISE((*promise), res, loader.load(hash.as_slice(), true, ext_cell_creator)); - LOG(INFO) << "yus load cell async 4 - 2 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 2 " << &(ext_cell_creator.get_created_cells()) << " " << hash; if (res.status != CellLoader::LoadResult::Ok) { - LOG(INFO) << "yus load cell async 4 - 3 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 3 " << &(ext_cell_creator.get_created_cells()) << " " << hash; promise->set_error(td::Status::Error("cell not found")); return; } - LOG(INFO) << "yus load cell async 4 - 4 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 4 " << &(ext_cell_creator.get_created_cells()) << " " << hash; Ref cell = res.cell(); executor->execute_sync([hash, db, res = std::move(res), ext_cell_creator = std::move(ext_cell_creator)]() mutable { - LOG(INFO) << "yus load cell async 4 - 5 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 5 " << &(ext_cell_creator.get_created_cells()) << " " << hash; db->hash_table_.apply(hash.as_slice(), [&](CellInfo &info) { db->update_cell_info_loaded(info, hash.as_slice(), std::move(res)); }); - LOG(INFO) << "yus load cell async 4 - 6 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 6 " << &(ext_cell_creator.get_created_cells()) << " " << hash; for (auto &ext_cell : ext_cell_creator.get_created_cells()) { auto ext_cell_hash = ext_cell->get_hash(); db->hash_table_.apply(ext_cell_hash.as_slice(), [&](CellInfo &info) { db->update_cell_info_created_ext(info, std::move(ext_cell)); }); } - LOG(INFO) << "yus load cell async 4 - 7 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 7 " << &(ext_cell_creator.get_created_cells()) << " " << hash; }); promise->set_result(std::move(cell)); - LOG(INFO) << "yus load cell async 4 - 8 " << &(ext_cell_creator.get_created_cells()) << " " << hash; + LOG(DEBUG) << "yus load cell async 4 - 8 " << &(ext_cell_creator.get_created_cells()) << " " << hash; }); - LOG(INFO) << "yus load cell async 5 " << &(ext_cell_creator.get_created_cells()); + LOG(DEBUG) << "yus load cell async 5 " << &(ext_cell_creator.get_created_cells()); } CellInfo &get_cell_info_force(td::Slice hash) { return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index 17444239c..9efc3f902 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -136,7 +136,7 @@ class Scheduler { } }); auto name = PSLICE() << "#" << it << ":io"; - LOG(INFO) << "yus scheduler start " << name << "scheduler size " << schedulers_.size(); + LOG(DEBUG) << "yus scheduler start " << name << "scheduler size " << schedulers_.size(); thread.set_name(name); thread.detach(); } @@ -190,13 +190,13 @@ class Scheduler { bool skip_timeouts_{false}; void init() { - LOG(INFO) << " yus " << "create actor scheduler size " << infos_.size(); + LOG(DEBUG) << "yus " << "create actor scheduler size " << infos_.size(); CHECK(infos_.size() < 256); CHECK(!group_info_); group_info_ = std::make_shared(infos_.size()); td::uint8 id = 0; for (const auto &info : infos_) { - LOG(INFO) << " yus NodeInfo " << id << "cpu thread " << info.cpu_threads_; + LOG(DEBUG) << "yus NodeInfo " << id << "cpu thread " << info.cpu_threads_; schedulers_.emplace_back( td::make_unique(group_info_, core::SchedulerId{id}, info.cpu_threads_, skip_timeouts_)); id++; @@ -400,7 +400,7 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); - LOG(INFO) << "yus register acotr " << actor_info_ptr->get_name() << "thread id " << state.get_scheduler_id().value(); + LOG(DEBUG) << "yus register acotr " << actor_info_ptr->get_name() << "thread id " << state.get_scheduler_id().value(); core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 98f797144..bfc4ffe2c 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -77,7 +77,7 @@ void ActorExecutor::send(ActorSignals signals) { } void ActorExecutor::start() noexcept { - LOG(ERROR) << "yus START " << actor_info_.get_name() << " " << tag("from_queue", options_.from_queue); + LOG(DEBUG) << "yus START " << actor_info_.get_name() << " " << tag("from_queue", options_.from_queue); if (is_closed()) { return; } @@ -117,13 +117,13 @@ void ActorExecutor::start() noexcept { while (flush_one_signal(signals)) { if (actor_execute_context_.has_immediate_flags()) { - LOG(ERROR) << " yus " << actor_info_.get_name() << " has_immediate_flags return "; + LOG(DEBUG) << "yus " << actor_info_.get_name() << " has_immediate_flags return "; return; } } while (flush_one_message()) { if (actor_execute_context_.has_immediate_flags()) { - LOG(ERROR) << " yus " << actor_info_.get_name() << " has_immediate_flags return " << &actor_execute_context_; + LOG(DEBUG) << "yus " << actor_info_.get_name() << " has_immediate_flags return " << &actor_execute_context_; return; } } @@ -175,7 +175,7 @@ void ActorExecutor::finish() noexcept { } if (actor_locker_.try_unlock(flags())) { if (add_to_queue) { - LOG(INFO) << "yus " << actor_info_ptr->get_name() << " finishing "; + LOG(DEBUG) << "yus " << actor_info_ptr->get_name() << " finishing "; dispatcher_.add_to_queue(std::move(actor_info_ptr), flags().get_scheduler_id(), !flags().is_shared()); } break; @@ -232,9 +232,9 @@ bool ActorExecutor::flush_one_signal(ActorSignals &signals) { bool ActorExecutor::flush_one_message() { auto message = actor_info_.mailbox().reader().read(); // LOG(ERROR) << "flush one message " << !!message << " " << actor_info_.get_name(); - LOG(INFO) << "yus " << "flush one message " << " name " << actor_info_.get_name(); + LOG(DEBUG) << "yus " << "flush one message " << " name " << actor_info_.get_name(); if (!message) { - LOG(INFO) << "yus " << " name " << actor_info_.get_name() << " no message return"; + LOG(DEBUG) << "yus " << " name " << actor_info_.get_name() << " no message return"; pending_signals_.clear_signal(ActorSignals::Message); return false; } diff --git a/tdactor/td/actor/core/ActorExecutor.h b/tdactor/td/actor/core/ActorExecutor.h index edd2a2568..8e3066e97 100644 --- a/tdactor/td/actor/core/ActorExecutor.h +++ b/tdactor/td/actor/core/ActorExecutor.h @@ -57,7 +57,7 @@ class ActorExecutor { : actor_info_(actor_info), dispatcher_(dispatcher), options_(options) { old_log_tag_ = LOG_TAG2; LOG_TAG2 = actor_info.get_name().c_str(); - LOG(INFO) << "yus " << LOG_TAG2; + LOG(DEBUG) << "yus " << LOG_TAG2; start(); } ActorExecutor(const ActorExecutor &) = delete; diff --git a/tdactor/td/actor/core/CpuWorker.cpp b/tdactor/td/actor/core/CpuWorker.cpp index a8bd74245..70ab5c690 100644 --- a/tdactor/td/actor/core/CpuWorker.cpp +++ b/tdactor/td/actor/core/CpuWorker.cpp @@ -30,26 +30,26 @@ namespace core { void CpuWorker::run() { auto thread_id = get_thread_id(); auto &dispatcher = *SchedulerContext::get(); - LOG(INFO) << "yus start work " << thread_id; + LOG(DEBUG) << "yus start work " << thread_id; MpmcWaiter::Slot slot; waiter_.init_slot(slot, thread_id); auto &debug = dispatcher.get_debug(); - LOG(INFO) << "yus thread_id" << thread_id << "worker id " << this->id_; + LOG(DEBUG) << "yus thread_id" << thread_id << "worker id " << this->id_; while (true) { SchedulerMessage message; if (try_pop(message, thread_id)) { waiter_.stop_wait(slot); if (!message) { - LOG(INFO) << "yus no message return" << " thread_id " << thread_id; + LOG(DEBUG) << "yus no message return" << " thread_id " << thread_id; return; } auto lock = debug.start(message->get_name()); - LOG(INFO) << "yus actor " << message->get_name() << " " << " mailbox number " + LOG(DEBUG) << "yus actor " << message->get_name() << " " << " mailbox number " << message->mailbox().reader().calc_size(); ActorExecutor executor(*message, dispatcher, ActorExecutor::Options().with_from_queue()); } else { - LOG(INFO) << "yus no message, IDLE"; + LOG(DEBUG) << "yus no message, IDLE"; waiter_.wait(slot); } } @@ -59,7 +59,7 @@ bool CpuWorker::try_pop_local(SchedulerMessage &message) { SchedulerMessage::Raw *raw_message; if (local_queues_[id_].try_pop(raw_message)) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << "yus " << message->get_name() << "from local queue_" << &local_queues_[id_]; + LOG(DEBUG) << "yus " << message->get_name() << "from local queue_" << &local_queues_[id_]; return true; } return false; @@ -69,7 +69,7 @@ bool CpuWorker::try_pop_global(SchedulerMessage &message, size_t thread_id) { SchedulerMessage::Raw *raw_message; if (queue_.try_pop(raw_message, thread_id)) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << "yus " << message->get_name() << "from global queue_ "; + LOG(DEBUG) << "yus " << message->get_name() << "from global queue_ "; return true; } return false; @@ -87,14 +87,14 @@ bool CpuWorker::try_pop(SchedulerMessage &message, size_t thread_id) { } } - LOG(INFO) << " yus " << thread_id << " try to steal"; + LOG(DEBUG) << "yus " << thread_id << " try to steal"; for (size_t i = 1; i < local_queues_.size(); i++) { size_t pos = (i + id_) % local_queues_.size(); SchedulerMessage::Raw *raw_message; if (local_queues_[id_].steal(raw_message, local_queues_[pos])) { message = SchedulerMessage(SchedulerMessage::acquire_t{}, raw_message); - LOG(INFO) << " yus " << thread_id << "local queue id " << id_ << " steal from " << pos << " " << " name " + LOG(DEBUG) << "yus " << thread_id << "local queue id " << id_ << " steal from " << pos << " " << " name " << message->get_name(); return true; } diff --git a/tdactor/td/actor/core/Scheduler.cpp b/tdactor/td/actor/core/Scheduler.cpp index 0a3c5e8c2..9613b8cb6 100644 --- a/tdactor/td/actor/core/Scheduler.cpp +++ b/tdactor/td/actor/core/Scheduler.cpp @@ -68,7 +68,7 @@ Scheduler::Scheduler(std::shared_ptr scheduler_group_info, S scheduler_group_info_->iocp.init(); } #endif - LOG(INFO) << "yus scheduler create " << "id " << id.value() << "count " << cpu_threads_count << "scheduler_group_ " + LOG(DEBUG) << "yus scheduler create " << "id " << id.value() << "count " << cpu_threads_count << "scheduler_group_ " << scheduler_group_info_->active_scheduler_count << " " << scheduler_group_info_->schedulers.size(); } @@ -79,7 +79,7 @@ Scheduler::~Scheduler() { } void Scheduler::start() { - LOG(INFO) << "yus start scheduler, cpu size " << cpu_threads_.size(); + LOG(DEBUG) << "yus start scheduler, cpu size " << cpu_threads_.size(); for (size_t i = 0; i < cpu_threads_.size(); i++) { cpu_threads_[i] = td::thread([this, i] { this->run_in_context_impl(*this->info_->cpu_workers[i], [this, i] { @@ -87,7 +87,7 @@ void Scheduler::start() { }); }); auto name = PSLICE() << "#" << info_->id.value() << ":cpu#" << i; - LOG(INFO) << "yus create " << "cpu_workers " << name; + LOG(DEBUG) << "yus create " << "cpu_workers " << name; cpu_threads_[i].set_name(name); } #if TD_PORT_WINDOWS @@ -174,18 +174,18 @@ void Scheduler::ContextImpl::add_to_queue(ActorInfoPtr actor_info_ptr, Scheduler auto raw = actor_info_ptr.release(); auto should_notify = info.cpu_local_queue[cpu_worker_id_.value()].push(raw, [&](auto value) { info.cpu_queue->push(value, get_thread_id()); - LOG(ERROR) << "yus add to queue overflow: " << name << " sche id " << scheduler_id.value() << " worker id " + LOG(DEBUG) << "yus add to queue overflow: " << name << " sche id " << scheduler_id.value() << " worker id " << cpu_worker_id_.value() << " thread id " << get_thread_id(); }); if (should_notify) { info.cpu_queue_waiter->notify(); } - LOG(ERROR) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " + LOG(DEBUG) << "yus add to queue: " << name << " sche id " << scheduler_id.value() << " " << get_scheduler_id().value() << " worker id " << cpu_worker_id_.value() << " thread id " << get_thread_id(); return; } - LOG(ERROR) << "yus add to global queue: " << scheduler_id.value() << "thread id " << get_thread_id(); + LOG(DEBUG) << "yus add to global queue: " << scheduler_id.value() << "thread id " << get_thread_id(); info.cpu_queue->push(actor_info_ptr.release(), get_thread_id()); info.cpu_queue_waiter->notify(); } @@ -220,7 +220,7 @@ void Scheduler::ContextImpl::set_alarm_timestamp(const ActorInfoPtr &actor_info_ // 2. Update timeout only when it has increased // 3. Use signal-like logic to combile multiple timeout updates into one if (!has_heap()) { - LOG(INFO) << " yus " << "set alarm " << actor_info_ptr->get_name() << " "; + LOG(DEBUG) << "yus " << "set alarm " << actor_info_ptr->get_name() << " "; add_to_queue(actor_info_ptr, {}, true); return; } diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h index 8fa3b198f..673e9018f 100644 --- a/tdutils/td/utils/StealingQueue.h +++ b/tdutils/td/utils/StealingQueue.h @@ -90,7 +90,7 @@ class StealingQueue { n -= n / 2; n = td::min(n, static_cast(head + N - tail)); if (n == 0) { - LOG(INFO) << "yus steal failed" << " other head " << other_head << " other tail " << other_tail << " N " << N + LOG(DEBUG) << "yus steal failed" << " other head " << other_head << " other tail " << other_tail << " N " << N << " own head " << head << " own tail " << tail; return false; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index ea8b43375..115d34cfd 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -143,7 +143,7 @@ void CellDbIn::start_up() { } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { - LOG(INFO) << " yus celldbin " << this->get_name() << " " + LOG(DEBUG) << "yus celldbin " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise)); } @@ -446,15 +446,15 @@ void CellDbIn::migrate_cells() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - LOG(INFO) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << " "; + LOG(DEBUG) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << " "; if (!started_) { - LOG(INFO) << "directly send_closure to celldbin"; + LOG(DEBUG) << "yus directly send_closure to celldbin"; td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { auto P = td::PromiseCreator::lambda( [cell_db_in = cell_db_.get(), hash, promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { - LOG(INFO) << "yus err then send to cell db in"; + LOG(DEBUG) << "yus err then send to cell db in"; td::actor::send_closure(cell_db_in, &CellDbIn::load_cell, hash, std::move(promise)); } else { promise.set_result(R.move_as_ok()); From 9b1165f711ba11c26f942e56d827a14072a774a4 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 14:56:22 +0800 Subject: [PATCH 094/101] root db --- validator/db/celldb.cpp | 42 +++++++++++++++++++++-------------------- validator/db/celldb.hpp | 5 +++-- validator/db/rootdb.cpp | 42 +++++++++++++++++++++++++++++++++++++++-- validator/db/rootdb.hpp | 1 + 4 files changed, 66 insertions(+), 24 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index d997496dc..5f4e57828 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -468,16 +468,18 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise static int64_t ranCount = 0; ranCount++; if (ranCount % 1000 == 0) { - LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + // LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); ranCount = 0; } if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; - td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); + // td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); + td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; auto P = td::PromiseCreator::lambda( - [cell_db_in = cell_db_read_[ranNum].get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { + [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; if (R.is_error()) { LOG(DEBUG) << "yus err then send to cell db in"; @@ -512,23 +514,23 @@ void CellDb::start_up() { boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); - td::RocksDbOptions db_options; - auto statistics_ = td::RocksDb::create_statistics(); - if (!opts_->get_disable_rocksdb_stats()) { - db_options.snapshot_statistics = std::make_shared(); - } - db_options.statistics = statistics_; - if (opts_->get_celldb_cache_size()) { - db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); - } - db_options.use_direct_reads = opts_->get_celldb_direct_io(); - auto rock_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); - - cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rock_db); - for (int i = 0; i < 1000; i++) { - cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rock_db); - } + // td::RocksDbOptions db_options; + // auto statistics_ = td::RocksDb::create_statistics(); + // if (!opts_->get_disable_rocksdb_stats()) { + // db_options.snapshot_statistics = std::make_shared(); + // } + // db_options.statistics = statistics_; + // if (opts_->get_celldb_cache_size()) { + // db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); + // LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + // } + // db_options.use_direct_reads = opts_->get_celldb_direct_io(); + // auto rock_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + + cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); + // for (int i = 0; i < 10; i++) { + // cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rock_db); + // } on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", cell_db_.get())), diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 70220049b..8256d9fe4 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -171,8 +171,8 @@ class CellDb : public CellDbBase { void get_cell_db_reader(td::Promise> promise); void get_last_deleted_mc_state(td::Promise promise); - CellDb(td::actor::ActorId root_db, std::string path, td::Ref opts) - : root_db_(root_db), path_(path), opts_(opts) { + CellDb(td::actor::ActorId root_db, std::string path, td::Ref opts, std::shared_ptr rocks_db) + : root_db_(root_db), path_(path), opts_(opts), rocks_db_(rocks_db) { } void start_up() override; @@ -184,6 +184,7 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; td::actor::ActorOwn cell_db_read_[1000]; + std::shared_ptr rocks_db_; std::unique_ptr boc_; bool started_ = false; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index d41068465..66a15abdf 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -26,6 +26,8 @@ #include "common/checksum.h" #include "validator/stats-merger.h" #include "td/actor/MultiPromise.h" +#include +#include namespace ton { @@ -248,6 +250,14 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } +int getRandom(){ + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(0, 99); + int random_number = distr(gen); + return random_number; +} + void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { LOG(INFO) << "get_block_state, counter" << counter_ << ", 1"; if (handle->inited_state_boc()) { @@ -273,7 +283,16 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promisestate(), std::move(P), counter_); + + int ranNum = getRandom(); + static int64_t ranCount = 0; + ranCount++; + if (ranCount % 1000 == 0) { + LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + ranCount = 0; + } + + td::actor::send_closure(cell_db_read_[ranNum], &CellDb::load_cell, handle->state(), std::move(P), counter_); LOG(INFO) << "get_block_state, counter" << counter_ << ", 4-1"; } else { LOG(INFO) << "get_block_state, counter" << counter_ << ", 7-0"; @@ -424,7 +443,26 @@ void RootDb::get_hardforks(td::Promise> promise) { } void RootDb::start_up() { - cell_db_ = td::actor::create_actor("celldb", actor_id(this), root_path_ + "/celldb/", opts_); + + td::RocksDbOptions db_options; + auto statistics_ = td::RocksDb::create_statistics(); + if (!opts_->get_disable_rocksdb_stats()) { + db_options.snapshot_statistics = std::make_shared(); + } + db_options.statistics = statistics_; + if (opts_->get_celldb_cache_size()) { + db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + } + db_options.use_direct_reads = opts_->get_celldb_direct_io(); + auto path = root_path_ + "/celldb/"; + auto rock_db = std::make_shared(td::RocksDb::open(path, std::move(db_options)).move_as_ok()); + + cell_db_ = td::actor::create_actor("celldb", actor_id(this), path, opts_, rock_db); + for (int i = 0; i < 100; i++){ + cell_db_read_[i] = td::actor::create_actor("celldb", actor_id(this), path, opts_, rock_db); + } + state_db_ = td::actor::create_actor("statedb", actor_id(this), root_path_ + "/state/"); static_files_db_ = td::actor::create_actor("staticfilesdb", actor_id(this), root_path_ + "/static/"); archive_db_ = td::actor::create_actor("archive", actor_id(this), root_path_, opts_); diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 02349c038..b8aae5bc0 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -145,6 +145,7 @@ class RootDb : public Db { td::Ref opts_; td::actor::ActorOwn cell_db_; + td::actor::ActorOwn cell_db_read_[100]; td::actor::ActorOwn state_db_; td::actor::ActorOwn static_files_db_; td::actor::ActorOwn archive_db_; From e64fba950e2cda914c8f9e44c9594381dcc7b4bc Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 15:03:44 +0800 Subject: [PATCH 095/101] update --- validator/db/celldb.cpp | 4 ++-- validator/db/rootdb.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 5f4e57828..8ac0a0825 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -454,7 +454,7 @@ void CellDbIn::migrate_cells() { } } -int getRandom(){ +int getRandom1(){ std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(0, 999); @@ -464,7 +464,7 @@ int getRandom(){ void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; - int ranNum = getRandom(); + int ranNum = getRandom1(); static int64_t ranCount = 0; ranCount++; if (ranCount % 1000 == 0) { diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 66a15abdf..90273c8ce 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -250,7 +250,7 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } -int getRandom(){ +int getRandom2(){ std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(0, 99); @@ -284,7 +284,7 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise Date: Tue, 22 Oct 2024 15:16:00 +0800 Subject: [PATCH 096/101] update --- validator/manager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.hpp b/validator/manager.hpp index 8110e1316..529088562 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -678,7 +678,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::unique_ptr callback_; td::actor::ActorOwn db_; - bool started_ = false; + bool started_ = true; bool allow_validate_ = false; private: From 8c39d203afe7173fd6e4eaf17f04d9a6bcbe83ed Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 16:05:00 +0800 Subject: [PATCH 097/101] update --- validator/db/celldb.cpp | 19 ++++++++++--------- validator/db/celldb.hpp | 2 +- validator/db/rootdb.cpp | 4 ++-- validator/db/rootdb.hpp | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 8ac0a0825..b8b791aca 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -457,7 +457,7 @@ void CellDbIn::migrate_cells() { int getRandom1(){ std::random_device rd; std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(0, 999); + std::uniform_int_distribution<> distr(0, 9); int random_number = distr(gen); return random_number; } @@ -468,18 +468,19 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise static int64_t ranCount = 0; ranCount++; if (ranCount % 1000 == 0) { - // LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; - LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); + LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size() << ", ranNum: " << ranNum; + // LOG(ERROR) << "yus " << this->get_name() << " " << this->get_actor_info_ptr()->mailbox().reader().calc_size(); ranCount = 0; } if (!started_) { LOG(INFO) << " load_cell: counter" << counter_ << ", 2"; - // td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); - td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + td::actor::send_closure(cell_db_read_[ranNum], &CellDbIn::load_cell, hash, std::move(promise)); + // td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { LOG(INFO) << " load_cell: counter" << counter_ << ", 3"; auto P = td::PromiseCreator::lambda( - [cell_db_in = cell_db_.get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { + //[cell_db_in = cell_db_.get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { + [cell_db_in = cell_db_read_[ranNum].get(), hash, promise = std::move(promise), counter_](td::Result> R) mutable { LOG(INFO) << " load_cell: counter" << counter_ << ", 5"; if (R.is_error()) { LOG(DEBUG) << "yus err then send to cell db in"; @@ -528,9 +529,9 @@ void CellDb::start_up() { // auto rock_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); - // for (int i = 0; i < 10; i++) { - // cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rock_db); - // } + for (int i = 0; i < 10; i++) { + cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, cell_db_); + } on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", cell_db_.get())), diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 8256d9fe4..83a30d49c 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -183,7 +183,7 @@ class CellDb : public CellDbBase { td::Ref opts_; td::actor::ActorOwn cell_db_; - td::actor::ActorOwn cell_db_read_[1000]; + td::actor::ActorOwn cell_db_read_[10]; std::shared_ptr rocks_db_; std::unique_ptr boc_; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 90273c8ce..95b900313 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -253,7 +253,7 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, int getRandom2(){ std::random_device rd; std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(0, 99); + std::uniform_int_distribution<> distr(0, 9); int random_number = distr(gen); return random_number; } @@ -459,7 +459,7 @@ void RootDb::start_up() { auto rock_db = std::make_shared(td::RocksDb::open(path, std::move(db_options)).move_as_ok()); cell_db_ = td::actor::create_actor("celldb", actor_id(this), path, opts_, rock_db); - for (int i = 0; i < 100; i++){ + for (int i = 0; i < 10; i++){ cell_db_read_[i] = td::actor::create_actor("celldb", actor_id(this), path, opts_, rock_db); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index b8aae5bc0..b2e313175 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -145,7 +145,7 @@ class RootDb : public Db { td::Ref opts_; td::actor::ActorOwn cell_db_; - td::actor::ActorOwn cell_db_read_[100]; + td::actor::ActorOwn cell_db_read_[10]; td::actor::ActorOwn state_db_; td::actor::ActorOwn static_files_db_; td::actor::ActorOwn archive_db_; From 32d6314f3d8847e530cd6568472f10a8e8912594 Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 16:10:52 +0800 Subject: [PATCH 098/101] update --- validator/db/celldb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index b8b791aca..a3e25213e 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -530,7 +530,7 @@ void CellDb::start_up() { cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); for (int i = 0; i < 10; i++) { - cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, cell_db_); + cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, cell_db_, rocks_db_); } on_load_callback_ = [actor = std::make_shared>( From 6e5e60785081dcbc2d2ff3a415a425cf792c33ac Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 16:17:16 +0800 Subject: [PATCH 099/101] update --- validator/db/celldb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index a3e25213e..d92648cf1 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -530,7 +530,7 @@ void CellDb::start_up() { cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); for (int i = 0; i < 10; i++) { - cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, cell_db_, rocks_db_); + cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); } on_load_callback_ = [actor = std::make_shared>( From a509286a34888c040efe74bb75a83fcaf44ce4de Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 16:26:37 +0800 Subject: [PATCH 100/101] update --- validator/db/celldb.cpp | 8 ++++---- validator/db/celldb.hpp | 6 +++++- validator/db/rootdb.cpp | 12 ++---------- validator/db/rootdb.hpp | 2 +- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index d92648cf1..71c027765 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -454,17 +454,17 @@ void CellDbIn::migrate_cells() { } } -int getRandom1(){ +int GetRandom(){ std::random_device rd; std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(0, 9); + std::uniform_int_distribution<> distr(0, THREAD_COUNTS -1 ); int random_number = distr(gen); return random_number; } void CellDb::load_cell(RootHash hash, td::Promise> promise, std::uint64_t counter_) { // LOG(INFO) << " load_cell: counter" << counter_ << ", 1"; - int ranNum = getRandom1(); + int ranNum = GetRandom(); static int64_t ranCount = 0; ranCount++; if (ranCount % 1000 == 0) { @@ -529,7 +529,7 @@ void CellDb::start_up() { // auto rock_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); - for (int i = 0; i < 10; i++) { + for (int i = 0; i < THREAD_COUNTS; i++) { cell_db_read_[i] = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_, rocks_db_); } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 83a30d49c..aeb92dd3a 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -37,6 +37,10 @@ namespace ton { namespace validator { +const int THREAD_COUNTS = 20; + +int GetRandom(); + class RootDb; class CellDb; @@ -183,7 +187,7 @@ class CellDb : public CellDbBase { td::Ref opts_; td::actor::ActorOwn cell_db_; - td::actor::ActorOwn cell_db_read_[10]; + td::actor::ActorOwn cell_db_read_[THREAD_COUNTS]; std::shared_ptr rocks_db_; std::unique_ptr boc_; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 95b900313..be98946b7 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -250,14 +250,6 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } -int getRandom2(){ - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution<> distr(0, 9); - int random_number = distr(gen); - return random_number; -} - void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise, std::uint64_t counter_) { LOG(INFO) << "get_block_state, counter" << counter_ << ", 1"; if (handle->inited_state_boc()) { @@ -284,7 +276,7 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise(td::RocksDb::open(path, std::move(db_options)).move_as_ok()); cell_db_ = td::actor::create_actor("celldb", actor_id(this), path, opts_, rock_db); - for (int i = 0; i < 10; i++){ + for (int i = 0; i < THREAD_COUNTS; i++){ cell_db_read_[i] = td::actor::create_actor("celldb", actor_id(this), path, opts_, rock_db); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index b2e313175..1bee9e424 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -145,7 +145,7 @@ class RootDb : public Db { td::Ref opts_; td::actor::ActorOwn cell_db_; - td::actor::ActorOwn cell_db_read_[10]; + td::actor::ActorOwn cell_db_read_[THREAD_COUNTS]; td::actor::ActorOwn state_db_; td::actor::ActorOwn static_files_db_; td::actor::ActorOwn archive_db_; From c73ff176763b90b366fbe7989cdafc5c83f3fb8c Mon Sep 17 00:00:00 2001 From: jianguo Date: Tue, 22 Oct 2024 17:48:43 +0800 Subject: [PATCH 101/101] update --- validator/db/celldb.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index aeb92dd3a..0f88066a4 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -37,7 +37,7 @@ namespace ton { namespace validator { -const int THREAD_COUNTS = 20; +const int THREAD_COUNTS = 10; int GetRandom();