From dddd55f0dad51580c60ccbebb0c687f169f374a3 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Thu, 30 May 2024 15:33:52 +0800 Subject: [PATCH] [fix](segment cache) estimate momory consumed by segment The memory consumed in segment cache is 0 after https://github.com/apache/doris/pull/35432/files. The pr also tracks memory usage of column readers. --- be/src/olap/primary_key_index.cpp | 2 ++ .../olap/rowset/segment_v2/column_reader.cpp | 8 +++++++- be/src/olap/rowset/segment_v2/column_reader.h | 2 +- .../segment_v2/indexed_column_reader.cpp | 8 ++++++++ .../rowset/segment_v2/indexed_column_reader.h | 2 ++ .../rowset/segment_v2/ordinal_page_index.cpp | 13 ++++++++++++ .../rowset/segment_v2/ordinal_page_index.h | 2 ++ be/src/olap/rowset/segment_v2/segment.cpp | 20 ++++++++++++++----- be/src/olap/rowset/segment_v2/segment.h | 1 + .../olap/rowset/segment_v2/zone_map_index.cpp | 11 ++++++++++ .../olap/rowset/segment_v2/zone_map_index.h | 2 ++ be/src/olap/short_key_index.cpp | 14 +++++++++++++ be/src/olap/short_key_index.h | 1 + 13 files changed, 79 insertions(+), 7 deletions(-) diff --git a/be/src/olap/primary_key_index.cpp b/be/src/olap/primary_key_index.cpp index 6d1b1772a91f9e..6ea4bc30d0a016 100644 --- a/be/src/olap/primary_key_index.cpp +++ b/be/src/olap/primary_key_index.cpp @@ -32,6 +32,8 @@ namespace doris { +static bvar::Adder g_primary_key_index_memory_bytes("doris_primary_key_index_memory_bytes"); + Status PrimaryKeyIndexBuilder::init() { // TODO(liaoxin) using the column type directly if there's only one column in unique key columns const auto* type_info = get_scalar_type_info(); diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 4e005c43a7c11a..4498e9631f9077 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -77,6 +77,8 @@ namespace doris { namespace segment_v2 { +static bvar::Adder g_column_reader_memory_bytes("doris_column_reader_memory_bytes"); + Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, uint64_t num_rows, const io::FileReaderSPtr& file_reader, std::unique_ptr* reader) { @@ -206,9 +208,13 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& _meta_is_nullable = meta.is_nullable(); _meta_dict_page = meta.dict_page(); _meta_compression = meta.compression(); + + g_column_reader_memory_bytes << sizeof(*this); } -ColumnReader::~ColumnReader() = default; +ColumnReader::~ColumnReader() { + g_column_reader_memory_bytes << -sizeof(*this); +} Status ColumnReader::init(const ColumnMetaPB* meta) { _type_info = get_type_info(meta); diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index ae2b29cf983531..94494b4d23c8bf 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -117,7 +117,7 @@ class ColumnReader { enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING, ALL_DICT_ENCODING }; - ~ColumnReader(); + virtual ~ColumnReader(); // create a new column iterator. Client should delete returned iterator Status new_iterator(ColumnIterator** iterator); diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp index b5f974fb8a0214..59251b5595dd07 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp @@ -56,6 +56,8 @@ static bvar::Adder g_index_reader_pk_pages("doris_pk", "index_reader_p static bvar::PerSecond> g_index_reader_pk_bytes_per_second( "doris_pk", "index_reader_pk_pages_per_second", &g_index_reader_pk_pages, 60); +static bvar::Adder g_index_reader_memory_bytes("doris_index_reader_memory_bytes"); + using strings::Substitute; Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) { @@ -91,6 +93,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) { } } _num_values = _meta.num_values(); + + g_index_reader_memory_bytes << sizeof(*this); return Status::OK(); } @@ -134,6 +138,10 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle, return st; } +IndexedColumnReader::~IndexedColumnReader() { + g_index_reader_memory_bytes << -sizeof(*this); +} + /////////////////////////////////////////////////////////////////////////////// Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) { diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h index ef6e2754609e71..d156643a21c11d 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h @@ -51,6 +51,8 @@ class IndexedColumnReader { explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta) : _file_reader(std::move(file_reader)), _meta(meta) {} + ~IndexedColumnReader(); + Status load(bool use_page_cache, bool kept_in_memory); // read a page specified by `pp' from `file' into `handle' diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp index 5761cc42b9d6c7..fc9a676774d559 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp @@ -29,9 +29,13 @@ #include "olap/olap_common.h" #include "olap/rowset/segment_v2/page_handle.h" #include "olap/rowset/segment_v2/page_io.h" +#include "ordinal_page_index.h" #include "util/slice.h" namespace doris { + +static bvar::Adder g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes"); + namespace segment_v2 { void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer& data_pp) { @@ -122,6 +126,9 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory, _pages[i] = reader.get_value(i); } _ordinals[_num_pages] = _num_values; + + g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() * sizeof(ordinal_t) + + _pages.size() * sizeof(PagePointer) + sizeof(OrdinalIndexReader); return Status::OK(); } @@ -146,5 +153,11 @@ OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal return OrdinalPageIndexIterator(this, left); } +OrdinalIndexReader::~OrdinalIndexReader() { + g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) - + _pages.size() * sizeof(PagePointer) - sizeof(OrdinalIndexReader); +} + + } // namespace segment_v2 } // namespace doris diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h b/be/src/olap/rowset/segment_v2/ordinal_page_index.h index 77f0b8518163a0..adf342ffeb6022 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h @@ -72,6 +72,8 @@ class OrdinalIndexReader { _meta_pb.reset(new OrdinalIndexPB(meta_pb)); } + ~OrdinalIndexReader(); + // load and parse the index page into memory Status load(bool use_page_cache, bool kept_in_memory); diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index dc934ba5fce025..65153f2f1622d5 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -73,7 +73,7 @@ #include "vec/olap/vgeneric_iterators.h" namespace doris::segment_v2 { -bvar::Adder g_total_segment_num("doris_total_segment_num"); +static bvar::Adder g_total_segment_num("doris_total_segment_num"); class InvertedIndexIterator; Status Segment::open(io::FileSystemSPtr fs, const std::string& path, uint32_t segment_id, @@ -112,6 +112,17 @@ Status Segment::_open() { // DCHECK(footer.has_short_key_index_page()); _sk_index_page = _footer_pb->short_key_index_page(); _num_rows = _footer_pb->num_rows(); + + // An estimated memory usage of a segment + _meta_mem_usage += _footer_pb->ByteSizeLong(); + _meta_mem_usage += sizeof(*this); + _meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader; + + // 1024 comes from SegmentWriterOptions + _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4); + // 0.01 comes from PrimaryKeyIndexBuilder::init + _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8; + return Status::OK(); } @@ -298,7 +309,7 @@ Status Segment::_load_pk_bloom_filter() { auto status = [this]() { return _load_pk_bf_once.call([this] { RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader, *_pk_index_meta)); - _meta_mem_usage += _pk_index_reader->get_bf_memory_size(); + // _meta_mem_usage += _pk_index_reader->get_bf_memory_size(); return Status::OK(); }); }(); @@ -335,7 +346,7 @@ Status Segment::_load_index_impl() { if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta != nullptr) { _pk_index_reader.reset(new PrimaryKeyIndexReader()); RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader, *_pk_index_meta)); - _meta_mem_usage += _pk_index_reader->get_memory_size(); + // _meta_mem_usage += _pk_index_reader->get_memory_size(); return Status::OK(); } else { // read and parse short key index page @@ -357,7 +368,7 @@ Status Segment::_load_index_impl() { DCHECK_EQ(footer.type(), SHORT_KEY_PAGE); DCHECK(footer.has_short_key_page_footer()); - _meta_mem_usage += body.get_size(); + // _meta_mem_usage += body.get_size(); _sk_index_decoder.reset(new ShortKeyIndexDecoder); return _sk_index_decoder->parse(body, footer.short_key_page_footer()); } @@ -427,7 +438,6 @@ Status Segment::_create_column_readers(const SegmentFooterPB& footer) { RETURN_IF_ERROR(ColumnReader::create(opts, footer.columns(iter->second), footer.num_rows(), _file_reader, &reader)); _column_readers.emplace(column.unique_id(), std::move(reader)); - _meta_mem_usage += config::estimated_mem_per_column_reader; } // init by column path diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index 8cb4502590819b..e6c9a169dc80b7 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -257,6 +257,7 @@ class Segment : public std::enable_shared_from_this { // used to hold short key index page in memory PageHandle _sk_index_handle; // short key index decoder + // all content is in memory std::unique_ptr _sk_index_decoder; // primary key index reader std::unique_ptr _pk_index_reader; diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index e232e448ff6df3..2caecd217fe475 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -39,6 +39,8 @@ namespace doris { struct uint24_t; +static bvar::Adder g_zone_map_memory_bytes("doris_zone_map_memory_bytes"); + namespace segment_v2 { template @@ -173,9 +175,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, return Status::Corruption("Failed to parse zone map"); } } + + g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) * _page_zone_maps.size() + + sizeof(IndexedColumnMetaPB); + return Status::OK(); } +ZoneMapIndexReader::~ZoneMapIndexReader() { + // Maybe wrong due to load failures. + g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() - + sizeof(IndexedColumnMetaPB); +} #define APPLY_FOR_PRIMITITYPE(M) \ M(TYPE_TINYINT) \ M(TYPE_SMALLINT) \ diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h b/be/src/olap/rowset/segment_v2/zone_map_index.h index eeb87eb7da772f..923bd2c2046da7 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.h +++ b/be/src/olap/rowset/segment_v2/zone_map_index.h @@ -151,6 +151,8 @@ class ZoneMapIndexReader { _page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps)); } + virtual ~ZoneMapIndexReader(); + // load all page zone maps into memory Status load(bool use_page_cache, bool kept_in_memory); diff --git a/be/src/olap/short_key_index.cpp b/be/src/olap/short_key_index.cpp index 69622cee454fda..868089867059cd 100644 --- a/be/src/olap/short_key_index.cpp +++ b/be/src/olap/short_key_index.cpp @@ -22,12 +22,15 @@ #include #include "gutil/strings/substitute.h" +#include "short_key_index.h" #include "util/coding.h" using strings::Substitute; namespace doris { +static bvar::Adder g_short_key_index_memory_bytes("doris_short_key_index_memory_bytes"); + Status ShortKeyIndexBuilder::add_item(const Slice& key) { put_varint32(&_offset_buf, _key_buf.size()); _key_buf.append(key.data, key.size); @@ -85,7 +88,18 @@ Status ShortKeyIndexDecoder::parse(const Slice& body, const segment_v2::ShortKey return Status::Corruption("Still has data after parse all key offset"); } _parsed = true; + + g_short_key_index_memory_bytes << sizeof(_footer) + _key_data.size + + _offsets.size() * sizeof(uint32_t) + sizeof(*this); + return Status::OK(); } +ShortKeyIndexDecoder::~ShortKeyIndexDecoder() { + if (_parsed) { + g_short_key_index_memory_bytes << -sizeof(_footer) - _key_data.size - + _offsets.size() * sizeof(uint32_t) - sizeof(*this); + } +} + } // namespace doris diff --git a/be/src/olap/short_key_index.h b/be/src/olap/short_key_index.h index 79303a3edbb50d..93f4b614908476 100644 --- a/be/src/olap/short_key_index.h +++ b/be/src/olap/short_key_index.h @@ -135,6 +135,7 @@ class ShortKeyIndexIterator { class ShortKeyIndexDecoder { public: ShortKeyIndexDecoder() : _parsed(false) {} + virtual ~ShortKeyIndexDecoder(); // client should assure that body is available when this class is used Status parse(const Slice& body, const segment_v2::ShortKeyFooterPB& footer);