Skip to content

Commit

Permalink
Fixes clang-format usage (#929)
Browse files Browse the repository at this point in the history 
* up formatting


---------

Co-authored-by: kst-morozov <kst-morozov@ydb.tech>
  • Loading branch information
@k-morozov
k-morozov and kst-morozov authored Jan 10, 2024
1 parent ea9fe63 commit f2c7443
Show file tree
Hide file tree
Showing 5 changed files with 69 additions and 58 deletions.
60 changes: 36 additions & 24 deletions ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/core/kqp/ut/common/columnshard.h>
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/core/testlib/common_helper.h>
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>

namespace NKikimr {
namespace NKqp {
Expand All @@ -15,12 +15,11 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
constexpr size_t size_single_table = 13152;

const TVector<TTestHelper::TColumnSchema> schema = {
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8),
TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32)
};
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8),
TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32)};

class TOlapStatsController: public NYDBTest::NColumnShard::TController {
class TOlapStatsController : public NYDBTest::NColumnShard::TController {
public:
TDuration GetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override {
return TDuration::MilliSeconds(10);
Expand All @@ -32,7 +31,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {

Y_UNIT_TEST(AddRowsTableStandalone) {
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<TOlapStatsController>();

TKikimrSettings runnerSettings;
runnerSettings.WithSampleTables = false;

Expand All @@ -44,8 +43,8 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
testHelper.CreateTable(testTable);
{
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
for(size_t i=0; i<inserted_rows; i++) {

for (size_t i = 0; i < inserted_rows; i++) {
tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull();
}

Expand Down Expand Up @@ -78,21 +77,25 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
testTableStore.SetName("/Root/TableStoreTest").SetPrimaryKey({"id"}).SetSchema(schema);
testHelper.CreateTable(testTableStore);
TTestHelper::TColumnTable testTable;
testTable.SetName("/Root/TableStoreTest/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
testTable.SetName("/Root/TableStoreTest/ColumnTableTest")
.SetPrimaryKey({"id"})
.SetSharding({"id"})
.SetSchema(schema);
testHelper.CreateTable(testTable);

{
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
for(size_t i=0; i<inserted_rows; i++) {
for (size_t i = 0; i < inserted_rows; i++) {
tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull();
}
testHelper.InsertData(testTable, tableInserter);
}

Sleep(TDuration::Seconds(1));

auto settings = TDescribeTableSettings().WithTableStatistics(true);
auto describeResult = testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest", settings).GetValueSync();
auto describeResult =
testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest", settings).GetValueSync();

UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());

Expand All @@ -117,23 +120,32 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {

Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).SetPriority(NActors::NLog::PRI_DEBUG).Initialize();

for(size_t t=0; t<tables_in_store; t++) {
for (size_t t = 0; t < tables_in_store; t++) {
TTestHelper::TColumnTable testTable;
testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t)).SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t))
.SetPrimaryKey({"id"})
.SetSharding({"id"})
.SetSchema(schema);
testHelper.CreateTable(testTable);

TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
for(size_t i=0; i < t+ inserted_rows; i++) {
tableInserter.AddRow().Add(i + t * tables_in_store).Add("test_res_" + std::to_string(i + t * tables_in_store)).AddNull();
for (size_t i = 0; i < t + inserted_rows; i++) {
tableInserter.AddRow()
.Add(i + t * tables_in_store)
.Add("test_res_" + std::to_string(i + t * tables_in_store))
.AddNull();
}
testHelper.InsertData(testTable, tableInserter);;
testHelper.InsertData(testTable, tableInserter);
}

Sleep(TDuration::Seconds(20));

auto settings = TDescribeTableSettings().WithTableStatistics(true);
for(size_t t=0; t<tables_in_store; t++) {
auto describeResult = testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t), settings).GetValueSync();
for (size_t t = 0; t < tables_in_store; t++) {
auto describeResult =
testHelper.GetSession()
.DescribeTable("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t), settings)
.GetValueSync();
UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
const auto& description = describeResult.GetTableDescription();

Expand All @@ -142,5 +154,5 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
}
}

} // namespace NKqp
} // namespace NKikimr
} // namespace NKqp
} // namespace NKikimr
22 changes: 12 additions & 10 deletions ydb/core/tx/columnshard/columnshard.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,13 +293,14 @@ void TColumnShard::UpdateResourceMetrics(const TActorContext& ctx, const TUsage&
metrics->TryUpdate(ctx);
}

void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats * tabletStats) {
void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats,
::NKikimrTableStats::TTableStats* tabletStats) {
NOlap::TSnapshot lastIndexUpdate = TablesManager.GetPrimaryIndexSafe().LastUpdate();
auto activeIndexStats = indexStats.Active(); // data stats excluding inactive and evicted
auto activeIndexStats = indexStats.Active(); // data stats excluding inactive and evicted

if (activeIndexStats.Rows < 0 || activeIndexStats.Bytes < 0) {
LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows
<< " Bytes: " << activeIndexStats.Bytes << TabletID());
LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows << " Bytes: " << activeIndexStats.Bytes
<< TabletID());

activeIndexStats.Rows = (activeIndexStats.Rows < 0) ? 0 : activeIndexStats.Rows;
activeIndexStats.Bytes = (activeIndexStats.Bytes < 0) ? 0 : activeIndexStats.Bytes;
Expand All @@ -309,7 +310,7 @@ void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats, :
tabletStats->SetDataSize(activeIndexStats.Bytes + TabletCounters->Simple()[COUNTER_COMMITTED_BYTES].Get());

// TODO: we need row/dataSize counters for evicted data (managed by tablet but stored outside)
//tabletStats->SetIndexSize(); // TODO: calc size of internal tables
// tabletStats->SetIndexSize(); // TODO: calc size of internal tables

tabletStats->SetLastAccessTime(LastAccessTime.MilliSeconds());
tabletStats->SetLastUpdateTime(lastIndexUpdate.GetPlanStep());
Expand All @@ -330,7 +331,7 @@ void TColumnShard::FillTxTableStats(::NKikimrTableStats::TTableStats* tableStats
}

void TColumnShard::FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
ev->Record.SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
ev->Record.SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
ev->Record.SetGeneration(Executor()->Generation());
ev->Record.SetRound(StatsReportRound++);
ev->Record.SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId);
Expand All @@ -346,13 +347,14 @@ void TColumnShard::FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDa
}
}

void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
void TColumnShard::FillColumnTableStats(const TActorContext& ctx,
std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
if (!TablesManager.HasPrimaryIndex()) {
return;
}
const auto& tablesIndexStats = TablesManager.MutablePrimaryIndex().GetStats();
LOG_S_DEBUG("There are stats for " << tablesIndexStats.size() << " tables");
for(const auto& [tableLocalID, columnStats] : tablesIndexStats) {
for (const auto& [tableLocalID, columnStats] : tablesIndexStats) {
if (!columnStats) {
LOG_S_ERROR("SendPeriodicStats: empty stats");
continue;
Expand All @@ -362,7 +364,7 @@ void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_pt
periodicTableStats->SetDatashardId(TabletID());
periodicTableStats->SetTableLocalId(tableLocalID);

periodicTableStats->SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
periodicTableStats->SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready
periodicTableStats->SetGeneration(Executor()->Generation());
periodicTableStats->SetRound(StatsReportRound++);
periodicTableStats->SetNodeId(ctx.ExecutorThread.ActorSystem->NodeId);
Expand Down Expand Up @@ -411,4 +413,4 @@ void TColumnShard::SendPeriodicStats() {
NTabletPipe::SendData(ctx, StatsReportPipe, ev.release());
}

}
} // namespace NKikimr::NColumnShard
3 changes: 2 additions & 1 deletion ydb/core/tx/columnshard/columnshard_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,11 +488,12 @@ class TColumnShard
void SendPeriodicStats();
void FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev);
void FillColumnTableStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev);
void ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats * tabletStats);
void ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats* tabletStats);
void FillTxTableStats(::NKikimrTableStats::TTableStats* tableStats) const;

static TDuration GetControllerPeriodicWakeupActivationPeriod();
static TDuration GetControllerStatsReportInterval();

public:
const std::shared_ptr<NOlap::IStoragesManager>& GetStoragesManager() const {
return StoragesManager;
Expand Down
40 changes: 18 additions & 22 deletions ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class TTxStoreTableStats: public TTxStoreStats<TEvDataShard::TEvPeriodicTableSta
bool PersistSingleStats(const TPathId& pathId, const TStatsQueue<TEvDataShard::TEvPeriodicTableStats>::TItem& item, TTransactionContext& txc, const TActorContext& ctx) override;
void ScheduleNextBatch(const TActorContext& ctx) override;

template<typename T>
template <typename T>
TPartitionStats PrepareStats(const TActorContext& ctx, const T& rec) const;
};

Expand Down Expand Up @@ -124,10 +124,8 @@ THolder<TProposeRequest> MergeRequest(
return std::move(request);
}

template<typename T>
TPartitionStats TTxStoreTableStats::PrepareStats(
const TActorContext& ctx, const T& rec) const {

template <typename T>
TPartitionStats TTxStoreTableStats::PrepareStats(const TActorContext& ctx, const T& rec) const {
const auto& tableStats = rec.GetTableStats();
const auto& tabletMetrics = rec.GetTabletMetrics();

Expand Down Expand Up @@ -183,7 +181,8 @@ TPartitionStats TTxStoreTableStats::PrepareStats(
}

bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
const TStatsQueueItem<TEvDataShard::TEvPeriodicTableStats>& item, NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
const TStatsQueueItem<TEvDataShard::TEvPeriodicTableStats>& item,
NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
const auto& rec = item.Ev->Get()->Record;
const auto datashardId = TTabletId(rec.GetDatashardId());

Expand All @@ -208,19 +207,17 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
TShardIdx shardIdx = Self->TabletIdToShardIdx[datashardId];

LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"TTxStoreTableStats.PersistSingleStats: main stats from"
<< " datashardId(TabletID)=" << datashardId
<< " maps to shardIdx: " << shardIdx
<< ", pathId: " << pathId
<< ", pathId map=" << Self->PathsById[pathId]->Name
<< ", is column=" << isColumnTable
<< ", is olap=" << isOlapStore);
"TTxStoreTableStats.PersistSingleStats: main stats from"
<< " datashardId(TabletID)=" << datashardId << " maps to shardIdx: " << shardIdx
<< ", pathId: " << pathId << ", pathId map=" << Self->PathsById[pathId]->Name
<< ", is column=" << isColumnTable << ", is olap=" << isOlapStore);

const TPartitionStats newStats = PrepareStats(ctx, rec);

LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"Add stats from shard with datashardId(TabletID)=" << datashardId << ", pathId " << pathId.LocalPathId
<< ": RowCount " << newStats.RowCount << ", DataSize " << newStats.DataSize);
"Add stats from shard with datashardId(TabletID)=" << datashardId << ", pathId " << pathId.LocalPathId
<< ": RowCount " << newStats.RowCount << ", DataSize "
<< newStats.DataSize);

NIceDb::TNiceDb db(txc.DB);

Expand Down Expand Up @@ -261,29 +258,28 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
updateSubdomainInfo = true;

const auto tables = rec.GetTables();
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"OLAP store contains " << tables.size() << " tables.");
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "OLAP store contains " << tables.size() << " tables.");

for(const auto& table : tables) {
for (const auto& table : tables) {
const TPartitionStats newTableStats = PrepareStats(ctx, table);

const TPathId tablePathId = TPathId(TOwnerId(pathId.OwnerId), TLocalPathId(table.GetTableLocalId()));

if (Self->ColumnTables.contains(tablePathId)) {
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"add stats for exists table with pathId=" << tablePathId);
"add stats for exists table with pathId=" << tablePathId);

auto columnTable = Self->ColumnTables.TakeVerified(tablePathId);
columnTable->UpdateTableStats(tablePathId, newTableStats);
} else {
LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"failed add stats for table with pathId=" << tablePathId);
"failed add stats for table with pathId=" << tablePathId);
}
}

} else if (isColumnTable) {
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "PersistSingleStats: ColumnTable rec.GetColumnTables() size="
<< rec.GetTables().size());
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"PersistSingleStats: ColumnTable rec.GetColumnTables() size=" << rec.GetTables().size());

auto columnTable = Self->ColumnTables.TakeVerified(pathId);
oldAggrStats = columnTable->GetStats().Aggregated;
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ void TPathDescriber::DescribeOlapStore(TPathId pathId, TPathElement::TPtr pathEl
}

void TPathDescriber::DescribeColumnTable(TPathId pathId, TPathElement::TPtr pathEl) {
const auto tableInfo = Self->ColumnTables.GetVerified(pathId);
const auto tableInfo = Self->ColumnTables.GetVerified(pathId);
Y_UNUSED(pathEl);

auto* pathDescription = Result->Record.MutablePathDescription();
Expand Down

0 comments on commit f2c7443

Please sign in to comment.