Skip to content

Commit

Permalink
[native] Add counter to track effectiveness of memory pushback
Browse files Browse the repository at this point in the history
Adds a new counter `presto_cpp.memory_pushback_reduction_bytes`
that keeps track of the actual used memory reduction for every
memory pushback attempt.
  • Loading branch information
bikramSingh91 authored and SthuthiGhosh9400 committed Oct 15, 2024
1 parent 938e636 commit b21f816
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,12 @@ void PeriodicMemoryChecker::pushbackMemory() {
}
RECORD_HISTOGRAM_METRIC_VALUE(
kCounterMemoryPushbackLatencyMs, latencyUs / 1000);
LOG(INFO) << "Shrunk " << velox::succinctBytes(freedBytes);
const auto actualFreedBytes = std::max<int64_t>(
0, static_cast<int64_t>(currentMemBytes) - systemUsedMemoryBytes());
RECORD_HISTOGRAM_METRIC_VALUE(
kCounterMemoryPushbackLatencyMs, actualFreedBytes);
LOG(INFO) << "Memory pushback shrunk " << velox::succinctBytes(freedBytes)
<< " Effective bytes shrunk: "
<< velox::succinctBytes(actualFreedBytes);
}
} // namespace facebook::presto
9 changes: 9 additions & 0 deletions presto-native-execution/presto_cpp/main/common/Counters.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ void registerPrestoMetrics() {
DEFINE_METRIC(kCounterMemoryPushbackCount, facebook::velox::StatType::COUNT);
DEFINE_HISTOGRAM_METRIC(
kCounterMemoryPushbackLatencyMs, 10'000, 0, 100'000, 50, 90, 99, 100);
DEFINE_HISTOGRAM_METRIC(
kCounterMemoryPushbackLatencyMs,
100l * 1024 * 1024, // 100MB
0,
15l * 1024 * 1024 * 1024, // 15GB
50,
90,
99,
100);

// NOTE: Metrics type exporting for file handle cache counters are in
// PeriodicTaskManager because they have dynamic names. The following counters
Expand Down
5 changes: 5 additions & 0 deletions presto-native-execution/presto_cpp/main/common/Counters.h
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,9 @@ constexpr folly::StringPiece kCounterMemoryPushbackCount{
/// reports P50, P90, P99, and P100.
constexpr folly::StringPiece kCounterMemoryPushbackLatencyMs{
"presto_cpp.memory_pushback_latency_ms"};
/// Distribution of reduction in memory usage achieved by each memory pushback
/// attempt. This is to gauge its effectiveness. In range of [0, 15GB] with 150
/// buckets and reports P50, P90, P99, and P100.
constexpr folly::StringPiece kCounterMemoryPushbackReductionBytes{
"presto_cpp.memory_pushback_reduction_bytes"};
} // namespace facebook::presto

0 comments on commit b21f816

Please sign in to comment.