From 4ba7216bc9e2624683d52cf7501a8b8eb1eab32d Mon Sep 17 00:00:00 2001 From: Natasha Sehgal Date: Thu, 19 Dec 2024 12:54:42 +0530 Subject: [PATCH 1/6] Add session properties for scaled table scan configs --- .../NativeWorkerSessionPropertyProvider.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java b/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java index 126a79382782..675380cf430c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java @@ -73,6 +73,8 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_SCALED_WRITER_MAX_PARTITIONS_PER_WRITER = "native_scaled_writer_max_partitions_per_writer"; public static final String NATIVE_SCALED_WRITER_MIN_PARTITION_PROCESSED_BYTES_REBALANCE_THRESHOLD = "native_scaled_writer_min_partition_processed_bytes_rebalance_threshold"; public static final String NATIVE_SCALED_WRITER_MIN_PROCESSED_BYTES_REBALANCE_THRESHOLD = "native_scaled_writer_min_processed_bytes_rebalance_threshold"; + public static final String NATIVE_TABLE_SCAN_SCALED_PROCESSING_ENABLED = "native_table_scan_scaled_processing_enabled"; + public static final String NATIVE_TABLE_SCAN_SCALE_UP_MEMORY_USAGE_RATIO = "native_table_scan_scale_up_memory_usage_ratio"; private final List> sessionProperties; @Inject @@ -311,6 +313,20 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "Minimum amount of data processed by all the logical table partitions " + "to trigger skewed partition rebalancing by scale writer exchange.", 256L << 20, + !nativeExecution), + booleanProperty( + NATIVE_TABLE_SCAN_SCALED_PROCESSING_ENABLED, + "If set to true, enables the scaled table scan processing.", + false, + !nativeExecution), + doubleProperty( + NATIVE_TABLE_SCAN_SCALE_UP_MEMORY_USAGE_RATIO, + "The query memory usage ratio used by scan controller to decide if it can " + + "increase the number of running scan threads. When the query memory usage " + + "is below this ratio, the scan controller keeps increasing the running scan " + + "thread for scale up, and stop once exceeds this ratio. The value is in the " + + "range of [0, 1].", + 0.7, !nativeExecution)); } From 2daa462083cf4adfb157eddc45b01cd6786c11b5 Mon Sep 17 00:00:00 2001 From: Natasha Sehgal Date: Fri, 20 Dec 2024 13:51:10 +0530 Subject: [PATCH 2/6] add documentation for new table scan session properties --- .../sphinx/presto_cpp/properties-session.rst | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index 6aa098e11bcd..cd2a7abff7a7 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -419,3 +419,24 @@ writer scaling if it is detected as overloaded by scale writer exchange. Minimum amount of data processed by all the logical table partitions to trigger skewed partition rebalancing by scale writer exchange. + +``native_table_scan_scaled_processing_enabled`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +Enables scaled processing for table scans. +When enabled, Presto will attempt to scale up table scans to improve performance. + +``native_table_scan_scale_up_memory_usage_ratio`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``double`` +* **Minimum value:** ``0`` +* **Maximum value:** ``1`` +* **Default value:** ``0.7`` + +Controls the ratio of available memory that can be used for scaling up table scans. +A higher value allows more memory to be allocated for scaling up table scans, +while a lower value limits the amount of memory used. From 5e7d440bd12ba0e8f9238a80e84ef57d1ee03685 Mon Sep 17 00:00:00 2001 From: Natasha Sehgal Date: Thu, 2 Jan 2025 07:09:19 -0800 Subject: [PATCH 3/6] update property styling --- presto-docs/src/main/sphinx/presto_cpp/properties-session.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index cd2a7abff7a7..82c9b14a1ff4 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -430,7 +430,7 @@ Enables scaled processing for table scans. When enabled, Presto will attempt to scale up table scans to improve performance. ``native_table_scan_scale_up_memory_usage_ratio`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** ``double`` * **Minimum value:** ``0`` From 7b769ed366fc0514a8a53c2dc438d80cc80281f9 Mon Sep 17 00:00:00 2001 From: Natasha Sehgal Date: Thu, 2 Jan 2025 12:08:19 -0800 Subject: [PATCH 4/6] Improve Table Scan property names --- presto-docs/src/main/sphinx/presto_cpp/properties-session.rst | 3 +-- .../NativeWorkerSessionPropertyProvider.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index 82c9b14a1ff4..02d6b417ac7c 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -426,8 +426,7 @@ trigger skewed partition rebalancing by scale writer exchange. * **Type:** ``boolean`` * **Default value:** ``false`` -Enables scaled processing for table scans. -When enabled, Presto will attempt to scale up table scans to improve performance. +If set to ``true``, enables scaling the table scan concurrency on each worker. ``native_table_scan_scale_up_memory_usage_ratio`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java b/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java index 675380cf430c..4f65719bb1c1 100644 --- a/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java @@ -316,7 +316,7 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) !nativeExecution), booleanProperty( NATIVE_TABLE_SCAN_SCALED_PROCESSING_ENABLED, - "If set to true, enables the scaled table scan processing.", + "If set to true, enables scaling the table scan concurrency on each worker.", false, !nativeExecution), doubleProperty( From 8963addd5259fe4ccd969e57bd7a692d142429c1 Mon Sep 17 00:00:00 2001 From: Natasha Sehgal Date: Fri, 20 Dec 2024 14:01:37 +0530 Subject: [PATCH 5/6] Add table scan session properties to native workers --- .../presto_cpp/main/SessionProperties.cpp | 17 +++++++++++++++++ .../presto_cpp/main/SessionProperties.h | 9 +++++++++ .../main/tests/SessionPropertiesTest.cpp | 8 ++++++-- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index fab89fda0bf7..eec0a28c21e4 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -443,6 +443,23 @@ SessionProperties::SessionProperties() { false, QueryConfig::kScaleWriterMinProcessedBytesRebalanceThreshold, std::to_string(c.scaleWriterMinProcessedBytesRebalanceThreshold())); + + addSessionProperty( + kTableScanScaledProcessingEnabled, + "If set to true, enables scaled processing for table scans.", + BOOLEAN(), + false, + QueryConfig::kTableScanScaledProcessingEnabled, + std::to_string(c.tableScanScaledProcessingEnabled())); + + addSessionProperty( + kTableScanScaleUpMemoryUsageRatio, + "Controls the ratio of available memory that can be used for scaling up table scans. " + "The value is in the range of (0, 1].", + DOUBLE(), + false, + QueryConfig::kTableScanScaleUpMemoryUsageRatio, + std::to_string(c.tableScanScaleUpMemoryUsageRatio())); } const std::unordered_map>& diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index 6d77bb8026f3..ec3ab1cc199d 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -287,6 +287,15 @@ class SessionProperties { static constexpr const char* kShuffleCompressionEnabled = "exchange_compression"; + /// If set to true, enables scaled processing for table scans. + static constexpr const char* kTableScanScaledProcessingEnabled = + "native_table_scan_scaled_processing_enabled"; + + /// Controls the ratio of available memory that can be used for scaling up + /// table scans. The value is in the range of (0, 1]. + static constexpr const char* kTableScanScaleUpMemoryUsageRatio = + "native_table_scan_scale_up_memory_usage_ratio"; + SessionProperties(); const std::unordered_map>& diff --git a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp index 6b6ad6533466..96f9f04c4bc2 100644 --- a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp @@ -31,7 +31,9 @@ TEST_F(SessionPropertiesTest, validateMapping) { SessionProperties::kScaleWriterMaxPartitionsPerWriter, SessionProperties:: kScaleWriterMinPartitionProcessedBytesRebalanceThreshold, - SessionProperties::kScaleWriterMinProcessedBytesRebalanceThreshold}; + SessionProperties::kScaleWriterMinProcessedBytesRebalanceThreshold, + SessionProperties::kTableScanScaledProcessingEnabled, + SessionProperties::kTableScanScaleUpMemoryUsageRatio}; const std::vector veloxConfigNames = { core::QueryConfig::kAdjustTimestampToTimezone, core::QueryConfig::kDriverCpuTimeSliceLimitMs, @@ -40,7 +42,9 @@ TEST_F(SessionPropertiesTest, validateMapping) { core::QueryConfig::kScaleWriterMaxPartitionsPerWriter, core::QueryConfig:: kScaleWriterMinPartitionProcessedBytesRebalanceThreshold, - core::QueryConfig::kScaleWriterMinProcessedBytesRebalanceThreshold}; + core::QueryConfig::kScaleWriterMinProcessedBytesRebalanceThreshold, + core::QueryConfig::kTableScanScaledProcessingEnabled, + core::QueryConfig::kTableScanScaleUpMemoryUsageRatio}; auto sessionProperties = SessionProperties().getSessionProperties(); const auto len = names.size(); for (auto i = 0; i < len; i++) { From 7530bc6aa74df21ee6c68c581123dce3253c9685 Mon Sep 17 00:00:00 2001 From: Natasha Sehgal Date: Fri, 3 Jan 2025 09:17:51 -0800 Subject: [PATCH 6/6] no test --- .../presto_cpp/main/tests/SessionPropertiesTest.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp index 96f9f04c4bc2..6b6ad6533466 100644 --- a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp @@ -31,9 +31,7 @@ TEST_F(SessionPropertiesTest, validateMapping) { SessionProperties::kScaleWriterMaxPartitionsPerWriter, SessionProperties:: kScaleWriterMinPartitionProcessedBytesRebalanceThreshold, - SessionProperties::kScaleWriterMinProcessedBytesRebalanceThreshold, - SessionProperties::kTableScanScaledProcessingEnabled, - SessionProperties::kTableScanScaleUpMemoryUsageRatio}; + SessionProperties::kScaleWriterMinProcessedBytesRebalanceThreshold}; const std::vector veloxConfigNames = { core::QueryConfig::kAdjustTimestampToTimezone, core::QueryConfig::kDriverCpuTimeSliceLimitMs, @@ -42,9 +40,7 @@ TEST_F(SessionPropertiesTest, validateMapping) { core::QueryConfig::kScaleWriterMaxPartitionsPerWriter, core::QueryConfig:: kScaleWriterMinPartitionProcessedBytesRebalanceThreshold, - core::QueryConfig::kScaleWriterMinProcessedBytesRebalanceThreshold, - core::QueryConfig::kTableScanScaledProcessingEnabled, - core::QueryConfig::kTableScanScaleUpMemoryUsageRatio}; + core::QueryConfig::kScaleWriterMinProcessedBytesRebalanceThreshold}; auto sessionProperties = SessionProperties().getSessionProperties(); const auto len = names.size(); for (auto i = 0; i < len; i++) {