diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties.rst b/presto-docs/src/main/sphinx/presto_cpp/properties.rst index 5a704f622d5f..e5270c9ab08d 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties.rst @@ -85,6 +85,14 @@ alphabetical order. This property provides function signatures for built-in aggregation functions which are compatible with Velox. +``presto.default-namespace`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Default value:** ``presto.default`` + + Specifies the namespace prefix for native C++ functions. + Worker Properties ----------------- diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 3cedadc480f2..9f70c4a93f2a 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -238,6 +238,7 @@ void PrestoServer::run() { address_ = fmt::format("[{}]", address_); } nodeLocation_ = nodeConfig->nodeLocation(); + prestoBuiltinFunctionPrefix_ = systemConfig->prestoDefaultNamespacePrefix(); } catch (const velox::VeloxUserError& e) { PRESTO_STARTUP_LOG(ERROR) << "Failed to start server due to " << e.what(); exit(EXIT_FAILURE); @@ -1244,11 +1245,12 @@ void PrestoServer::registerCustomOperators() { } void PrestoServer::registerFunctions() { - static const std::string kPrestoDefaultPrefix{"presto.default."}; - velox::functions::prestosql::registerAllScalarFunctions(kPrestoDefaultPrefix); + velox::functions::prestosql::registerAllScalarFunctions( + prestoBuiltinFunctionPrefix_); velox::aggregate::prestosql::registerAllAggregateFunctions( - kPrestoDefaultPrefix); - velox::window::prestosql::registerAllWindowFunctions(kPrestoDefaultPrefix); + prestoBuiltinFunctionPrefix_); + velox::window::prestosql::registerAllWindowFunctions( + prestoBuiltinFunctionPrefix_); if (SystemConfig::instance()->registerTestFunctions()) { velox::functions::prestosql::registerAllScalarFunctions( "json.test_schema."); diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index 985a4161e1e1..1b8a50d7931a 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -288,6 +288,7 @@ class PrestoServer { std::string address_; std::string nodeLocation_; folly::SSLContextPtr sslContext_; + std::string prestoBuiltinFunctionPrefix_; }; } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index dfd37e507164..225f0257c880 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -238,6 +238,7 @@ SystemConfig::SystemConfig() { STR_PROP(kCacheVeloxTtlCheckInterval, "1h"), BOOL_PROP(kEnableRuntimeMetricsCollection, false), BOOL_PROP(kPlanValidatorFailOnNestedLoopJoin, false), + STR_PROP(kPrestoDefaultNamespacePrefix, "presto.default"), }; } @@ -753,6 +754,10 @@ bool SystemConfig::enableRuntimeMetricsCollection() const { return optionalProperty(kEnableRuntimeMetricsCollection).value(); } +std::string SystemConfig::prestoDefaultNamespacePrefix() const { + return optionalProperty(kPrestoDefaultNamespacePrefix).value().append("."); +} + NodeConfig::NodeConfig() { registeredProps_ = std::unordered_map>{ diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index 77cfa1feac6a..9b5ff2461bbc 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -647,6 +647,10 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kPlanValidatorFailOnNestedLoopJoin{ "velox-plan-validator-fail-on-nested-loop-join"}; + // Specifies the default Presto namespace prefix. + static constexpr std::string_view kPrestoDefaultNamespacePrefix{ + "presto.default-namespace"}; + SystemConfig(); virtual ~SystemConfig() = default; @@ -881,6 +885,7 @@ class SystemConfig : public ConfigBase { bool enableRuntimeMetricsCollection() const; bool prestoNativeSidecar() const; + std::string prestoDefaultNamespacePrefix() const; }; /// Provides access to node properties defined in node.properties file. diff --git a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt index 357259514732..1761f70856c8 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt @@ -18,8 +18,13 @@ target_link_libraries( presto_common_test presto_common presto_exception + velox_aggregates velox_exception velox_file + velox_functions_prestosql + velox_function_registry + velox_presto_types + velox_window ${RE2} GTest::gtest GTest::gtest_main) diff --git a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp index 9f3282a2308a..cf1fa1c97ede 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp @@ -12,17 +12,59 @@ * limitations under the License. */ #include +#include #include "presto_cpp/main/common/ConfigReader.h" #include "presto_cpp/main/common/Configs.h" #include "velox/common/base/Exceptions.h" #include "velox/common/base/tests/GTestUtils.h" #include "velox/common/file/File.h" #include "velox/common/file/FileSystems.h" +#include "velox/exec/Aggregate.h" +#include "velox/exec/Window.h" +#include "velox/functions/FunctionRegistry.h" +#include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" +#include "velox/functions/prestosql/registration/RegistrationFunctions.h" +#include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" namespace facebook::presto::test { using namespace velox; +namespace { + +template +bool validateDefaultNamespacePrefix( + const FunctionMap& functionMap, + const std::string& prestoDefaultNamespacePrefix) { + static const std::unordered_set kBlockList = { + "row_constructor", "in", "is_null"}; + + std::vector prestoDefaultNamespacePrefixParts; + folly::split( + '.', + prestoDefaultNamespacePrefix, + prestoDefaultNamespacePrefixParts, + true); + + for (const auto& [functionName, _] : functionMap) { + // Skip internal functions. They don't have any prefix. + if ((kBlockList.count(functionName) != 0) || + (functionName.find("$internal$") != std::string::npos)) { + continue; + } + + std::vector parts; + folly::split('.', functionName, parts, true); + if ((parts.size() != 3) || + (parts[0] != prestoDefaultNamespacePrefixParts[0]) || + (parts[1] != prestoDefaultNamespacePrefixParts[1])) { + return false; + } + } + return true; +} +} // namespace + class ConfigTest : public testing::Test { protected: void SetUp() override { @@ -310,4 +352,35 @@ TEST_F(ConfigTest, readConfigEnvVarTest) { unsetenv(kEmptyEnvVarName.c_str()); } +TEST_F(ConfigTest, prestoDefaultNamespacePrefix) { + SystemConfig config; + init( + config, + {{std::string(SystemConfig::kPrestoDefaultNamespacePrefix), + "presto.default"}}); + std::string prestoBuiltinFunctionPrefix = + config.prestoDefaultNamespacePrefix(); + + velox::functions::prestosql::registerAllScalarFunctions( + prestoBuiltinFunctionPrefix); + velox::aggregate::prestosql::registerAllAggregateFunctions( + prestoBuiltinFunctionPrefix); + velox::window::prestosql::registerAllWindowFunctions( + prestoBuiltinFunctionPrefix); + + // Get all registered scalar functions in Velox + auto scalarFunctions = getFunctionSignatures(); + ASSERT_TRUE(validateDefaultNamespacePrefix( + scalarFunctions, prestoBuiltinFunctionPrefix)); + + // Get all registered aggregate functions in Velox + auto aggregateFunctions = exec::aggregateFunctions().copy(); + ASSERT_TRUE(validateDefaultNamespacePrefix( + aggregateFunctions, prestoBuiltinFunctionPrefix)); + + // Get all registered window functions in Velox + auto windowFunctions = exec::windowFunctions(); + ASSERT_TRUE(validateDefaultNamespacePrefix( + windowFunctions, prestoBuiltinFunctionPrefix)); +} } // namespace facebook::presto::test diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp index 2c2b2a3c5ea0..159c53bafed2 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp @@ -14,6 +14,7 @@ #include "presto_cpp/main/types/PrestoToVeloxExpr.h" #include +#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/presto_protocol/Base64Util.h" #include "velox/common/base/Exceptions.h" #include "velox/functions/prestosql/types/JsonType.h" @@ -33,24 +34,41 @@ std::string toJsonString(const T& value) { } std::string mapScalarFunction(const std::string& name) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const std::unordered_map kFunctionNames = { // Operator overrides: com.facebook.presto.common.function.OperatorType - {"presto.default.$operator$add", "presto.default.plus"}, - {"presto.default.$operator$between", "presto.default.between"}, - {"presto.default.$operator$divide", "presto.default.divide"}, - {"presto.default.$operator$equal", "presto.default.eq"}, - {"presto.default.$operator$greater_than", "presto.default.gt"}, - {"presto.default.$operator$greater_than_or_equal", "presto.default.gte"}, + {"presto.default.$operator$add", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "plus")}, + {"presto.default.$operator$between", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "between")}, + {"presto.default.$operator$divide", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "divide")}, + {"presto.default.$operator$equal", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "eq")}, + {"presto.default.$operator$greater_than", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "gt")}, + {"presto.default.$operator$greater_than_or_equal", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "gte")}, {"presto.default.$operator$is_distinct_from", - "presto.default.distinct_from"}, - {"presto.default.$operator$less_than", "presto.default.lt"}, - {"presto.default.$operator$less_than_or_equal", "presto.default.lte"}, - {"presto.default.$operator$modulus", "presto.default.mod"}, - {"presto.default.$operator$multiply", "presto.default.multiply"}, - {"presto.default.$operator$negation", "presto.default.negate"}, - {"presto.default.$operator$not_equal", "presto.default.neq"}, - {"presto.default.$operator$subtract", "presto.default.minus"}, - {"presto.default.$operator$subscript", "presto.default.subscript"}, + formatDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "distinct_from")}, + {"presto.default.$operator$less_than", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "lt")}, + {"presto.default.$operator$less_than_or_equal", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "lte")}, + {"presto.default.$operator$modulus", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "mod")}, + {"presto.default.$operator$multiply", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "muliply")}, + {"presto.default.$operator$negation", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "negate")}, + {"presto.default.$operator$not_equal", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "neq")}, + {"presto.default.$operator$subtract", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "minus")}, + {"presto.default.$operator$subscript", + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "subscript")}, // Special form function overrides. {"presto.default.in", "in"}, }; @@ -66,11 +84,15 @@ std::string mapScalarFunction(const std::string& name) { } std::string mapAggregateOrWindowFunction(const std::string& name) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const std::unordered_map kFunctionNames = { {"presto.default.$internal$max_data_size_for_stats", - "presto.default.max_data_size_for_stats"}, + formatDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "max_data_size_for_stats")}, {"presto.default.$internal$sum_data_size_for_stats", - "presto.default.sum_data_size_for_stats"}, + formatDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "sum_data_size_for_stats")}, }; std::string lowerCaseName = boost::to_lower_copy(name); auto it = kFunctionNames.find(name); @@ -167,6 +189,8 @@ std::optional convertCastToVarcharWithMaxLength( const std::string& returnType, const std::vector& args, bool nullOnFailure) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); if (nullOnFailure) { VELOX_NYI("TRY_CAST of varchar to {} is not supported.", returnType); } @@ -190,7 +214,7 @@ std::optional convertCastToVarcharWithMaxLength( std::make_shared(velox::BIGINT(), 1LL), std::make_shared(velox::BIGINT(), (int64_t)length), }, - "presto.default.substr"); + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "substr")); } /// Converts cast and try_cast functions to CastTypedExpr with nullOnFailure @@ -207,6 +231,8 @@ std::optional tryConvertCast( const std::string& returnType, const std::vector& args, const TypeParser* typeParser) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const char* kCast = "presto.default.$operator$cast"; static const char* kTryCast = "presto.default.try_cast"; static const char* kJsonToArrayCast = @@ -237,7 +263,10 @@ std::optional tryConvertCast( return std::make_shared( type, std::vector{std::make_shared( - velox::JSON(), args, "presto.default.json_parse")}, + velox::JSON(), + args, + formatDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "json_parse"))}, false); } else { return std::nullopt; @@ -274,7 +303,6 @@ std::optional tryConvertTry( const std::vector& args, const TypeParser* typeParser) { static const char* kTry = "presto.default.$internal$try"; - if (signature.kind != protocol::FunctionKind::SCALAR) { return std::nullopt; } @@ -300,8 +328,11 @@ std::optional tryConvertLiteralArray( const std::vector& args, velox::memory::MemoryPool* pool, const TypeParser* typeParser) { - static const char* kLiteralArray = "presto.default.$literal$array"; - static const char* kFromBase64 = "presto.default.from_base64"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const char* kLiteralArray = "presto.default.$literal.$array"; + static const std::string kFromBase64 = + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "from_base64"); if (signature.kind != protocol::FunctionKind::SCALAR) { return std::nullopt; @@ -342,7 +373,10 @@ std::optional tryConvertLiteralArray( std::optional VeloxExprConverter::tryConvertDate( const protocol::CallExpression& pexpr) const { - static const char* kDate = "presto.default.date"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const std::string kDate = + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "date"); auto builtin = std::static_pointer_cast( pexpr.functionHandle); @@ -363,8 +397,12 @@ std::optional VeloxExprConverter::tryConvertDate( std::optional VeloxExprConverter::tryConvertLike( const protocol::CallExpression& pexpr) const { - static const char* kLike = "presto.default.like"; - static const char* kLikePatternType = "presto.default.like_pattern"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const std::string kLike = + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "like"); + static const std::string kLikePatternType = formatDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "like_pattern"); static const char* kLikeReturnType = "LikePattern"; static const char* kCast = "presto.default.$operator$cast"; @@ -500,9 +538,13 @@ bool isTrueConstant(const TypedExprPtr& expression) { std::shared_ptr makeEqualsExpr( const TypedExprPtr& a, const TypedExprPtr& b) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); std::vector inputs{a, b}; return std::make_shared( - velox::BOOLEAN(), std::move(inputs), "presto.default.eq"); + velox::BOOLEAN(), + std::move(inputs), + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "eq")); } std::shared_ptr makeCastExpr( diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h index f63a84ec35ad..214f1b80d8ca 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h @@ -14,10 +14,19 @@ #pragma once #include +#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/types/TypeParser.h" #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/core/Expressions.h" +namespace { +inline std::string formatDefaultNamespacePrefix( + const std::string& prestoDefaultNamespacePrefix, + const std::string& functionName) { + return fmt::format("{}{}", prestoDefaultNamespacePrefix, functionName); +} +} // namespace + namespace facebook::presto { class VeloxExprConverter { diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp index eeab749ff299..181e14297c15 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp @@ -493,7 +493,10 @@ std::shared_ptr isFunctionCall( /// CallExpression. Returns nullptr if input expression is something else. std::shared_ptr isNot( const std::shared_ptr& expression) { - static const std::string_view kNot = "presto.default.not"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const std::string kNot = + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "not"); return isFunctionCall(expression, kNot); } @@ -1514,11 +1517,13 @@ namespace { core::WindowNode::Function makeRowNumberFunction( const protocol::VariableReferenceExpression& rowNumberVariable, const TypeParser& typeParser) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); core::WindowNode::Function function; function.functionCall = std::make_shared( stringToType(rowNumberVariable.type, typeParser), std::vector{}, - "presto.default.row_number"); + formatDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "row_number")); function.frame.type = core::WindowNode::WindowType::kRows; function.frame.startType = core::WindowNode::BoundType::kUnboundedPreceding; diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.h b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.h index 4e0168dbafe0..52b656339559 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.h +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.h @@ -14,6 +14,7 @@ #include #include +#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/operators/ShuffleInterface.h" #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/core/Expressions.h"