Add e2e test for remote functions

prestodb · Dec 9, 2024 · c8c9686 · c8c9686
1 parent b68af58
commit c8c9686
Show file tree

Hide file tree

Showing 7 changed files with 449 additions and 9 deletions.
diff --git a/.gitignore b/.gitignore
@@ -66,3 +66,4 @@ presto-native-execution/deps-install
 # Compiled executables used for docker build
 /docker/presto-cli-*-executable.jar
 /docker/presto-server-*.tar.gz
+/docker/presto-remote-function-server-executable.jar
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -3,12 +3,15 @@ FROM quay.io/centos/centos:stream9
 ARG PRESTO_VERSION
 ARG PRESTO_PKG=presto-server-$PRESTO_VERSION.tar.gz
 ARG PRESTO_CLI_JAR=presto-cli-$PRESTO_VERSION-executable.jar
+ARG PRESTO_REMOTE_SERVER_JAR=presto-remote-function-server-executable.jar
 ARG JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0
 
 ENV PRESTO_HOME="/opt/presto-server"
 
 COPY $PRESTO_PKG .
 COPY $PRESTO_CLI_JAR /opt/presto-cli
+COPY $PRESTO_REMOTE_SERVER_JAR /opt/presto-remote-server
+
 
 RUN dnf install -y java-11-openjdk less procps python3 \
     && ln -s $(which python3) /usr/bin/python \
@@ -19,6 +22,7 @@ RUN dnf install -y java-11-openjdk less procps python3 \
     && rm -rf ./presto-server-$PRESTO_VERSION \
     && chmod +x /opt/presto-cli \
     && ln -s /opt/presto-cli /usr/local/bin/ \
+    && chmod +x /opt/presto-remote-server \
     # clean cache jobs
     && mv /etc/yum/protected.d/systemd.conf /etc/yum/protected.d/systemd.conf.bak \
     && dnf clean all \

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -2,4 +2,6 @@
 
 set -e
 
+java -Dconfig=/opt/function-server/etc/config.properties -jar /opt/presto-remote-server  >> log1.txt 2>&1
+
 $PRESTO_HOME/bin/launcher run
diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml
@@ -362,6 +362,12 @@
                                                 <include>presto-cli-*-executable.jar</include>
                                             </includes>
                                         </resource>
+                                        <resource>
+                                            <directory>${project.parent.basedir}/presto-main/target</directory>
+                                            <includes>
+                                                <include>presto-remote-function-server-executable.jar</include>
+                                            </includes>
+                                        </resource>
                                         <resource>
                                             <directory>${project.parent.basedir}/presto-server/target</directory>
                                             <includes>
@@ -433,7 +439,7 @@
                                                 <args>
                                                     <BUILD_TYPE>Release</BUILD_TYPE>
                                                     <DEPENDENCY_IMAGE>presto-native-dependency:latest</DEPENDENCY_IMAGE>
-                                                    <EXTRA_CMAKE_FLAGS>-DPRESTO_ENABLE_TESTING=OFF</EXTRA_CMAKE_FLAGS>
+                                                    <EXTRA_CMAKE_FLAGS>-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON</EXTRA_CMAKE_FLAGS>
                                                     <NUM_THREADS>2</NUM_THREADS>
                                                     <BASE_IMAGE>ubuntu:22.04</BASE_IMAGE>
                                                     <OSNAME>ubuntu</OSNAME>

diff --git a/...e-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunnerUtils.java b/...e-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunnerUtils.java
@@ -83,6 +83,19 @@ public static void createNativeWorkerConfigProperties(int coordinatorPort, Strin
         createPropertiesFile("testcontainers/" + nodeId + "/etc/config.properties", properties);
     }
 
+    public static void createNativeWorkerConfigPropertiesWithFunctionServer(int coordinatorPort, int functionServerPort, String nodeId)
+            throws IOException
+    {
+        Properties properties = new Properties();
+        properties.setProperty("presto.version", "testversion");
+        properties.setProperty("http-server.http.port", "7777");
+        properties.setProperty("discovery.uri", "http://presto-coordinator:" + coordinatorPort);
+        properties.setProperty("system-memory-gb", "2");
+        properties.setProperty("native.sidecar", "false");
+        properties.setProperty("remote-function-server.rest.url", "http://presto-coordinator:" + functionServerPort);
+        createPropertiesFile("testcontainers/" + nodeId + "/etc/config.properties", properties);
+    }
+
     public static void createCoordinatorConfigProperties(int port)
             throws IOException
     {
@@ -93,6 +106,8 @@ public static void createCoordinatorConfigProperties(int port)
         properties.setProperty("http-server.http.port", Integer.toString(port));
         properties.setProperty("discovery-server.enabled", "true");
         properties.setProperty("discovery.uri", "http://presto-coordinator:" + port);
+        properties.setProperty("list-built-in-functions-only", "false");
+        properties.setProperty("native-execution-enabled", "false");
 
         // Get native worker system properties and add them to the coordinator properties
         Map<String, String> nativeWorkerProperties = NativeQueryRunnerUtils.getNativeWorkerSystemProperties();
@@ -103,6 +118,52 @@ public static void createCoordinatorConfigProperties(int port)
         createPropertiesFile("testcontainers/coordinator/etc/config.properties", properties);
     }
 
+    public static void createFunctionNamespaceRemoteProperties()
+            throws IOException
+    {
+        Properties properties = new Properties();
+        properties.setProperty("function-namespace-manager.name", "rest");
+        properties.setProperty("supported-function-languages", "Java");
+        properties.setProperty("function-implementation-type", "REST");
+
+        String directoryPath = "testcontainers/function-namespace";
+        File directory = new File(directoryPath);
+        if (!directory.exists()) {
+            directory.mkdirs();
+        }
+
+        createPropertiesFile("testcontainers/coordinator/etc/function-namespace/remote.properties", properties);
+    }
+
+    public static void createFunctionNamespaceRemotePropertiesWithFunctionServer(int functionServerPort)
+            throws IOException
+    {
+        Properties properties = new Properties();
+        properties.setProperty("function-namespace-manager.name", "rest");
+        properties.setProperty("supported-function-languages", "Java");
+        properties.setProperty("function-implementation-type", "REST");
+        properties.setProperty("rest-based-function-manager.rest.url", "http://localhost:" + functionServerPort);
+
+        String directoryPath = "testcontainers/function-namespace";
+        File directory = new File(directoryPath);
+        if (!directory.exists()) {
+            directory.mkdirs();
+        }
+
+        createPropertiesFile("testcontainers/coordinator/etc/function-namespace/remote.properties", properties);
+    }
+
+    public static void createFunctionServerConfigProperties(int functionServerPort)
+            throws IOException
+    {
+        Properties properties = new Properties();
+        properties.setProperty("http-server.http.port", String.valueOf(functionServerPort));
+        properties.setProperty("regex-library", "RE2J");
+        properties.setProperty("parse-decimal-literals-as-double", "true");
+
+        createPropertiesFile("testcontainers/coordinator/etc/function-server/config.properties", properties);
+    }
+
     public static void createCoordinatorJvmConfig()
             throws IOException
 
@@ -161,6 +222,11 @@ public static void createCoordinatorEntryPointScript()
     {
         String scriptContent = "#!/bin/sh\n" +
                 "set -e\n" +
+                "java " +
+//                "-Dplugin.dir=/opt/presto-remote-server/function-server-plugin " +
+//                "-Dconfig=/opt/presto-remote-server/function-server-etc/config.properties " +
+//                "-jar /opt/presto-remote-server  >> log1.txt 2>&1 & \n" +
+                "-Dconfig=/opt/function-server/etc/config.properties -jar /opt/presto-remote-server  >> log1.txt 2>&1 & \n" +
                 "$PRESTO_HOME/bin/launcher run\n";
         createScriptFile("testcontainers/coordinator/entrypoint.sh", scriptContent);
     }
@@ -204,10 +270,6 @@ public static void createPropertiesFile(String filePath, Properties properties)
             parentDir.mkdirs();
         }
 
-        if (file.exists()) {
-            throw new IOException("File exists: " + filePath);
-        }
-
         try (OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8)) {
             for (String key : properties.stringPropertyNames()) {
                 writer.write(key + "=" + properties.getProperty(key) + "\n");
@@ -224,10 +286,6 @@ public static void createScriptFile(String filePath, String scriptContent)
             parentDir.mkdirs();
         }
 
-        if (file.exists()) {
-            throw new IOException("File exists: " + filePath);
-        }
-
         try (OutputStream output = new FileOutputStream(file);
                 OutputStreamWriter writer = new OutputStreamWriter(output, StandardCharsets.UTF_8)) {
             writer.write(scriptContent);
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,6 @@

		set -e

		java -Dconfig=/opt/function-server/etc/config.properties -jar /opt/presto-remote-server >> log1.txt 2>&1

		$PRESTO_HOME/bin/launcher run