Merge branch 'master' into RFC_5_UDF_STATS_PHASE_1

prestodb · Dec 12, 2024 · 9551c16 · 9551c16
2 parents 40b6f47 + 5c9fdb7
commit 9551c16
Show file tree

Hide file tree

Showing 286 changed files with 5,899 additions and 1,764 deletions.
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
@@ -65,7 +65,7 @@ executors:
   build:
     docker:
       # These images are managed by the Presto Release team.
-      - image: prestodb/presto-native-dependency:latest
+      - image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
     resource_class: 2xlarge
     environment:
       MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError -XX:CompressedClassSpaceSize=3g"

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,9 +1,50 @@
+AGENT_MAVEN = '''
+    apiVersion: v1
+    kind: Pod
+    spec:
+        containers:
+        - name: maven
+          image: maven:3.8.6-openjdk-8-slim
+          env:
+          - name: MAVEN_OPTS
+            value: "-Xmx8000m -Xms8000m"
+          resources:
+            requests:
+              memory: "16Gi"
+              cpu: "4000m"
+            limits:
+              memory: "16Gi"
+              cpu: "4000m"
+          tty: true
+          command:
+          - cat
+'''
+
+AGENT_DIND = '''
+    apiVersion: v1
+    kind: Pod
+    spec:
+        containers:
+        - name: dind
+          image: docker:20.10.16-dind-alpine3.15
+          securityContext:
+            privileged: true
+          tty: true
+          resources:
+            requests:
+              memory: "29Gi"
+              cpu: "7500m"
+            limits:
+              memory: "29Gi"
+              cpu: "7500m"
+'''
+
 pipeline {
 
     agent none
 
     environment {
-        AWS_CREDENTIAL_ID  = 'aws-jenkins'
+        AWS_CREDENTIAL_ID  = 'oss-presto-aws'
         AWS_DEFAULT_REGION = 'us-east-1'
         AWS_ECR            = 'public.ecr.aws/oss-presto'
         AWS_S3_PREFIX      = 's3://oss-prestodb/presto'
@@ -33,7 +74,7 @@ pipeline {
             agent {
                 kubernetes {
                     defaultContainer 'maven'
-                    yamlFile 'jenkins/agent-maven.yaml'
+                    yaml AGENT_MAVEN
                 }
             }
 
@@ -132,7 +173,7 @@ pipeline {
             agent {
                 kubernetes {
                     defaultContainer 'dind'
-                    yamlFile 'jenkins/agent-dind.yaml'
+                    yaml AGENT_DIND
                 }
             }
 

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ See [PrestoDB: Mission and Architecture](ARCHITECTURE.md).
 
 * Mac OS X or Linux
 * Java 8 Update 151 or higher (8u151+), 64-bit. Both Oracle JDK and OpenJDK are supported.
-* Maven 3.6.1+ (for building)
+* Maven 3.6.3+ (for building)
 * Python 2.4+ (for running with the launcher script)
 
 <details> <!-- from: https://github.com/prestodb/presto/blob/master/README.md -->

diff --git a/jenkins/agent-dind.yaml b/jenkins/agent-dind.yaml
diff --git a/jenkins/agent-maven.yaml b/jenkins/agent-maven.yaml
diff --git a/pom.xml b/pom.xml
@@ -77,11 +77,13 @@
         <dep.ratis.version>2.2.0</dep.ratis.version>
         <dep.errorprone.version>2.18.0</dep.errorprone.version>
         <dep.guava.version>32.1.0-jre</dep.guava.version>
-        <dep.jackson.version>2.11.0</dep.jackson.version>
+        <dep.jackson.version>2.15.4</dep.jackson.version>
         <dep.j2objc.version>2.8</dep.j2objc.version>
         <dep.avro.version>1.11.4</dep.avro.version>
         <dep.commons.compress.version>1.26.2</dep.commons.compress.version>
         <dep.protobuf-java.version>3.25.5</dep.protobuf-java.version>
+        <dep.netty.version>4.1.115.Final</dep.netty.version>
+        <dep.snakeyaml.version>2.0</dep.snakeyaml.version>
 
         <!--
           America/Bahia_Banderas has:
@@ -208,10 +210,12 @@
         <dependencies>
             <dependency>
                 <groupId>io.netty</groupId>
-                <artifactId>netty-handler</artifactId>
-                <version>4.1.107.Final</version>
+                <artifactId>netty-bom</artifactId>
+                <version>${dep.netty.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
             </dependency>
-            
+
             <dependency>
                 <groupId>com.facebook.presto</groupId>
                 <artifactId>presto-testing-docker</artifactId>
@@ -797,6 +801,36 @@
                 <version>${dep.jackson.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+                <version>${dep.jackson.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-annotations</artifactId>
+                <version>${dep.jackson.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.fasterxml.jackson.dataformat</groupId>
+                <artifactId>jackson-dataformat-smile</artifactId>
+                <version>${dep.jackson.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.fasterxml.jackson.dataformat</groupId>
+                <artifactId>jackson-dataformat-yaml</artifactId>
+                <version>${dep.jackson.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>${dep.jackson.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>com.jayway.jsonpath</groupId>
                 <artifactId>json-path</artifactId>
@@ -2057,22 +2091,6 @@
                 <version>1.1.3</version>
             </dependency>
 
-            <dependency>
-                <groupId>com.facebook.presto.cassandra</groupId>
-                <artifactId>cassandra-server</artifactId>
-                <version>2.1.16-1</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>com.googlecode.json-simple</groupId>
-                        <artifactId>json-simple</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>io.netty</groupId>
-                        <artifactId>netty-all</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-
             <dependency>
                 <groupId>com.facebook.presto</groupId>
                 <artifactId>presto-delta</artifactId>
@@ -2218,6 +2236,12 @@
                 <artifactId>cassandra-driver-core</artifactId>
                 <version>3.11.5</version>
             </dependency>
+
+            <dependency>
+                <groupId>org.yaml</groupId>
+                <artifactId>snakeyaml</artifactId>
+                <version>${dep.snakeyaml.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -2286,6 +2310,14 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-enforcer-plugin</artifactId>
+                    <version>3.3.0</version>
+                    <dependencies>
+                        <dependency>
+                            <groupId>org.codehaus.mojo</groupId>
+                            <artifactId>extra-enforcer-rules</artifactId>
+                            <version>1.6.2</version>
+                        </dependency>
+                    </dependencies>
                     <configuration>
                         <rules>
                             <requireUpperBoundDeps>
@@ -2310,6 +2342,8 @@
                         <ignoredClassPatterns combine.children="append">
                             <!-- Duplicate class is being brought in by commons-io & log4j-api -->
                             <ignoredClassPattern>META-INF.versions.9.module-info</ignoredClassPattern>
+                            <!-- Duplicate class is being brought in by several netty dependencies-->
+                            <ignoredClassPattern>META-INF.versions.11.module-info</ignoredClassPattern>
                         </ignoredClassPatterns>
                     </configuration>
                 </plugin>
@@ -2648,7 +2682,7 @@
                                 <configuration>
                                     <executable>ln</executable>
                                     <arguments>
-                                        <argument>-sT</argument>
+                                        <argument>-sfT</argument>
                                         <argument>${out-of-tree-build-root}/${project.groupId}:${project.artifactId}</argument>
                                         <argument>${project.build.directory}</argument>
                                     </arguments>

diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java
@@ -55,6 +55,7 @@
 import com.facebook.presto.sql.tree.Revoke;
 import com.facebook.presto.sql.tree.RevokeRoles;
 import com.facebook.presto.sql.tree.Rollback;
+import com.facebook.presto.sql.tree.SetProperties;
 import com.facebook.presto.sql.tree.SetRole;
 import com.facebook.presto.sql.tree.SetSession;
 import com.facebook.presto.sql.tree.ShowCatalogs;
@@ -142,6 +143,7 @@ private StatementUtils() {}
         builder.put(DropFunction.class, QueryType.CONTROL);
         builder.put(Use.class, QueryType.CONTROL);
         builder.put(SetSession.class, QueryType.CONTROL);
+        builder.put(SetProperties.class, QueryType.DATA_DEFINITION);
         builder.put(ResetSession.class, QueryType.CONTROL);
         builder.put(StartTransaction.class, QueryType.CONTROL);
         builder.put(Commit.class, QueryType.CONTROL);

diff --git a/presto-benchto-benchmarks/pom.xml b/presto-benchto-benchmarks/pom.xml
@@ -26,6 +26,32 @@
             <groupId>com.teradata.benchto</groupId>
             <artifactId>benchto-driver</artifactId>
             <version>0.4</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-web</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-autoconfigure</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-beans</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.hibernate</groupId>
+                    <artifactId>hibernate-validator</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

diff --git a/presto-bigquery/pom.xml b/presto-bigquery/pom.xml
@@ -353,6 +353,7 @@
                                 <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
                                 <exclude>javax.annotation:javax.annotation-api</exclude>
                                 <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
+                                <exclude>com.google.api.grpc:proto-google-common-protos</exclude>
                             </excludes>
                         </requireUpperBoundDeps>
                     </rules>

diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml
@@ -156,8 +156,8 @@
         </dependency>
 
         <dependency>
-            <groupId>com.facebook.presto.cassandra</groupId>
-            <artifactId>cassandra-server</artifactId>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
             <scope>test</scope>
         </dependency>
 
@@ -185,6 +185,17 @@
         </dependency>
     </dependencies>
 
+    <!--- dependencyManagement is to fix the Require upper bound dependencies error for slf4j  -->
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+                <version>1.7.36</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <build>
         <plugins>
             <plugin>