Skip to content

Commit

Permalink
[Hotfix][Zeta] Fix the dependency conflict between the guava in hadoo…
Browse files Browse the repository at this point in the history
…p-aws and hive-exec (#7986)
  • Loading branch information
dailai authored Nov 7, 2024
1 parent 34ceb9e commit a7837f1
Show file tree
Hide file tree
Showing 7 changed files with 140 additions and 29 deletions.
2 changes: 2 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@
<prometheus.simpleclient.version>0.16.0</prometheus.simpleclient.version>
<enableSourceJarCreation>true</enableSourceJarCreation>

<hadoop-aws.version>3.1.4</hadoop-aws.version>

</properties>

<dependencyManagement>
Expand Down
27 changes: 4 additions & 23 deletions seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,6 @@
<artifactId>connector-file-s3</artifactId>
<name>SeaTunnel : Connectors V2 : File : S3</name>

<properties>
<hadoop-aws.version>3.1.4</hadoop-aws.version>
<guava.version>27.0-jre</guava.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>

<dependency>
Expand All @@ -67,14 +52,10 @@
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop-aws.version}</version>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-hadoop-aws</artifactId>
<version>${project.version}</version>
<classifier>optional</classifier>
<scope>provided</scope>
<exclusions>
<exclusion>
Expand Down
16 changes: 12 additions & 4 deletions seatunnel-dist/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@
<!-- Imap storage dependency package -->
<hadoop-aliyun.version>3.1.4</hadoop-aliyun.version>
<json-smart.version>2.4.7</json-smart.version>
<hadoop-aws.version>3.1.4</hadoop-aws.version>
<aws-java-sdk.version>1.11.271</aws-java-sdk.version>
<netty-buffer.version>4.1.89.Final</netty-buffer.version>
<hive.exec.version>3.1.3</hive.exec.version>
Expand Down Expand Up @@ -735,9 +734,10 @@

<!-- hadoop jar -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop-aws.version}</version>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-hadoop-aws</artifactId>
<version>${project.version}</version>
<classifier>optional</classifier>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -960,6 +960,14 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-hadoop-aws</artifactId>
<version>${project.version}</version>
<classifier>optional</classifier>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<finalName>apache-seatunnel-${project.version}</finalName>
Expand Down
4 changes: 2 additions & 2 deletions seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@
<scope>provided</scope>
</dependencySet>

<!-- =================== JDBC Connector Drivers And SeaTunnel Hadoop3 Uber Jar =================== -->
<!-- =================== JDBC Connector Drivers, SeaTunnel Hadoop3 Uber Jar And SeaTunnel Hadoop AWS Uber Jar =================== -->
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
Expand All @@ -181,9 +181,9 @@
<include>com.amazon.redshift:redshift-jdbc42:jar</include>
<include>net.snowflake.snowflake-jdbc:jar</include>
<include>com.xugudb:xugu-jdbc:jar</include>
<include>org.apache.hadoop:hadoop-aws:jar</include>
<include>com.amazonaws:aws-java-sdk-bundle:jar</include>
<include>org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional</include>
<include>org.apache.seatunnel:seatunnel-hadoop-aws:jar:*:optional</include>
<!--Add hadoop aliyun jar -->
<include>org.apache.hadoop:hadoop-aliyun:jar</include>
<include>com.aliyun.oss:aliyun-sdk-oss:jar</include>
Expand Down
13 changes: 13 additions & 0 deletions seatunnel-dist/src/main/assembly/assembly-bin.xml
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,19 @@
<scope>provided</scope>
</dependencySet>

<!-- ============ SeaTunnel Hadoop-AWS Uber Jar============ -->
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
<unpack>false</unpack>
<includes>
<include>org.apache.seatunnel:seatunnel-hadoop-aws:jar:*:optional</include>
</includes>
<outputFileNameMapping>${artifact.file.name}</outputFileNameMapping>
<outputDirectory>/lib</outputDirectory>
<scope>provided</scope>
</dependencySet>

<!-- ============ Connectors Jars And Transforms V2 Jar ============ -->
<!-- SeaTunnel connectors for Demo -->
<dependencySet>
Expand Down
1 change: 1 addition & 0 deletions seatunnel-shade/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
<module>seatunnel-hazelcast</module>
<module>seatunnel-janino</module>
<module>seatunnel-jetty9-9.4.56</module>
<module>seatunnel-hadoop-aws</module>

</modules>

Expand Down
106 changes: 106 additions & 0 deletions seatunnel-shade/seatunnel-hadoop-aws/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-shade</artifactId>
<version>${revision}</version>
</parent>

<artifactId>seatunnel-hadoop-aws</artifactId>

<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop-aws.version}</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>shade</goal>
</goals>
<phase>package</phase>
<configuration>
<finalName>seatunnel-hadoop-aws</finalName>
<createSourcesJar>${enableSourceJarCreation}</createSourcesJar>
<shadeSourcesContent>true</shadeSourcesContent>
<shadedArtifactAttached>false</shadedArtifactAttached>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>${seatunnel.shade.package}.com.google.common</shadedPattern>
<includes>
<include>com.google.common.base.*</include>
<include>com.google.common.cache.*</include>
<include>com.google.common.collect.*</include>
</includes>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>attach-artifacts</id>
<goals>
<goal>attach-artifact</goal>
</goals>
<phase>package</phase>
<configuration>
<artifacts>
<artifact>
<file>${basedir}/target/seatunnel-hadoop-aws.jar</file>
<type>jar</type>
<classifier>optional</classifier>
</artifact>
</artifacts>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>

0 comments on commit a7837f1

Please sign in to comment.