Skip to content

Commit

Permalink
Arrow Flight connector template
Browse files Browse the repository at this point in the history 
Co-authored-by: sai bhaskar reddy <sai.bhaskar.reddy.sabbasani1@ibm.com>
Co-authored-by: SthuthiGhosh9400 <Sthuthi.Ghosh@ibm.com>
Co-authored-by: lithinwxd <Lithin.Purushothaman@ibm.com>
Co-authored-by: Steve Burnett <burnett@pobox.com>
Co-authored-by: elbinpallimalilibm <elbin.pallimalil@ibm.com>
Co-authored-by: Tim Meehan <tim@timdmeehan.com>
  • Loading branch information
@sabbasani @SthuthiGhosh9400
@lithinwxd @steveburnett @elbinpallimalilibm @tdcmeehan
6 people committed Dec 30, 2024
1 parent 004ee32 commit f8b2dc0
Show file tree
Hide file tree
Showing 51 changed files with 6,126 additions and 1 deletion.
82 changes: 82 additions & 0 deletions .github/workflows/arrow-flight-tests.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
name: arrow flight tests

on:
pull_request:

env:
CONTINUOUS_INTEGRATION: true
MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError"
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true"
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end"
RETRY: .github/bin/retry

jobs:
changes:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
codechange: ${{ steps.filter.outputs.codechange }}
steps:
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
codechange:
- '!presto-docs/**'
test:
runs-on: ubuntu-latest
needs: changes
strategy:
fail-fast: false
matrix:
modules:
- ":presto-base-arrow-flight" # Only run tests for the `presto-base-arrow-flight` module

timeout-minutes: 80
concurrency:
group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

steps:
# Checkout the code only if there are changes in the relevant files
- uses: actions/checkout@v4
if: needs.changes.outputs.codechange == 'true'
with:
show-progress: false

# Set up Java for the build environment
- uses: actions/setup-java@v2
if: needs.changes.outputs.codechange == 'true'
with:
distribution: 'temurin'
java-version: 8

# Cache Maven dependencies to speed up the build
- name: Cache local Maven repository
if: needs.changes.outputs.codechange == 'true'
id: cache-maven
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-2-
# Resolve Maven dependencies (if cache is not found)
- name: Populate Maven cache
if: steps.cache-maven.outputs.cache-hit != 'true' && needs.changes.outputs.codechange == 'true'
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies --no-transfer-progress && .github/bin/download_nodejs

# Install dependencies for the target module
- name: Maven Install
if: needs.changes.outputs.codechange == 'true'
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
# Run Maven tests for the target module
- name: Maven Tests
if: needs.changes.outputs.codechange == 'true'
run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }}
3 changes: 2 additions & 1 deletion .github/workflows/test-other-modules.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,4 +84,5 @@ jobs:
!presto-test-coverage,
!presto-iceberg,
!presto-singlestore,
!presto-native-sidecar-plugin'
!presto-native-sidecar-plugin,
!presto-base-arrow-flight'
27 changes: 27 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
<dep.protobuf-java.version>3.25.5</dep.protobuf-java.version>
<dep.netty.version>4.1.115.Final</dep.netty.version>
<dep.snakeyaml.version>2.0</dep.snakeyaml.version>
<dep.arrow.version>17.0.0</dep.arrow.version>

<!--
America/Bahia_Banderas has:
Expand Down Expand Up @@ -204,6 +205,7 @@
<module>presto-hana</module>
<module>presto-openapi</module>
<module>presto-native-sidecar-plugin</module>
<module>presto-base-arrow-flight</module>
</modules>

<dependencyManagement>
Expand Down Expand Up @@ -2242,6 +2244,31 @@
<artifactId>snakeyaml</artifactId>
<version>${dep.snakeyaml.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-jdbc</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>
<version>${dep.arrow.version}</version>
</dependency>

</dependencies>
</dependencyManagement>

Expand Down
259 changes: 259 additions & 0 deletions presto-base-arrow-flight/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-root</artifactId>
<version>0.291-SNAPSHOT</version>
</parent>
<artifactId>presto-base-arrow-flight</artifactId>
<name>presto-base-arrow-flight</name>
<description>Presto - Base Arrow Flight Connector</description>

<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<error_prone_annotations.version>2.28.0</error_prone_annotations.version>
<perfmark-api.version>0.27.0</perfmark-api.version>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-jdbc</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<!-- Excluding to resolve upper bound errors -->
<exclusion>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>bootstrap</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>log</artifactId>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-spi</artifactId>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>slice</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>log-manager</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-common</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>

<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<optional>true</optional>
</dependency>

<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>configuration</artifactId>
</dependency>

<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>

<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi3-core</artifactId>
</dependency>

<!-- for testing -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.airlift.tpch</groupId>
<artifactId>tpch</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-tpch</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>json</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-testng-services</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.airlift</groupId>
<artifactId>testing</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-main</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-tests</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<version>${error_prone_annotations.version}</version>
</dependency>

<dependency>
<groupId>io.perfmark</groupId>
<artifactId>perfmark-api</artifactId>
<version>${perfmark-api.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-Xss10M</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.basepom.maven</groupId>
<artifactId>duplicate-finder-maven-plugin</artifactId>
<version>1.2.1</version>
<configuration>
<ignoredClassPatterns>
<ignoredClassPattern>module-info</ignoredClassPattern>
<ignoredClassPattern>META-INF.versions.9.module-info</ignoredClassPattern>
</ignoredClassPatterns>
<ignoredResourcePatterns>
<ignoredResourcePattern>arrow-git.properties</ignoredResourcePattern>
<ignoredResourcePattern>about.html</ignoredResourcePattern>
</ignoredResourcePatterns>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Loading

0 comments on commit f8b2dc0

Please sign in to comment.