From e32e6b1de14f628786bdb08770a8468ba1df5e46 Mon Sep 17 00:00:00 2001
From: Lynn Rozen <lynn.rozen@treeverse.io>
Date: Wed, 12 Jul 2023 19:31:57 +0300
Subject: [PATCH] unify to 1 job

---
 .github/workflows/tests.yml | 70 ++++++++++++++++++++++---------------
 test/requirements.txt       |  2 +-
 test/run-test.py            |  2 +-
 3 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5b8287f..ee0a56a 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -12,35 +12,35 @@ on:
 #  packages: write
 
 jobs:
-  build-lakefs-iceberg:
-    name: Build lakeFS Iceberg package
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check-out code
-        uses: actions/checkout@v3
-
-      - name: Set up JDK 11
-        uses: actions/setup-java@v3
-        with:
-          distribution: "temurin"
-          java-version: "11"
-          cache: "maven"
-
-      - name: Build lakeFS Iceberg
-        run: mvn clean install --batch-mode --update-snapshots -DskipTests -P\!sign-artifacts
-#        run: mvn -DfinalName=lakefs-iceberg --batch-mode --update-snapshots package -DskipTests
-#        run: mvn -Passembly -DfinalName=lakefs-iceberg --batch-mode --update-snapshots package -DskipTests
-
-      - name: Store lakeFS Iceberg
-        uses: actions/upload-artifact@v3
-        with:
-          name: lakefs-iceberg
-          path: target/lakefs-iceberg-1.0-SNAPSHOT.jar
+#  build-lakefs-iceberg:
+#    name: Build lakeFS Iceberg package
+#    runs-on: ubuntu-20.04
+#    steps:
+#      - name: Check-out code
+#        uses: actions/checkout@v3
+#
+#      - name: Set up JDK 11
+#        uses: actions/setup-java@v3
+#        with:
+#          distribution: "temurin"
+#          java-version: "11"
+#          cache: "maven"
+#
+#      - name: Build lakeFS Iceberg
+#        run: mvn clean install --batch-mode --update-snapshots -DskipTests -P\!sign-artifacts
+##        run: mvn -DfinalName=lakefs-iceberg --batch-mode --update-snapshots package -DskipTests
+##        run: mvn -Passembly -DfinalName=lakefs-iceberg --batch-mode --update-snapshots package -DskipTests
+#
+#      - name: Store lakeFS Iceberg
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: lakefs-iceberg
+#          path: target/lakefs-iceberg-1.0-SNAPSHOT.jar
 
 
   iceberg-isolation-test:
     name: Test lakeFS isolation with iceberg
-    needs: [ build-lakefs-iceberg ]
+#    needs: [ build-lakefs-iceberg ]
     runs-on: ubuntu-20.04
     services:
       lakefs:
@@ -59,11 +59,23 @@ jobs:
       - name: Check-out code
         uses: actions/checkout@v3
 
-      - name: Download lakeFS Iceberg
-        uses: actions/download-artifact@v3
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
         with:
-          name: lakefs-iceberg
-          path: target/
+          distribution: "temurin"
+          java-version: "11"
+          cache: "maven"
+
+      - name: Build lakeFS Iceberg
+        run: mvn clean install --batch-mode --update-snapshots -DskipTests -P\!sign-artifacts
+      #        run: mvn -DfinalName=lakefs-iceberg --batch-mode --update-snapshots package -DskipTests
+      #        run: mvn -Passembly -DfinalName=lakefs-iceberg --batch-mode --update-snapshots package -DskipTests
+
+#      - name: Download lakeFS Iceberg
+#        uses: actions/download-artifact@v3
+#        with:
+#          name: lakefs-iceberg
+#          path: target/
 
 #      - name: Download Spark App
 #        uses: actions/download-artifact@v3
diff --git a/test/requirements.txt b/test/requirements.txt
index 7eaad59..2c10c27 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,3 +1,3 @@
 lakefs_client==0.104.0
-pyspark==3.4.1
+pyspark==3.3.2
 tenacity==8.2.2
\ No newline at end of file
diff --git a/test/run-test.py b/test/run-test.py
index 7f1353b..2cca4aa 100644
--- a/test/run-test.py
+++ b/test/run-test.py
@@ -49,7 +49,7 @@ def main():
     spark_config.set("spark.hadoop.fs.s3a.access.key", lakefs_access_key)
     spark_config.set("spark.hadoop.fs.s3a.secret.key", lakefs_secret_key)
     spark_config.set("spark.hadoop.fs.s3a.path.style.access", "true")
-    spark_config.set("spark.jars.packages", "io.lakefs:lakefs-iceberg:1.0-SNAPSHOT")
+    spark_config.set("spark.jars.packages", "io.lakefs:lakefs-iceberg:1.0-SNAPSHOT,org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.3.0")
 
     spark = SparkSession.builder.config(conf=spark_config).getOrCreate()