SBT publish... and add tests to main itself

.github/workflows/ci-scala.yaml

@@ -0,0 +1,26 @@
+name: Scala CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request: {}
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK 11
+      uses: actions/setup-java@v3
+      with:
+        java-version: '11'
+        distribution: 'temurin'
+        cache: 'sbt'
+    - name: Run tests
+      run: sbt test

build.sbt

@@ -1,4 +1,9 @@
+lazy val projectVersion = "0.0.1"
+
 lazy val project = Project("lakefs-spark-extension", file("./src/"))
+  .settings(
+    version := projectVersion,
+  )
 
 semanticdbEnabled := true
 semanticdbVersion := scalafixSemanticdb.revision
@@ -7,13 +12,30 @@ scalacOptions += "-Ywarn-unused-import"
 libraryDependencies ++= Seq(
   "io.lakefs" % "api-client" % "0.91.0",
   "org.apache.spark" %% "spark-sql" % "3.2.4" % "provided",
+  "org.apache.spark" %% "spark-hive" % "3.2.4" % "test",
+  "joda-time" % "joda-time" % "2.12.5" % "test",
 
   // TODO(ariels): Wrap api-client calls in: "dev.failsafe" % "failsafe" % "3.2.4",
 
   "org.scalatest" %% "scalatest" % "3.2.16" % "test",
   "org.scalatestplus" %% "scalacheck-1-17" % "3.2.16.0" % "test",
 )
 
+val nexus = "https://s01.oss.sonatype.org/"
+lazy val publishSettings = Seq(
+  publishTo := {
+    if (isSnapshot.value) Some("snapshots" at nexus + "content/repositories/snapshots")
+    else Some("releases" at nexus + "service/local/staging/deploy/maven2")
+  },
+  // Remove all additional repository other than Maven Central from POM
+  pomIncludeRepository := { _ => false },
+  credentials ++= Seq(
+    Credentials(Path.userHome / ".sbt" / "credentials"),
+    Credentials(Path.userHome / ".sbt" / "sonatype_credentials")
+  )
+)
+
+ThisBuild / isSnapshot := false
 ThisBuild / versionScheme := Some("early-semver")
 ThisBuild / organization := "io.lakefs"
 ThisBuild / organizationName := "Treeverse Labs"

project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.9.3

project/pgp-sign.sbt

@@ -0,0 +1,6 @@
+credentials += Credentials(
+  "GnuPG Key ID",
+  "gpg",
+  "F638279A7281EE8EAE58C4B98D3CFE7E7D0262DA", // key identifier
+  "ignored" // this field is ignored; passwords are supplied by pinentry
+)

project/plugins.sbt

@@ -0,0 +1,2 @@
+addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.11.0")
+addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2")

src/test/scala/io/lakefs/iceberg/extension/ExtensionSpec.scala

@@ -0,0 +1,36 @@
+package io.lakefs.iceberg.extension
+
+import org.scalatest.funspec.AnyFunSpec
+import org.scalatest.matchers.should
+
+import org.apache.spark.sql.Row
+
+class ExtensionSpec extends AnyFunSpec
+    with SparkSessionWithExtensionSetup[LakeFSSparkSessionExtensions]
+    with should.Matchers {
+
+  val _ = new LakeFSSparkSessionExtensions()
+
+  describe("Extension") {
+    it("should SELECT") {
+      withSparkSession(spark => {
+        import spark.implicits._
+
+        spark.sql("CREATE DATABASE first")
+        val df = Seq(("a", 1), ("b", 2), ("c", 3)).toDF
+        df.writeTo("spark_catalog.first.table").create()
+
+        spark.sql("CREATE DATABASE second")
+        val df2 = Seq(("a", 1), ("xyzzy", 2), ("c", 3), ("d", 4)).toDF
+        df2.writeTo("spark_catalog.second.table").create()
+
+        val diff = spark.sql("SELECT * FROM schema_diff('spark_catalog', 'first', 'second', 'table')")
+          .collect()
+          .toSet
+        diff should equal(Set(Row("-", "b", 2), Row("+", "xyzzy", 2), Row("+", "d", 4)))
+      })
+    }
+
+    // TODO(ariels): Test SQL identifier quotation.
+  }
+}

src/test/scala/io/lakefs/iceberg/extension/SparkSessionSetup.scala

@@ -0,0 +1,34 @@
+package io.lakefs.iceberg.extension
+
+import java.nio.file.Files
+import org.apache.commons.io.FileUtils
+import scala.reflect.ClassTag
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.SparkConf
+
+trait SparkSessionWithExtensionSetup[E] {
+  def withSparkSession(testMethod: (SparkSession) => Any)(implicit tag: ClassTag[E]) {
+    val tmpDir = Files.createTempDirectory("sparktest").toString
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("Spark test")
+      .set("spark.sql.extensions", tag.runtimeClass.getCanonicalName)
+      .set("spark.sql.warehouse.dir", tmpDir)
+    val spark = new SparkSession.Builder()
+      .appName("extension-test")
+      .config(conf)
+      .enableHiveSupport()
+      .getOrCreate
+    try {
+      testMethod(spark)
+    } finally {
+      // Clean up catalog dir
+      FileUtils.deleteDirectory(new java.io.File(tmpDir))
+      // local metastore_db always created in current working directory, and
+      // current working directory cannot be changed in Java.
+      FileUtils.deleteDirectory(new java.io.File("./metastore_db"))
+      spark.close()
+    }
+  }
+}