Initial commit of Iceberg integration. (deephaven#5277)

* Initial commit of Iceberg integration.

* Tests simplified and passing.

* Exposing 'iceberg-aws' in gradle.

* Addressing PR comments.

buildSrc/src/main/groovy/Classpaths.groovy

@@ -123,6 +123,18 @@ class Classpaths {
  static final String GUAVA_NAME = 'guava'
  static final String GUAVA_VERSION = '33.2.0-jre'
 
+ static final String HADOOP_GROUP = 'org.apache.hadoop'
+ static final String HADOOP_VERSION = '3.4.0'
+
+ static final String ICEBERG_GROUP = 'org.apache.iceberg'
+ static final String ICEBERG_VERSION = '1.5.0'
+
+ static final String AWSSDK_GROUP = 'software.amazon.awssdk'
+ static final String AWSSDK_VERSION = '2.23.19'
+
+ static final String TESTCONTAINER_GROUP = 'org.testcontainers'
+ static final String TESTCONTAINER_VERSION = '1.19.4'
+
  static boolean addDependency(Configuration conf, String group, String name, String version, Action<? super DefaultExternalModuleDependency> configure = Actions.doNothing()) {
  if (!conf.dependencies.find { it.name == name && it.group == group}) {
  DefaultExternalModuleDependency dep = dependency group, name, version
@@ -295,7 +307,7 @@ class Classpaths {
  /** configName controls only the Configuration's classpath, all transitive dependencies are runtimeOnly */
  static void inheritParquetHadoopConfiguration(Project p, String configName = JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME) {
  Configuration config = p.configurations.getByName(configName)
- addDependency(config, 'org.apache.hadoop', 'hadoop-common', '3.4.0') {
+ addDependency(config, HADOOP_GROUP, 'hadoop-common', HADOOP_VERSION) {
  it.setTransitive(false)
  // Do not take any extra dependencies of this project transitively. We just want a few classes for
  // configuration and compression codecs. For any additional required dependencies, add them separately, as
@@ -314,4 +326,35 @@ class Classpaths {
  it.because('hadoop-common required dependency for Configuration')
  }
  }
+
+ static void inheritIcebergHadoop(Project p, String configName = JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME) {
+ Configuration config = p.configurations.getByName(configName)
+ addDependency(config, HADOOP_GROUP, 'hadoop-common', HADOOP_VERSION)
+ addDependency(config, HADOOP_GROUP, 'hadoop-hdfs-client', HADOOP_VERSION)
+ }
+
+
+ static void inheritIcebergCore(Project p, String configName = JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME) {
+ Configuration config = p.configurations.getByName(configName)
+ addDependency(config, p.getDependencies().platform(ICEBERG_GROUP + ":iceberg-bom:" + ICEBERG_VERSION))
+
+ addDependency(config, ICEBERG_GROUP, 'iceberg-core', ICEBERG_VERSION)
+ addDependency(config, ICEBERG_GROUP, 'iceberg-bundled-guava', ICEBERG_VERSION)
+ }
+
+ static void inheritAWSSDK(Project p, String configName = JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME) {
+ Configuration config = p.configurations.getByName(configName)
+ addDependency(config, p.getDependencies().platform(AWSSDK_GROUP + ":bom:" + AWSSDK_VERSION))
+
+ addDependency(config, AWSSDK_GROUP, 's3', AWSSDK_VERSION)
+ addDependency(config, AWSSDK_GROUP, 'aws-crt-client', AWSSDK_VERSION)
+ }
+
+ static void inheritTestContainers(Project p, String configName = JavaPlugin.TEST_IMPLEMENTATION_CONFIGURATION_NAME) {
+ Configuration config = p.configurations.getByName(configName)
+ addDependency(config, TESTCONTAINER_GROUP, 'testcontainers', TESTCONTAINER_VERSION)
+ addDependency(config, TESTCONTAINER_GROUP, 'junit-jupiter', TESTCONTAINER_VERSION)
+ addDependency(config, TESTCONTAINER_GROUP, 'localstack', TESTCONTAINER_VERSION)
+ addDependency(config, TESTCONTAINER_GROUP, 'minio', TESTCONTAINER_VERSION)
+ }
 }

extensions/iceberg/build.gradle

@@ -0,0 +1,39 @@
+plugins {
+ id 'java-library'
+ id 'io.deephaven.project.register'
+}
+
+description 'Iceberg: Support to read iceberg catalogs.'
+
+dependencies {
+ api project(':engine-api')
+ api project(':engine-table')
+
+ implementation project(':engine-base')
+ implementation project(':log-factory')
+ implementation project(':Configuration')
+
+ Classpaths.inheritAutoService(project)
+ Classpaths.inheritImmutables(project)
+
+ Classpaths.inheritParquetHadoop(project)
+
+ implementation project(':extensions-parquet-base')
+ implementation project(':extensions-parquet-table')
+
+ Classpaths.inheritIcebergCore(project)
+ Classpaths.inheritIcebergHadoop(project)
+
+ Classpaths.inheritJUnitPlatform(project)
+ Classpaths.inheritAssertJ(project)
+
+ testImplementation 'org.junit.jupiter:junit-jupiter'
+ testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
+ testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+
+ Classpaths.inheritTestContainers(project)
+
+ testRuntimeOnly project(':test-configs')
+ testRuntimeOnly project(':log-to-slf4j')
+ Classpaths.inheritSlf4j(project, 'slf4j-simple', 'testRuntimeOnly')
+}

extensions/iceberg/gradle.properties

@@ -0,0 +1 @@
+io.deephaven.project.ProjectType=JAVA_PUBLIC

extensions/iceberg/s3/build.gradle

@@ -0,0 +1,41 @@
+plugins {
+ id 'java-library'
+ id 'io.deephaven.project.register'
+}
+
+description 'Iceberg: Support to read iceberg catalogs.'
+
+dependencies {
+ implementation project(':extensions-iceberg')
+
+ // Bring in the AWS / S3 extensions
+ Classpaths.inheritIcebergCore(project)
+
+ implementation project(':extensions-s3')
+ implementation "org.apache.iceberg:iceberg-aws"
+ runtimeOnly "org.apache.iceberg:iceberg-aws-bundle"
+ Classpaths.inheritAWSSDK(project)
+
+ Classpaths.inheritTestContainers(project)
+
+ testImplementation TestTools.projectDependency(project, 'extensions-s3')
+ testImplementation TestTools.projectDependency(project, 'extensions-iceberg')
+
+ testRuntimeOnly project(':test-configs')
+ testRuntimeOnly project(':log-to-slf4j')
+ Classpaths.inheritSlf4j(project, 'slf4j-simple', 'testRuntimeOnly')
+}
+
+test {
+ useJUnitPlatform {
+ excludeTags("testcontainers")
+ }
+}
+
+tasks.register('testOutOfBand', Test) {
+ useJUnitPlatform {
+ includeTags("testcontainers")
+ }
+ systemProperty 'testcontainers.localstack.image', project.property('testcontainers.localstack.image')
+ systemProperty 'testcontainers.minio.image', project.property('testcontainers.minio.image')
+}

extensions/iceberg/s3/gradle.properties

@@ -0,0 +1,4 @@
+io.deephaven.project.ProjectType=JAVA_PUBLIC
+
+testcontainers.localstack.image=localstack/localstack:3.1.0
+testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z

extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java

@@ -0,0 +1,65 @@
+//
+// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.iceberg.util;
+
+import com.google.common.base.Strings;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.aws.AwsClientProperties;
+import org.apache.iceberg.aws.s3.S3FileIOProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.rest.RESTCatalog;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tools for accessing tables in the Iceberg table format.
+ */
+@SuppressWarnings("unused")
+public class IcebergToolsS3 extends IcebergTools {
+ private static final String S3_FILE_IO_CLASS = "org.apache.iceberg.aws.s3.S3FileIO";
+
+ public static IcebergCatalogAdapter createS3Rest(
+ @Nullable final String name,
+ @NotNull final String catalogURI,
+ @NotNull final String warehouseLocation,
+ @Nullable final String region,
+ @Nullable final String accessKeyId,
+ @Nullable final String secretAccessKey,
+ @Nullable final String endpointOverride) {
+
+ // Set up the properties map for the Iceberg catalog
+ final Map<String, String> properties = new HashMap<>();
+
+ final RESTCatalog catalog = new RESTCatalog();
+
+ properties.put(CatalogProperties.CATALOG_IMPL, catalog.getClass().getName());
+ properties.put(CatalogProperties.URI, catalogURI);
+ properties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation);
+
+ // Configure the properties map from the Iceberg instructions.
+ if (!Strings.isNullOrEmpty(accessKeyId) && !Strings.isNullOrEmpty(secretAccessKey)) {
+ properties.put(S3FileIOProperties.ACCESS_KEY_ID, accessKeyId);
+ properties.put(S3FileIOProperties.SECRET_ACCESS_KEY, secretAccessKey);
+ }
+ if (!Strings.isNullOrEmpty(region)) {
+ properties.put(AwsClientProperties.CLIENT_REGION, region);
+ }
+ if (!Strings.isNullOrEmpty(endpointOverride)) {
+ properties.put(S3FileIOProperties.ENDPOINT, endpointOverride);
+ }
+
+ // TODO: create a FileIO interface wrapping the Deephaven S3SeekableByteChannel/Provider
+ final FileIO fileIO = CatalogUtil.loadFileIO(S3_FILE_IO_CLASS, properties, null);
+
+ final String catalogName = name != null ? name : "IcebergCatalog-" + catalogURI;
+ catalog.initialize(catalogName, properties);
+
+ return new IcebergCatalogAdapter(catalog, fileIO);
+ }
+
+}

extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergLocalStackTest.java

@@ -0,0 +1,31 @@
+//
+// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.iceberg.util;
+
+
+import io.deephaven.extensions.s3.S3Instructions.Builder;
+import io.deephaven.extensions.s3.testlib.SingletonContainers;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+
+@Tag("testcontainers")
+public class IcebergLocalStackTest extends IcebergToolsTest {
+
+ @BeforeAll
+ static void initContainer() {
+ // ensure container is started so container startup time isn't associated with a specific test
+ SingletonContainers.LocalStack.init();
+ }
+
+ @Override
+ public Builder s3Instructions(Builder builder) {
+ return SingletonContainers.LocalStack.s3Instructions(builder);
+ }
+
+ @Override
+ public S3AsyncClient s3AsyncClient() {
+ return SingletonContainers.LocalStack.s3AsyncClient();
+ }
+}

extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergMinIOTest.java

@@ -0,0 +1,35 @@
+//
+// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.iceberg.util;
+
+
+import io.deephaven.extensions.s3.S3Instructions.Builder;
+import io.deephaven.extensions.s3.testlib.SingletonContainers;
+import io.deephaven.stats.util.OSUtil;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+
+@Tag("testcontainers")
+public class IcebergMinIOTest extends IcebergToolsTest {
+
+ @BeforeAll
+ static void initContainer() {
+ // TODO(deephaven-core#5116): MinIO testcontainers does not work on OS X
+ Assumptions.assumeFalse(OSUtil.runningMacOS(), "OSUtil.runningMacOS()");
+ // ensure container is started so container startup time isn't associated with a specific test
+ SingletonContainers.MinIO.init();
+ }
+
+ @Override
+ public Builder s3Instructions(Builder builder) {
+ return SingletonContainers.MinIO.s3Instructions(builder);
+ }
+
+ @Override
+ public S3AsyncClient s3AsyncClient() {
+ return SingletonContainers.MinIO.s3AsyncClient();
+ }
+}