-
Notifications
You must be signed in to change notification settings - Fork 80
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat!: Updated API for Iceberg read operations (#6268)
- Loading branch information
1 parent
fa27a8e
commit 703f527
Showing
11 changed files
with
313 additions
and
336 deletions.
There are no files selected for viewing
82 changes: 42 additions & 40 deletions
82
extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
81 changes: 0 additions & 81 deletions
81
extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergInstructions.java
This file was deleted.
Oops, something went wrong.
115 changes: 115 additions & 0 deletions
115
extensions/iceberg/src/main/java/io/deephaven/iceberg/util/IcebergReadInstructions.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| // | ||
| // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
| // | ||
| package io.deephaven.iceberg.util; | ||
|
|
||
| import io.deephaven.annotations.CopyableStyle; | ||
| import io.deephaven.engine.table.TableDefinition; | ||
| import org.apache.iceberg.Snapshot; | ||
| import org.immutables.value.Value; | ||
| import org.immutables.value.Value.Immutable; | ||
|
|
||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.OptionalLong; | ||
|
|
||
| /** | ||
| * This class provides instructions intended for reading Iceberg catalogs and tables. The default values documented in | ||
| * this class may change in the future. As such, callers may wish to explicitly set the values. | ||
| */ | ||
| @Immutable | ||
| @CopyableStyle | ||
| public abstract class IcebergReadInstructions { | ||
| /** | ||
| * The default {@link IcebergReadInstructions} to use when reading Iceberg data files. Providing this will use | ||
| * system defaults for cloud provider-specific parameters | ||
| */ | ||
| @SuppressWarnings("unused") | ||
| public static final IcebergReadInstructions DEFAULT = builder().build(); | ||
|
|
||
| public static Builder builder() { | ||
| return ImmutableIcebergReadInstructions.builder(); | ||
| } | ||
|
|
||
| /** | ||
| * The {@link TableDefinition} to use when reading Iceberg data files. | ||
| */ | ||
| public abstract Optional<TableDefinition> tableDefinition(); | ||
|
|
||
| /** | ||
| * The data instructions to use for reading the Iceberg data files (might be S3Instructions or other cloud | ||
| * provider-specific instructions). | ||
| */ | ||
| public abstract Optional<Object> dataInstructions(); | ||
|
|
||
| /** | ||
| * A {@link Map map} of rename instructions from Iceberg to Deephaven column names to use when reading the Iceberg | ||
| * data files. | ||
| */ | ||
| public abstract Map<String, String> columnRenames(); | ||
|
|
||
| /** | ||
| * Return a copy of this instructions object with the column renames replaced by {@code entries}. | ||
| */ | ||
| public abstract IcebergReadInstructions withColumnRenames(Map<String, ? extends String> entries); | ||
|
|
||
| /** | ||
| * The {@link IcebergUpdateMode} mode to use when reading the Iceberg data files. Default is | ||
| * {@link IcebergUpdateMode#staticMode()}. | ||
| */ | ||
| @Value.Default | ||
| public IcebergUpdateMode updateMode() { | ||
| return IcebergUpdateMode.staticMode(); | ||
| } | ||
|
|
||
| /** | ||
| * The identifier of the snapshot to load for reading. If both this and {@link #snapshot()} are provided, the | ||
| * {@link Snapshot#snapshotId()} should match this. Otherwise, only one of them should be provided. If neither is | ||
| * provided, the latest snapshot will be loaded. | ||
| */ | ||
| public abstract OptionalLong snapshotId(); | ||
|
|
||
| /** | ||
| * Return a copy of this instructions object with the snapshot ID replaced by {@code value}. | ||
| */ | ||
| public abstract IcebergReadInstructions withSnapshotId(long value); | ||
|
|
||
| /** | ||
| * The snapshot to load for reading. If both this and {@link #snapshotId()} are provided, the | ||
| * {@link Snapshot#snapshotId()} should match the {@link #snapshotId()}. Otherwise, only one of them should be | ||
| * provided. If neither is provided, the latest snapshot will be loaded. | ||
| */ | ||
| public abstract Optional<Snapshot> snapshot(); | ||
|
|
||
| /** | ||
| * Return a copy of this instructions object with the snapshot replaced by {@code value}. | ||
| */ | ||
| public abstract IcebergReadInstructions withSnapshot(Snapshot value); | ||
|
|
||
| public interface Builder { | ||
| Builder tableDefinition(TableDefinition tableDefinition); | ||
|
|
||
| Builder dataInstructions(Object s3Instructions); | ||
|
|
||
| Builder putColumnRenames(String key, String value); | ||
|
|
||
| Builder putAllColumnRenames(Map<String, ? extends String> entries); | ||
|
|
||
| Builder updateMode(IcebergUpdateMode updateMode); | ||
|
|
||
| Builder snapshotId(long snapshotId); | ||
|
|
||
| Builder snapshot(Snapshot snapshot); | ||
|
|
||
| IcebergReadInstructions build(); | ||
| } | ||
|
|
||
| @Value.Check | ||
| final void checkSnapshotId() { | ||
| if (snapshotId().isPresent() && snapshot().isPresent() && | ||
| snapshotId().getAsLong() != snapshot().get().snapshotId()) { | ||
| throw new IllegalArgumentException("If both snapshotID and snapshot are provided, the snapshot Ids " + | ||
| "must match, found " + snapshotId().getAsLong() + " and " + snapshot().get().snapshotId()); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.