Added additional snapshots with removes to IcebergToolsTest resources.

deephaven · Sep 3, 2024 · 273f5c1 · 273f5c1
1 parent 68e4546
commit 273f5c1
Show file tree

Hide file tree

Showing 28 changed files with 633 additions and 171 deletions.
diff --git a/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java b/extensions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/IcebergToolsTest.java
@@ -235,20 +235,22 @@ public void testListSnapshots() {
  adapter.listSnapshots(tableIdentifier)
  .forEach(snapshot -> snapshotIds.add(snapshot.snapshotId()));
 
- Assert.eq(snapshotIds.size(), "snapshots.size()", 4, "4 snapshots for sales/sales_multi");
+ Assert.eq(snapshotIds.size(), "snapshots.size()", 6, "6 snapshots for sales/sales_multi");
 
- Assert.eqTrue(snapshotIds.contains(2001582482032951248L), "snapshots.contains(2001582482032951248)");
- Assert.eqTrue(snapshotIds.contains(8325605756612719366L), "snapshots.contains(8325605756612719366L)");
- Assert.eqTrue(snapshotIds.contains(3247344357341484163L), "snapshots.contains(3247344357341484163L)");
- Assert.eqTrue(snapshotIds.contains(1792185872197984875L), "snapshots.contains(1792185872197984875L)");
+ Assert.eqTrue(snapshotIds.contains(8121674823173822790L), "snapshots.contains(8121674823173822790L)");
+ Assert.eqTrue(snapshotIds.contains(6040260770188877244L), "snapshots.contains(6040260770188877244L)");
+ Assert.eqTrue(snapshotIds.contains(5693547373255393922L), "snapshots.contains(5693547373255393922L)");
+ Assert.eqTrue(snapshotIds.contains(3445166522731196553L), "snapshots.contains(3445166522731196553L)");
+ Assert.eqTrue(snapshotIds.contains(1277776933184906785L), "snapshots.contains(1277776933184906785L)");
+ Assert.eqTrue(snapshotIds.contains(3825168261540020388L), "snapshots.contains(3825168261540020388L)");
 
  Table table = adapter.snapshots(tableIdentifier);
- Assert.eq(table.size(), "table.size()", 4, "4 snapshots for sales/sales_multi");
+ Assert.eq(table.size(), "table.size()", 6, "6 snapshots for sales/sales_multi");
  Assert.equals(table.getDefinition(), "table.getDefinition()", SNAPSHOT_DEFINITION);
 
  // Test the string versions of the methods
  table = adapter.snapshots("sales.sales_multi");
- Assert.eq(table.size(), "table.size()", 4, "4 snapshots for sales/sales_multi");
+ Assert.eq(table.size(), "table.size()", 6, "6 snapshots for sales/sales_multi");
  Assert.equals(table.getDefinition(), "table.getDefinition()", SNAPSHOT_DEFINITION);
  }
 
@@ -286,14 +288,15 @@ public void testOpenTableB() throws ExecutionException, InterruptedException, Ti
  final TableIdentifier tableId = TableIdentifier.of(ns, "sales_multi");
  io.deephaven.engine.table.Table table = adapter.readTable(tableId, instructions);
 
- Assert.eq(table.size(), "table.size()", 100_000, "100_000 rows in the table");
+ // This table ends up with zero records
+ Assert.eq(table.size(), "table.size()", 0, "0 rows in the table");
  Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
 
  // Test the string versions of the methods
  table = adapter.readTable("sales.sales_multi", instructions);
 
  // Verify we retrieved all the rows.
- Assert.eq(table.size(), "table.size()", 100_000, "100_000 rows in the table");
+ Assert.eq(table.size(), "table.size()", 0, "0 rows in the table");
  Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
  }
 
@@ -729,23 +732,33 @@ public void testOpenTableSnapshot() throws ExecutionException, InterruptedExcept
  // Verify we retrieved all the rows.
  final io.deephaven.engine.table.Table table0 =
  adapter.readTable(tableId, snapshots.get(0).snapshotId(), instructions);
- Assert.eq(table0.size(), "table0.size()", 18266, "18266 rows in the table");
+ Assert.eq(table0.size(), "table0.size()", 18073, "18266 rows in the table");
  Assert.equals(table0.getDefinition(), "table0.getDefinition()", SALES_MULTI_DEFINITION);
 
  final io.deephaven.engine.table.Table table1 =
  adapter.readTable(tableId, snapshots.get(1).snapshotId(), instructions);
- Assert.eq(table1.size(), "table1.size()", 54373, "54373 rows in the table");
+ Assert.eq(table1.size(), "table1.size()", 54433, "54373 rows in the table");
  Assert.equals(table1.getDefinition(), "table1.getDefinition()", SALES_MULTI_DEFINITION);
 
  final io.deephaven.engine.table.Table table2 =
  adapter.readTable(tableId, snapshots.get(2).snapshotId(), instructions);
- Assert.eq(table2.size(), "table2.size()", 72603, "72603 rows in the table");
+ Assert.eq(table2.size(), "table2.size()", 72551, "72603 rows in the table");
  Assert.equals(table2.getDefinition(), "table2.getDefinition()", SALES_MULTI_DEFINITION);
 
  final io.deephaven.engine.table.Table table3 =
  adapter.readTable(tableId, snapshots.get(3).snapshotId(), instructions);
  Assert.eq(table3.size(), "table3.size()", 100_000, "100_000 rows in the table");
  Assert.equals(table3.getDefinition(), "table3.getDefinition()", SALES_MULTI_DEFINITION);
+
+ final io.deephaven.engine.table.Table table4 =
+ adapter.readTable(tableId, snapshots.get(4).snapshotId(), instructions);
+ Assert.eq(table4.size(), "table4.size()", 100_000, "100_000 rows in the table");
+ Assert.equals(table4.getDefinition(), "table4.getDefinition()", SALES_MULTI_DEFINITION);
+
+ final io.deephaven.engine.table.Table table5 =
+ adapter.readTable(tableId, snapshots.get(5).snapshotId(), instructions);
+ Assert.eq(table5.size(), "table5.size()", 0, "0 rows in the table");
+ Assert.equals(table5.getDefinition(), "table5.getDefinition()", SALES_MULTI_DEFINITION);
  }
 
  @Test
@@ -760,39 +773,55 @@ public void testOpenTableSnapshotByID() throws ExecutionException, InterruptedEx
 
  // Verify we retrieved all the rows.
  io.deephaven.engine.table.Table table0 = adapter.readTable(tableId, snapshots.get(0), instructions);
- Assert.eq(table0.size(), "table0.size()", 18266, "18266 rows in the table");
+ Assert.eq(table0.size(), "table0.size()", 18073, "18073 rows in the table");
  Assert.equals(table0.getDefinition(), "table0.getDefinition()", SALES_MULTI_DEFINITION);
 
  io.deephaven.engine.table.Table table1 = adapter.readTable(tableId, snapshots.get(1), instructions);
- Assert.eq(table1.size(), "table1.size()", 54373, "54373 rows in the table");
+ Assert.eq(table1.size(), "table1.size()", 54433, "54433 rows in the table");
  Assert.equals(table1.getDefinition(), "table1.getDefinition()", SALES_MULTI_DEFINITION);
 
  io.deephaven.engine.table.Table table2 = adapter.readTable(tableId, snapshots.get(2), instructions);
- Assert.eq(table2.size(), "table2.size()", 72603, "72603 rows in the table");
+ Assert.eq(table2.size(), "table2.size()", 72551, "72551 rows in the table");
  Assert.equals(table2.getDefinition(), "table2.getDefinition()", SALES_MULTI_DEFINITION);
 
  io.deephaven.engine.table.Table table3 = adapter.readTable(tableId, snapshots.get(3), instructions);
  Assert.eq(table3.size(), "table3.size()", 100_000, "100_000 rows in the table");
  Assert.equals(table3.getDefinition(), "table3.getDefinition()", SALES_MULTI_DEFINITION);
 
+ io.deephaven.engine.table.Table table4 = adapter.readTable(tableId, snapshots.get(4), instructions);
+ Assert.eq(table4.size(), "table4.size()", 100_000, "100_000 rows in the table");
+ Assert.equals(table4.getDefinition(), "table4.getDefinition()", SALES_MULTI_DEFINITION);
+
+ io.deephaven.engine.table.Table table5 = adapter.readTable(tableId, snapshots.get(5), instructions);
+ Assert.eq(table5.size(), "table5.size()", 0, "0 rows in the table");
+ Assert.equals(table5.getDefinition(), "table5.getDefinition()", SALES_MULTI_DEFINITION);
+
  // Test the string versions of the methods
 
  // Verify we retrieved all the rows.
  table0 = adapter.readTable("sales.sales_multi", snapshots.get(0).snapshotId(), instructions);
- Assert.eq(table0.size(), "table0.size()", 18266, "18266 rows in the table");
+ Assert.eq(table0.size(), "table0.size()", 18073, "18073 rows in the table");
  Assert.equals(table0.getDefinition(), "table0.getDefinition()", SALES_MULTI_DEFINITION);
 
- table1 = adapter.readTable(tableId, snapshots.get(1).snapshotId(), instructions);
- Assert.eq(table1.size(), "table1.size()", 54373, "54373 rows in the table");
+ table1 = adapter.readTable("sales.sales_multi", snapshots.get(1).snapshotId(), instructions);
+ Assert.eq(table1.size(), "table1.size()", 54433, "54433 rows in the table");
  Assert.equals(table1.getDefinition(), "table1.getDefinition()", SALES_MULTI_DEFINITION);
 
- table2 = adapter.readTable(tableId, snapshots.get(2).snapshotId(), instructions);
- Assert.eq(table2.size(), "table2.size()", 72603, "72603 rows in the table");
+ table2 = adapter.readTable("sales.sales_multi", snapshots.get(2).snapshotId(), instructions);
+ Assert.eq(table2.size(), "table2.size()", 72551, "72551 rows in the table");
  Assert.equals(table2.getDefinition(), "table2.getDefinition()", SALES_MULTI_DEFINITION);
 
- table3 = adapter.readTable(tableId, snapshots.get(3).snapshotId(), instructions);
+ table3 = adapter.readTable("sales.sales_multi", snapshots.get(3).snapshotId(), instructions);
  Assert.eq(table3.size(), "table3.size()", 100_000, "100_000 rows in the table");
  Assert.equals(table3.getDefinition(), "table0.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table4 = adapter.readTable("sales.sales_multi", snapshots.get(4).snapshotId(), instructions);
+ Assert.eq(table4.size(), "table4.size()", 100_000, "100_000 rows in the table");
+ Assert.equals(table4.getDefinition(), "table4.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table5 = adapter.readTable("sales.sales_multi", snapshots.get(5).snapshotId(), instructions);
+ Assert.eq(table5.size(), "table5.size()", 0, "0 rows in the table");
+ Assert.equals(table5.getDefinition(), "table5.getDefinition()", SALES_MULTI_DEFINITION);
  }
 
  @Test
@@ -908,4 +937,43 @@ public void testTableDefinitionWithInstructions() {
  tableDef = adapter.getTableDefinition("sales.sales_multi", localInstructions);
  Assert.equals(tableDef, "tableDef", userTableDef);
  }
+
+ @Test
+ public void testManualRefreshingTable() {
+ final IcebergCatalogAdapter adapter = IcebergTools.createAdapter(resourceCatalog);
+
+ final IcebergInstructions localInstructions = IcebergInstructions.builder()
+ .dataInstructions(instructions.dataInstructions().get())
+ .updateMode(IcebergUpdateMode.manualRefreshingMode())
+ .build();
+
+ final IcebergTableAdapter tableAdapter = adapter.loadTable("sales.sales_multi");
+
+ final List<Snapshot> snapshots = tableAdapter.listSnapshots();
+
+ // Use string and current snapshot
+ final IcebergTable table = tableAdapter.table(snapshots.get(0).snapshotId(), localInstructions);
+ Assert.eq(table.size(), "table.size()", 18073, "18073 rows in the table");
+ Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table.update(snapshots.get(1).snapshotId());
+ Assert.eq(table.size(), "table.size()", 54433, "54433 rows in the table");
+ Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table.update(snapshots.get(2).snapshotId());
+ Assert.eq(table.size(), "table.size()", 72551, "72551 rows in the table");
+ Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table.update(snapshots.get(3).snapshotId());
+ Assert.eq(table.size(), "table.size()", 100_000, "100_000 rows in the table");
+ Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table.update(snapshots.get(4).snapshotId());
+ Assert.eq(table.size(), "table.size()", 100_000, "100_000 rows in the table");
+ Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
+
+ table.update(snapshots.get(5).snapshotId());
+ Assert.eq(table.size(), "table.size()", 0, "0 rows in the table");
+ Assert.equals(table.getDefinition(), "table.getDefinition()", SALES_MULTI_DEFINITION);
+ }
 }
diff --git a/...house/sales/sales_multi/data/00000-2-6f8d12ab-3efb-4a24-a6a6-46b2b34b986c-0-00001.parquet b/...house/sales/sales_multi/data/00000-2-6f8d12ab-3efb-4a24-a6a6-46b2b34b986c-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-3-57e22ac8-7e7d-4830-aedb-14dab6fabdd6-0-00001.parquet b/...house/sales/sales_multi/data/00000-3-57e22ac8-7e7d-4830-aedb-14dab6fabdd6-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-3-75fe700e-5fc0-40f8-9349-fcd5763dd691-0-00001.parquet b/...house/sales/sales_multi/data/00000-3-75fe700e-5fc0-40f8-9349-fcd5763dd691-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-4-3aacc74b-7f10-4e08-89ae-f1b1b578ce63-0-00001.parquet b/...house/sales/sales_multi/data/00000-4-3aacc74b-7f10-4e08-89ae-f1b1b578ce63-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-4-4be14704-ebcf-44c9-8482-90d09621c2ee-0-00001.parquet b/...house/sales/sales_multi/data/00000-4-4be14704-ebcf-44c9-8482-90d09621c2ee-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-5-47debf3c-b256-4a04-9248-d7c69bec4881-0-00001.parquet b/...house/sales/sales_multi/data/00000-5-47debf3c-b256-4a04-9248-d7c69bec4881-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-5-efdf14cf-15de-4aac-bafc-cf750f9f7f70-0-00001.parquet b/...house/sales/sales_multi/data/00000-5-efdf14cf-15de-4aac-bafc-cf750f9f7f70-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-6-01e5d979-ab08-46a6-bd9d-c8008dc66519-0-00001.parquet b/...house/sales/sales_multi/data/00000-6-01e5d979-ab08-46a6-bd9d-c8008dc66519-0-00001.parquet
diff --git a/...house/sales/sales_multi/data/00000-6-7860a7c9-b3cb-4b2a-a69b-0f1e5c068512-0-00001.parquet b/...house/sales/sales_multi/data/00000-6-7860a7c9-b3cb-4b2a-a69b-0f1e5c068512-0-00001.parquet
diff --git a/...a414-468b-b7b2-78558b2e8c1f.metadata.json → ...9c89-46e8-b0f7-b4ec128b0f5e.metadata.json b/...a414-468b-b7b2-78558b2e8c1f.metadata.json → ...9c89-46e8-b0f7-b4ec128b0f5e.metadata.json
@@ -1,9 +1,9 @@
 {
  "format-version" : 2,
- "table-uuid" : "49ca5b8c-6402-40f9-96b0-4c09f9f1b512",
+ "table-uuid" : "533f8120-9943-4b64-a0cf-6cfab18e4a79",
  "location" : "s3://warehouse/sales/sales_multi",
  "last-sequence-number" : 1,
- "last-updated-ms" : 1716234545155,
+ "last-updated-ms" : 1725396294486,
  "last-column-id" : 5,
  "current-schema-id" : 0,
  "schemas" : [ {
@@ -49,43 +49,43 @@
  } ],
  "properties" : {
  "owner" : "root",
- "created-at" : "2024-05-20T19:49:04.669537174Z",
+ "created-at" : "2024-09-03T20:44:52.796187633Z",
  "write.format.default" : "parquet",
  "write.parquet.compression-codec" : "zstd"
  },
- "current-snapshot-id" : 2001582482032951248,
+ "current-snapshot-id" : 8121674823173822790,
  "refs" : {
  "main" : {
- "snapshot-id" : 2001582482032951248,
+ "snapshot-id" : 8121674823173822790,
  "type" : "branch"
  }
  },
  "snapshots" : [ {
  "sequence-number" : 1,
- "snapshot-id" : 2001582482032951248,
- "timestamp-ms" : 1716234545155,
+ "snapshot-id" : 8121674823173822790,
+ "timestamp-ms" : 1725396294486,
  "summary" : {
  "operation" : "append",
- "spark.app.id" : "local-1716234462547",
+ "spark.app.id" : "local-1725396284766",
  "added-data-files" : "1",
- "added-records" : "18266",
- "added-files-size" : "176970",
+ "added-records" : "18073",
+ "added-files-size" : "175359",
  "changed-partition-count" : "1",
- "total-records" : "18266",
- "total-files-size" : "176970",
+ "total-records" : "18073",
+ "total-files-size" : "175359",
  "total-data-files" : "1",
  "total-delete-files" : "0",
  "total-position-deletes" : "0",
  "total-equality-deletes" : "0"
  },
- "manifest-list" : "s3://warehouse/sales/sales_multi/metadata/snap-2001582482032951248-1-a61cde47-0ced-4b74-b91a-9e32dd59b212.avro",
+ "manifest-list" : "s3://warehouse/sales/sales_multi/metadata/snap-8121674823173822790-1-ddb2a84e-df73-4cb9-a92f-18d737a54489.avro",
  "schema-id" : 0
  } ],
  "statistics" : [ ],
  "partition-statistics" : [ ],
  "snapshot-log" : [ {
- "timestamp-ms" : 1716234545155,
- "snapshot-id" : 2001582482032951248
+ "timestamp-ms" : 1725396294486,
+ "snapshot-id" : 8121674823173822790
  } ],
  "metadata-log" : [ ]
 }