Reference commit for generating test files for deephaven#6183

malhotrashivam · Oct 9, 2024 · ef910dc · ef910dc
1 parent e4c1ddc
commit ef910dc
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 11 deletions.
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriterImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriterImpl.java
@@ -67,6 +67,8 @@ final class ColumnWriterImpl implements ColumnWriter {
 
     private final EncodingStats.Builder encodingStatsBuilder = new EncodingStats.Builder();
 
+    private static int counter = 0;
+
     ColumnWriterImpl(
             final RowGroupWriterImpl owner,
             final CountingOutputStream countingOutput,
@@ -88,6 +90,12 @@ final class ColumnWriterImpl implements ColumnWriter {
         this.owner = owner;
         offsetIndexBuilder = OffsetIndexBuilder.getBuilder();
         statistics = Statistics.createStats(column.getPrimitiveType());
+        if (counter == 1) {
+            dictionaryOffset = firstDataPageOffset = 42;
+            // ^ This page offset corresponds to the end of first row group. To compute this, set it to 0 and look at
+            // the assertion failure.
+        }
+        counter++;
     }
 
     @Override

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -212,6 +212,8 @@ private static void write(
                     tableInfoBuilder, metadataFileWriter)) {
                 // Given the transformation, do not use the original table's "definition" for writing
                 write(t, writeInstructions, parquetFileWriter, computedCache);
+                write(t.head(0), writeInstructions, parquetFileWriter, computedCache);
+                write(t, writeInstructions, parquetFileWriter, computedCache);
             }
             destOutputStream.done();
         }
@@ -235,19 +237,19 @@ private static void write(
         final TrackingRowSet tableRowSet = table.getRowSet();
         final Map<String, ? extends ColumnSource<?>> columnSourceMap = table.getColumnSourceMap();
         final long nRows = table.size();
-        if (nRows > 0) {
-            final RowGroupWriter rowGroupWriter = parquetFileWriter.addRowGroup(nRows);
-            for (final Map.Entry<String, ? extends ColumnSource<?>> nameToSource : columnSourceMap.entrySet()) {
-                final String columnName = nameToSource.getKey();
-                final ColumnSource<?> columnSource = nameToSource.getValue();
-                try {
-                    writeColumnSource(tableRowSet, writeInstructions, rowGroupWriter, computedCache, columnName,
-                            columnSource);
-                } catch (IllegalAccessException e) {
-                    throw new RuntimeException("Failed to write column " + columnName, e);
-                }
+        // if (nRows > 0) {
+        final RowGroupWriter rowGroupWriter = parquetFileWriter.addRowGroup(nRows);
+        for (final Map.Entry<String, ? extends ColumnSource<?>> nameToSource : columnSourceMap.entrySet()) {
+            final String columnName = nameToSource.getKey();
+            final ColumnSource<?> columnSource = nameToSource.getValue();
+            try {
+                writeColumnSource(tableRowSet, writeInstructions, rowGroupWriter, computedCache, columnName,
+                        columnSource);
+            } catch (IllegalAccessException e) {
+                throw new RuntimeException("Failed to write column " + columnName, e);
             }
         }
+        // }
     }
 
     /**
@@ -502,6 +504,9 @@ private static <DATA_TYPE> void encodePlain(
             @NotNull final Map<String, Map<ParquetCacheTags, Object>> computedCache,
             @NotNull final String columnName,
             @NotNull final ColumnSource<DATA_TYPE> columnSource) throws IOException {
+        if (tableRowSet.isEmpty()) {
+            return;
+        }
         try (final TransferObject<?> transferObject = TransferObject.create(
                 tableRowSet, writeInstructions, computedCache, columnName, columnSource)) {
             final Statistics<?> statistics = columnWriter.getStats();

diff --git a/...ons/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/...ons/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -1735,6 +1735,17 @@ public void testAllNonPartitioningColumnTypes() {
         }
     }
 
+
+    @Test
+    public void testWritingEmptyRowGroup() {
+        final Table table = TableTools.emptyTable(10).update(
+                "integers = (int)(ii%3)");
+        final String dest = rootFile + File.separator + "testWritingEmptyRowGroup.parquet";
+        ParquetTools.writeTable(table, dest);
+        final Table fromDisk = ParquetTools.readTable(dest);
+        assertTableEquals(merge(table, table), fromDisk);
+    }
+
     @Test
     public void decimalLogicalTypeTest() {
         final Table expected = TableTools.emptyTable(100_000).update(