From f8ae4ef3ee14b5f64b7c0ce54387d8a9945a992d Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 30 Jan 2024 23:55:51 +0530
Subject: [PATCH 01/45] Copied methods from inside parquet hadoop to write
 metadata files

---
 extensions/parquet/base/build.gradle          |   1 +
 .../parquet/base/ColumnChunkReaderImpl.java   |   2 +-
 .../parquet/base/ParquetFileWriter.java       | 145 ++++++++++++++++++
 .../parquet/base/RowGroupReaderImpl.java      |  15 +-
 .../table/ParquetTableReadWriteTest.java      |  15 ++
 5 files changed, 176 insertions(+), 2 deletions(-)

diff --git a/extensions/parquet/base/build.gradle b/extensions/parquet/base/build.gradle
index 028731cd85a..3dc24f29c84 100644
--- a/extensions/parquet/base/build.gradle
+++ b/extensions/parquet/base/build.gradle
@@ -9,6 +9,7 @@ dependencies {
     api project(':util-channel')
 
     Classpaths.inheritParquetHadoop(project)
+    Classpaths.inheritParquetHadoopConfiguration(project)
 
     implementation project(':extensions-parquet-compression')
     implementation project(':Base')
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
index 9a1570aac53..88de4065b1e 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -125,7 +125,7 @@ private URI getURI() {
         if (uri != null) {
             return uri;
         }
-        if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(uri.getScheme())) {
+        if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
             return uri = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
         } else {
             // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index db4515ed364..9572c84a5ed 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -3,6 +3,8 @@
  */
 package io.deephaven.parquet.base;
 
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.crypto.InternalFileEncryptor;
 import org.apache.parquet.format.converter.ParquetMetadataConverter;
 import io.deephaven.util.channel.SeekableChannelsProvider;
 import io.deephaven.parquet.compress.CompressorAdapter;
@@ -12,18 +14,26 @@
 import org.apache.parquet.bytes.BytesUtils;
 
 import org.apache.parquet.format.Util;
+import org.apache.parquet.hadoop.Footer;
 import org.apache.parquet.hadoop.metadata.*;
 import org.apache.parquet.internal.column.columnindex.OffsetIndex;
 import org.apache.parquet.internal.hadoop.metadata.IndexReference;
+import org.apache.parquet.io.ParquetEncodingException;
 import org.apache.parquet.schema.MessageType;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.apache.parquet.format.Util.writeFileMetaData;
+import static org.apache.parquet.hadoop.ParquetFileWriter.CURRENT_VERSION;
+import static org.apache.parquet.hadoop.ParquetFileWriter.MAGIC;
 
 public final class ParquetFileWriter {
     private static final ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter();
@@ -38,6 +48,10 @@ public final class ParquetFileWriter {
     private final Map<String, String> extraMetaData;
     private final List<BlockMetaData> blocks = new ArrayList<>();
     private final List<List<OffsetIndex>> offsetIndexes = new ArrayList<>();
+    private final String filePath;
+    private final SeekableChannelsProvider channelsProvider;
+    private static final String PARQUET_METADATA_FILE = "_metadata";
+    private static final String PARQUET_COMMON_METADATA_FILE = "_common_metadata";
 
     public ParquetFileWriter(
             final String filePath,
@@ -55,6 +69,8 @@ public ParquetFileWriter(
         bufferedOutput.write(ParquetFileReader.MAGIC);
         this.type = type;
         this.compressorAdapter = DeephavenCompressorAdapterFactory.getInstance().getByName(codecName);
+        this.filePath = filePath;
+        this.channelsProvider = channelsProvider;
     }
 
     public RowGroupWriter addRowGroup(final long size) {
@@ -74,8 +90,137 @@ public void close() throws IOException {
         // Flush any buffered data and close the channel
         bufferedOutput.close();
         compressorAdapter.close();
+        // Write the metadata file
+        final Path metadataDirPath = new Path(new File(filePath).getParent());
+        final String actualFilePath = filePath.replace(".NEW_", "");
+        final Footer fsFooter = new Footer(new Path(actualFilePath), footer);
+        final ArrayList<Footer> footers = new ArrayList<>();
+        footers.add(fsFooter);
+        writeMetadataFile(metadataDirPath, footers);
     }
 
+    @Deprecated
+    public void writeMetadataFile(Path outputPath, List<Footer> footers) throws IOException {
+        ParquetMetadata metadataFooter = mergeFooters(outputPath, footers);
+
+        writeMetadataFile(outputPath, metadataFooter, PARQUET_METADATA_FILE);
+
+        metadataFooter.getBlocks().clear();
+        writeMetadataFile(outputPath, metadataFooter, PARQUET_COMMON_METADATA_FILE);
+    }
+
+    static GlobalMetaData mergeInto(
+            FileMetaData toMerge,
+            GlobalMetaData mergedMetadata) {
+        return mergeInto(toMerge, mergedMetadata, true);
+    }
+
+    static GlobalMetaData mergeInto(
+            FileMetaData toMerge,
+            GlobalMetaData mergedMetadata,
+            boolean strict) {
+        MessageType schema = null;
+        Map<String, Set<String>> newKeyValues = new HashMap<String, Set<String>>();
+        Set<String> createdBy = new HashSet<String>();
+        if (mergedMetadata != null) {
+            schema = mergedMetadata.getSchema();
+            newKeyValues.putAll(mergedMetadata.getKeyValueMetaData());
+            createdBy.addAll(mergedMetadata.getCreatedBy());
+        }
+        if ((schema == null && toMerge.getSchema() != null)
+                || (schema != null && !schema.equals(toMerge.getSchema()))) {
+            schema = mergeInto(toMerge.getSchema(), schema, strict);
+        }
+        for (Map.Entry<String, String> entry : toMerge.getKeyValueMetaData().entrySet()) {
+            Set<String> values = newKeyValues.get(entry.getKey());
+            if (values == null) {
+                values = new LinkedHashSet<String>();
+                newKeyValues.put(entry.getKey(), values);
+            }
+            values.add(entry.getValue());
+        }
+        createdBy.add(toMerge.getCreatedBy());
+        return new GlobalMetaData(
+                schema,
+                newKeyValues,
+                createdBy);
+    }
+
+    static MessageType mergeInto(MessageType toMerge, MessageType mergedSchema, boolean strict) {
+        if (mergedSchema == null) {
+            return toMerge;
+        }
+
+        return mergedSchema.union(toMerge, strict);
+    }
+
+    static ParquetMetadata mergeFooters(Path root, List<Footer> footers) {
+        return mergeFooters(root, footers, new StrictKeyValueMetadataMergeStrategy());
+    }
+
+    static ParquetMetadata mergeFooters(Path root, List<Footer> footers,
+            KeyValueMetadataMergeStrategy keyValueMergeStrategy) {
+        String rootPath = root.toUri().getPath();
+        GlobalMetaData fileMetaData = null;
+        List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();
+        for (Footer footer : footers) {
+            String footerPath = footer.getFile().toUri().getPath();
+            if (!footerPath.startsWith(rootPath)) {
+                throw new ParquetEncodingException(
+                        footerPath + " invalid: all the files must be contained in the root " + root);
+            }
+            footerPath = footerPath.substring(rootPath.length());
+            while (footerPath.startsWith("/")) {
+                footerPath = footerPath.substring(1);
+            }
+            fileMetaData = mergeInto(footer.getParquetMetadata().getFileMetaData(), fileMetaData);
+            for (BlockMetaData block : footer.getParquetMetadata().getBlocks()) {
+                block.setPath(footerPath);
+                blocks.add(block);
+            }
+        }
+        return new ParquetMetadata(fileMetaData.merge(keyValueMergeStrategy), blocks);
+    }
+
+    /**
+     * @deprecated metadata files are not recommended and will be removed in 2.0.0
+     */
+    @Deprecated
+    private void writeMetadataFile(Path outputPathRoot, ParquetMetadata metadataFooter, String parquetMetadataFile)
+            throws IOException {
+        Path metaDataPath = new Path(outputPathRoot, parquetMetadataFile);
+        writeMetadataFile(metaDataPath, metadataFooter);
+    }
+
+    /**
+     * @deprecated metadata files are not recommended and will be removed in 2.0.0
+     */
+    @Deprecated
+    private void writeMetadataFile(Path outputPath, ParquetMetadata metadataFooter)
+            throws IOException {
+        final PositionedBufferedOutputStream metadataOutputStream =
+                new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(outputPath.toString(), false),
+                        OUTPUT_BUFFER_SIZE);
+        metadataOutputStream.write(MAGIC);
+        serializeMetadataFooter(metadataFooter, metadataOutputStream, null, new ParquetMetadataConverter());
+        metadataOutputStream.close();
+    }
+
+    private static void serializeMetadataFooter(ParquetMetadata footer, PositionedBufferedOutputStream out,
+            InternalFileEncryptor fileEncryptor, ParquetMetadataConverter metadataConverter) throws IOException {
+
+        // Unencrypted file
+        if (null == fileEncryptor) {
+            long footerIndex = out.position();
+            org.apache.parquet.format.FileMetaData parquetMetadata =
+                    metadataConverter.toParquetMetadata(CURRENT_VERSION, footer);
+            writeFileMetaData(parquetMetadata, out);
+            BytesUtils.writeIntLittleEndian(out, (int) (out.position() - footerIndex));
+            out.write(MAGIC);
+        }
+    }
+
+
     private void serializeFooter(final ParquetMetadata footer) throws IOException {
         final long footerIndex = bufferedOutput.position();
         final org.apache.parquet.format.FileMetaData parquetMetadata =
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index 7ee0dbbdd3d..338c1489757 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -21,11 +21,14 @@
 import java.net.URI;
 import java.nio.channels.Channels;
 import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
+
 public class RowGroupReaderImpl implements RowGroupReader {
 
     private static final int BUFFER_SIZE = 65536;
@@ -79,9 +82,19 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path,
             return null;
         }
 
+        // TODO Pass this to lower layers
+        final URI columnChunkURI;
+        if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
+            columnChunkURI = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
+        } else {
+            // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
+            columnChunkURI = rootURI;
+        }
+
         OffsetIndex offsetIndex = null;
         if (columnChunk.isSetOffset_index_offset()) {
-            try (final SeekableByteChannel readChannel = channelsProvider.getReadChannel(channelContext, rootURI)) {
+            try (final SeekableByteChannel readChannel =
+                    channelsProvider.getReadChannel(channelContext, columnChunkURI)) {
                 readChannel.position(columnChunk.getOffset_index_offset());
                 offsetIndex = ParquetMetadataConverter.fromParquetOffsetIndex(Util.readOffsetIndex(
                         new BufferedInputStream(Channels.newInputStream(readChannel), BUFFER_SIZE)));
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index a59a259799f..dae6baf0594 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -467,6 +467,21 @@ private static void writeReadTableTest(final Table table, final File dest,
         checkSingleTable(table, dest);
     }
 
+    @Test
+    public void basicWriteReadTest() {
+        final Table table = TableTools.emptyTable(5).update("A=(int)i", "B=(long)i", "C=(double)i");
+        final String filename = "basicWriteReadTest.parquet";
+        final File destFile = new File(rootFile, filename);
+        writeTable(table, destFile, EMPTY);
+
+        final Table fromDisk = readTable(destFile);
+        assertTableEquals(table, fromDisk);
+
+        final File metadataFile = new File(rootFile, "_metadata");
+        final Table fromDiskWithMetadata = readTable(metadataFile);
+        assertTableEquals(table, fromDiskWithMetadata);
+    }
+
     @Test
     public void testVectorColumns() {
         final Table table = getTableFlat(20000, true, false);

From 0ef7f79fcb2abc21fa874ac1c8224d2f74af90e8 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Wed, 31 Jan 2024 18:14:15 +0530
Subject: [PATCH 02/45] Calling methods from inside ParquetHadoop for writing
 metadata files

---
 extensions/parquet/base/build.gradle          |   3 +
 .../parquet/base/ParquetFileWriter.java       | 155 +++---------------
 2 files changed, 22 insertions(+), 136 deletions(-)

diff --git a/extensions/parquet/base/build.gradle b/extensions/parquet/base/build.gradle
index 3dc24f29c84..41e0904a67c 100644
--- a/extensions/parquet/base/build.gradle
+++ b/extensions/parquet/base/build.gradle
@@ -11,6 +11,9 @@ dependencies {
     Classpaths.inheritParquetHadoop(project)
     Classpaths.inheritParquetHadoopConfiguration(project)
 
+//    implementation group: 'org.apache.commons', name: 'commons-configuration2', version: '2.0'
+//    implementation group: 'org.apache.hadoop', name: 'hadoop-auth', version: '3.3.3'
+
     implementation project(':extensions-parquet-compression')
     implementation project(':Base')
     implementation project(':Util')
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 9572c84a5ed..049d56a8e52 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -3,8 +3,9 @@
  */
 package io.deephaven.parquet.base;
 
+import org.apache.hadoop.conf.Configuration;
+
 import org.apache.hadoop.fs.Path;
-import org.apache.parquet.crypto.InternalFileEncryptor;
 import org.apache.parquet.format.converter.ParquetMetadataConverter;
 import io.deephaven.util.channel.SeekableChannelsProvider;
 import io.deephaven.parquet.compress.CompressorAdapter;
@@ -18,22 +19,16 @@
 import org.apache.parquet.hadoop.metadata.*;
 import org.apache.parquet.internal.column.columnindex.OffsetIndex;
 import org.apache.parquet.internal.hadoop.metadata.IndexReference;
-import org.apache.parquet.io.ParquetEncodingException;
 import org.apache.parquet.schema.MessageType;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static org.apache.parquet.format.Util.writeFileMetaData;
-import static org.apache.parquet.hadoop.ParquetFileWriter.CURRENT_VERSION;
-import static org.apache.parquet.hadoop.ParquetFileWriter.MAGIC;
 
 public final class ParquetFileWriter {
     private static final ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter();
@@ -49,9 +44,6 @@ public final class ParquetFileWriter {
     private final List<BlockMetaData> blocks = new ArrayList<>();
     private final List<List<OffsetIndex>> offsetIndexes = new ArrayList<>();
     private final String filePath;
-    private final SeekableChannelsProvider channelsProvider;
-    private static final String PARQUET_METADATA_FILE = "_metadata";
-    private static final String PARQUET_COMMON_METADATA_FILE = "_common_metadata";
 
     public ParquetFileWriter(
             final String filePath,
@@ -70,7 +62,6 @@ public ParquetFileWriter(
         this.type = type;
         this.compressorAdapter = DeephavenCompressorAdapterFactory.getInstance().getByName(codecName);
         this.filePath = filePath;
-        this.channelsProvider = channelsProvider;
     }
 
     public RowGroupWriter addRowGroup(final long size) {
@@ -91,136 +82,28 @@ public void close() throws IOException {
         bufferedOutput.close();
         compressorAdapter.close();
         // Write the metadata file
-        final Path metadataDirPath = new Path(new File(filePath).getParent());
-        final String actualFilePath = filePath.replace(".NEW_", "");
+        final File pqFile = new File(filePath);
+        final Path metadataDirPath = new Path(pqFile.getParent());
+        final String actualFilePath = pqFile.getAbsolutePath().replace(".NEW_", "");
         final Footer fsFooter = new Footer(new Path(actualFilePath), footer);
         final ArrayList<Footer> footers = new ArrayList<>();
         footers.add(fsFooter);
-        writeMetadataFile(metadataDirPath, footers);
-    }
-
-    @Deprecated
-    public void writeMetadataFile(Path outputPath, List<Footer> footers) throws IOException {
-        ParquetMetadata metadataFooter = mergeFooters(outputPath, footers);
-
-        writeMetadataFile(outputPath, metadataFooter, PARQUET_METADATA_FILE);
-
-        metadataFooter.getBlocks().clear();
-        writeMetadataFile(outputPath, metadataFooter, PARQUET_COMMON_METADATA_FILE);
-    }
-
-    static GlobalMetaData mergeInto(
-            FileMetaData toMerge,
-            GlobalMetaData mergedMetadata) {
-        return mergeInto(toMerge, mergedMetadata, true);
-    }
-
-    static GlobalMetaData mergeInto(
-            FileMetaData toMerge,
-            GlobalMetaData mergedMetadata,
-            boolean strict) {
-        MessageType schema = null;
-        Map<String, Set<String>> newKeyValues = new HashMap<String, Set<String>>();
-        Set<String> createdBy = new HashSet<String>();
-        if (mergedMetadata != null) {
-            schema = mergedMetadata.getSchema();
-            newKeyValues.putAll(mergedMetadata.getKeyValueMetaData());
-            createdBy.addAll(mergedMetadata.getCreatedBy());
-        }
-        if ((schema == null && toMerge.getSchema() != null)
-                || (schema != null && !schema.equals(toMerge.getSchema()))) {
-            schema = mergeInto(toMerge.getSchema(), schema, strict);
-        }
-        for (Map.Entry<String, String> entry : toMerge.getKeyValueMetaData().entrySet()) {
-            Set<String> values = newKeyValues.get(entry.getKey());
-            if (values == null) {
-                values = new LinkedHashSet<String>();
-                newKeyValues.put(entry.getKey(), values);
-            }
-            values.add(entry.getValue());
-        }
-        createdBy.add(toMerge.getCreatedBy());
-        return new GlobalMetaData(
-                schema,
-                newKeyValues,
-                createdBy);
-    }
-
-    static MessageType mergeInto(MessageType toMerge, MessageType mergedSchema, boolean strict) {
-        if (mergedSchema == null) {
-            return toMerge;
-        }
-
-        return mergedSchema.union(toMerge, strict);
+        final Configuration conf = new Configuration();
+        // Not setting it requires adding dependency on hadoop-commons and commons-configuration2
+        // Setting it means we will not cache the filesystem instance which leads to a small performance hit but that's
+        // okay since this is only used when writing the metadata files
+        // But this leads to additional warnings like
+        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
+        // WARNING: Cannot load filesystem: java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
+        // Provider org.apache.hadoop.fs.viewfs.ViewFileSystem could not be instantiated
+        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
+        // WARNING: java.lang.NoClassDefFoundError: org/apache/commons/configuration2/Configuration
+        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
+        // WARNING: java.lang.ClassNotFoundException: org.apache.commons.configuration2.Configuration
+        conf.setBoolean("fs.file.impl.disable.cache", true);
+        org.apache.parquet.hadoop.ParquetFileWriter.writeMetadataFile(conf, metadataDirPath, footers);
     }
 
-    static ParquetMetadata mergeFooters(Path root, List<Footer> footers) {
-        return mergeFooters(root, footers, new StrictKeyValueMetadataMergeStrategy());
-    }
-
-    static ParquetMetadata mergeFooters(Path root, List<Footer> footers,
-            KeyValueMetadataMergeStrategy keyValueMergeStrategy) {
-        String rootPath = root.toUri().getPath();
-        GlobalMetaData fileMetaData = null;
-        List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();
-        for (Footer footer : footers) {
-            String footerPath = footer.getFile().toUri().getPath();
-            if (!footerPath.startsWith(rootPath)) {
-                throw new ParquetEncodingException(
-                        footerPath + " invalid: all the files must be contained in the root " + root);
-            }
-            footerPath = footerPath.substring(rootPath.length());
-            while (footerPath.startsWith("/")) {
-                footerPath = footerPath.substring(1);
-            }
-            fileMetaData = mergeInto(footer.getParquetMetadata().getFileMetaData(), fileMetaData);
-            for (BlockMetaData block : footer.getParquetMetadata().getBlocks()) {
-                block.setPath(footerPath);
-                blocks.add(block);
-            }
-        }
-        return new ParquetMetadata(fileMetaData.merge(keyValueMergeStrategy), blocks);
-    }
-
-    /**
-     * @deprecated metadata files are not recommended and will be removed in 2.0.0
-     */
-    @Deprecated
-    private void writeMetadataFile(Path outputPathRoot, ParquetMetadata metadataFooter, String parquetMetadataFile)
-            throws IOException {
-        Path metaDataPath = new Path(outputPathRoot, parquetMetadataFile);
-        writeMetadataFile(metaDataPath, metadataFooter);
-    }
-
-    /**
-     * @deprecated metadata files are not recommended and will be removed in 2.0.0
-     */
-    @Deprecated
-    private void writeMetadataFile(Path outputPath, ParquetMetadata metadataFooter)
-            throws IOException {
-        final PositionedBufferedOutputStream metadataOutputStream =
-                new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(outputPath.toString(), false),
-                        OUTPUT_BUFFER_SIZE);
-        metadataOutputStream.write(MAGIC);
-        serializeMetadataFooter(metadataFooter, metadataOutputStream, null, new ParquetMetadataConverter());
-        metadataOutputStream.close();
-    }
-
-    private static void serializeMetadataFooter(ParquetMetadata footer, PositionedBufferedOutputStream out,
-            InternalFileEncryptor fileEncryptor, ParquetMetadataConverter metadataConverter) throws IOException {
-
-        // Unencrypted file
-        if (null == fileEncryptor) {
-            long footerIndex = out.position();
-            org.apache.parquet.format.FileMetaData parquetMetadata =
-                    metadataConverter.toParquetMetadata(CURRENT_VERSION, footer);
-            writeFileMetaData(parquetMetadata, out);
-            BytesUtils.writeIntLittleEndian(out, (int) (out.position() - footerIndex));
-            out.write(MAGIC);
-        }
-    }
-
-
     private void serializeFooter(final ParquetMetadata footer) throws IOException {
         final long footerIndex = bufferedOutput.position();
         final org.apache.parquet.format.FileMetaData parquetMetadata =

From 44d031f08df2702a47111258e5e45191480640ab Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 13 Feb 2024 23:10:18 +0530
Subject: [PATCH 03/45] Cleaned up the code and added a lot of TODOs for review

---
 .../parquet/base/MetadataFileWriter.java      | 80 +++++++++++++++++++
 .../parquet/base/MetadataFileWriterBase.java  | 25 ++++++
 .../parquet/base/NullMetadataFileWriter.java  | 22 +++++
 .../parquet/base/ParquetFileWriter.java       | 42 +++-------
 .../parquet/table/ParquetInstructions.java    | 35 +++++++-
 .../parquet/table/ParquetTableWriter.java     | 75 ++++++++++++-----
 .../deephaven/parquet/table/ParquetTools.java | 43 ++++++++--
 .../table/ParquetTableReadWriteTest.java      | 72 ++++++++++++++++-
 8 files changed, 330 insertions(+), 64 deletions(-)
 create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java
 create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java
 create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java
new file mode 100644
index 00000000000..c19c9f12c54
--- /dev/null
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java
@@ -0,0 +1,80 @@
+package io.deephaven.parquet.base;
+
+import io.deephaven.UncheckedDeephavenException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.hadoop.Footer;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.parquet.hadoop.ParquetFileWriter.writeMetadataFile;
+
+/**
+ * Uses the hadoop implementation of {@link org.apache.parquet.hadoop.ParquetFileWriter} to generate metadata files for
+ * provided Parquet files.
+ */
+public final class MetadataFileWriter implements MetadataFileWriterBase {
+    // TODO Please suggest a better name for this class and the interface
+
+    private final String metadataRootDir;
+    private final List<Footer> footers;
+
+    public MetadataFileWriter(final String metadataRootDir, final File[] destinations) {
+        // TODO Discuss with Ryan about this check
+        // This check exists inside the hadoop code but is done at the time of writing the metadata files
+        // I added this here so that we can do this check upfront to catch errors early.
+        for (final File destination : destinations) {
+            if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
+                throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
+                        + " directory, provided destination " + destination.getAbsolutePath() + " is not in " +
+                        metadataRootDir);
+            }
+        }
+        this.metadataRootDir = metadataRootDir;
+        this.footers = new ArrayList<>(destinations.length);
+    }
+
+    /**
+     * Added parquet metadata for provided parquet file.
+     *
+     * @param parquetFile The parquet file destination path
+     * @param parquetMetadata The parquet metadata
+     */
+    public void addFooter(final File parquetFile, final ParquetMetadata parquetMetadata) {
+        footers.add(new Footer(new Path(parquetFile.getAbsolutePath()), parquetMetadata));
+    }
+
+    public void writeMetadataFiles() {
+        final Configuration conf = new Configuration();
+        conf.setBoolean("fs.file.impl.disable.cache", true);
+
+        // TODO Discuss with Ryan about this setting
+        // Not setting it requires adding dependency on hadoop-auth and commons-configuration2
+        // Setting it means we will not cache the filesystem instance which leads to a small performance hit but that's
+        // okay since this is only used when writing the metadata files
+        // But this leads to additional warnings like
+        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
+        // WARNING: Cannot load filesystem: java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
+        // Provider org.apache.hadoop.fs.viewfs.ViewFileSystem could not be instantiated
+        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
+        // WARNING: java.lang.NoClassDefFoundError: org/apache/commons/configuration2/Configuration
+        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
+        // WARNING: java.lang.ClassNotFoundException: org.apache.commons.configuration2.Configuration
+        //
+        // Other option is to move the metadata file writing code outside of parquet-hadoop and then remove the code
+        // for this caching and filesystem stuff. Note that the parquet-hadoop method is deprecated, so we don't expect
+        // the code to
+        // change in the future.
+
+        try {
+            writeMetadataFile(conf, new Path(metadataRootDir), footers);
+        } catch (final IOException e) {
+            throw new UncheckedDeephavenException("Failed to write metadata files at " + metadataRootDir, e);
+        }
+        footers.clear();
+    }
+}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java
new file mode 100644
index 00000000000..45155e378b4
--- /dev/null
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java
@@ -0,0 +1,25 @@
+package io.deephaven.parquet.base;
+
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+
+import java.io.File;
+
+/**
+ * Used to write _metadata and _common_metadata files for Parquet.
+ */
+public interface MetadataFileWriterBase {
+
+    /**
+     * Add the parquet metadata for the provided parquet file to the list of metadata to be written to combined metadata
+     * files.
+     *
+     * @param parquetFile The parquet file destination path
+     * @param parquetMetadata The parquet metadata corresponding to the parquet file
+     */
+    void addFooter(File parquetFile, ParquetMetadata parquetMetadata);
+
+    /**
+     * Write the combined metadata files for all metadata accumulated so far and clear the list.
+     */
+    void writeMetadataFiles();
+}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java
new file mode 100644
index 00000000000..ef37065886b
--- /dev/null
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java
@@ -0,0 +1,22 @@
+package io.deephaven.parquet.base;
+
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+
+import java.io.File;
+
+/**
+ * A no-op implementation of MetadataFileWriterBase when we don't want to write metadata files for Parquet files.
+ */
+public final class NullMetadataFileWriter implements MetadataFileWriterBase {
+
+    public static final NullMetadataFileWriter INSTANCE = new NullMetadataFileWriter();
+
+    private NullMetadataFileWriter() {}
+
+    @Override
+    public void addFooter(final File parquetFile, final ParquetMetadata parquetMetadata) {}
+
+    @Override
+    public void writeMetadataFiles() {}
+}
+
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 049d56a8e52..44f08ffbdbc 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -3,9 +3,6 @@
  */
 package io.deephaven.parquet.base;
 
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.fs.Path;
 import org.apache.parquet.format.converter.ParquetMetadataConverter;
 import io.deephaven.util.channel.SeekableChannelsProvider;
 import io.deephaven.parquet.compress.CompressorAdapter;
@@ -15,11 +12,11 @@
 import org.apache.parquet.bytes.BytesUtils;
 
 import org.apache.parquet.format.Util;
-import org.apache.parquet.hadoop.Footer;
 import org.apache.parquet.hadoop.metadata.*;
 import org.apache.parquet.internal.column.columnindex.OffsetIndex;
 import org.apache.parquet.internal.hadoop.metadata.IndexReference;
 import org.apache.parquet.schema.MessageType;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
 import java.io.IOException;
@@ -43,25 +40,29 @@ public final class ParquetFileWriter {
     private final Map<String, String> extraMetaData;
     private final List<BlockMetaData> blocks = new ArrayList<>();
     private final List<List<OffsetIndex>> offsetIndexes = new ArrayList<>();
-    private final String filePath;
+    private final File metadataFilePath;
+    private final MetadataFileWriterBase metadataFileWriter;
 
     public ParquetFileWriter(
-            final String filePath,
+            final File destFile,
+            final File metadataFilePath,
             final SeekableChannelsProvider channelsProvider,
             final int targetPageSize,
             final ByteBufferAllocator allocator,
             final MessageType type,
             final String codecName,
-            final Map<String, String> extraMetaData) throws IOException {
+            final Map<String, String> extraMetaData,
+            @NotNull final MetadataFileWriterBase metadataFileWriter) throws IOException {
         this.targetPageSize = targetPageSize;
         this.allocator = allocator;
         this.extraMetaData = new HashMap<>(extraMetaData);
-        bufferedOutput = new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(filePath, false),
+        bufferedOutput = new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(destFile.getPath(), false),
                 OUTPUT_BUFFER_SIZE);
         bufferedOutput.write(ParquetFileReader.MAGIC);
         this.type = type;
         this.compressorAdapter = DeephavenCompressorAdapterFactory.getInstance().getByName(codecName);
-        this.filePath = filePath;
+        this.metadataFilePath = metadataFilePath;
+        this.metadataFileWriter = metadataFileWriter;
     }
 
     public RowGroupWriter addRowGroup(final long size) {
@@ -78,30 +79,11 @@ public void close() throws IOException {
         final ParquetMetadata footer =
                 new ParquetMetadata(new FileMetaData(type, extraMetaData, Version.FULL_VERSION), blocks);
         serializeFooter(footer);
+        metadataFileWriter.addFooter(metadataFilePath, footer);
         // Flush any buffered data and close the channel
         bufferedOutput.close();
         compressorAdapter.close();
-        // Write the metadata file
-        final File pqFile = new File(filePath);
-        final Path metadataDirPath = new Path(pqFile.getParent());
-        final String actualFilePath = pqFile.getAbsolutePath().replace(".NEW_", "");
-        final Footer fsFooter = new Footer(new Path(actualFilePath), footer);
-        final ArrayList<Footer> footers = new ArrayList<>();
-        footers.add(fsFooter);
-        final Configuration conf = new Configuration();
-        // Not setting it requires adding dependency on hadoop-commons and commons-configuration2
-        // Setting it means we will not cache the filesystem instance which leads to a small performance hit but that's
-        // okay since this is only used when writing the metadata files
-        // But this leads to additional warnings like
-        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
-        // WARNING: Cannot load filesystem: java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
-        // Provider org.apache.hadoop.fs.viewfs.ViewFileSystem could not be instantiated
-        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
-        // WARNING: java.lang.NoClassDefFoundError: org/apache/commons/configuration2/Configuration
-        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
-        // WARNING: java.lang.ClassNotFoundException: org.apache.commons.configuration2.Configuration
-        conf.setBoolean("fs.file.impl.disable.cache", true);
-        org.apache.parquet.hadoop.ParquetFileWriter.writeMetadataFile(conf, metadataDirPath, footers);
+
     }
 
     private void serializeFooter(final ParquetMetadata footer) throws IOException {
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
index b209af59c32..2cc99bad99d 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
@@ -115,6 +115,8 @@ public static int getDefaultTargetPageSize() {
         return defaultTargetPageSize;
     }
 
+    static final String DEFAULT_METADATA_ROOT_DIR = ""; // Empty = No metadata files written
+
     public ParquetInstructions() {}
 
     public final String getColumnNameFromParquetColumnNameOrDefault(final String parquetColumnName) {
@@ -164,6 +166,11 @@ public final String getColumnNameFromParquetColumnNameOrDefault(final String par
      */
     public abstract boolean isRefreshing();
 
+    /**
+     * @return the directory in which metadata files should be stored.
+     */
+    public abstract String getMetadataRootDir();
+
     @VisibleForTesting
     public static boolean sameColumnNamesAndCodecMappings(final ParquetInstructions i1, final ParquetInstructions i2) {
         if (i1 == EMPTY) {
@@ -238,6 +245,11 @@ public int getTargetPageSize() {
         public boolean isRefreshing() {
             return DEFAULT_IS_REFRESHING;
         }
+
+        @Override
+        public String getMetadataRootDir() {
+            return DEFAULT_METADATA_ROOT_DIR;
+        }
     };
 
     private static class ColumnInstructions {
@@ -306,6 +318,7 @@ private static final class ReadOnly extends ParquetInstructions {
         private final int targetPageSize;
         private final boolean isRefreshing;
         private final Object specialInstructions;
+        private final String metadataRootDir;
 
         private ReadOnly(
                 final KeyedObjectHashMap<String, ColumnInstructions> columnNameToInstructions,
@@ -316,7 +329,8 @@ private ReadOnly(
                 final boolean isLegacyParquet,
                 final int targetPageSize,
                 final boolean isRefreshing,
-                final Object specialInstructions) {
+                final Object specialInstructions,
+                final String metadataRootDir) {
             this.columnNameToInstructions = columnNameToInstructions;
             this.parquetColumnNameToInstructions = parquetColumnNameToColumnName;
             this.compressionCodecName = compressionCodecName;
@@ -326,6 +340,7 @@ private ReadOnly(
             this.targetPageSize = targetPageSize;
             this.isRefreshing = isRefreshing;
             this.specialInstructions = specialInstructions;
+            this.metadataRootDir = metadataRootDir;
         }
 
         private String getOrDefault(final String columnName, final String defaultValue,
@@ -419,6 +434,10 @@ public boolean isRefreshing() {
             return specialInstructions;
         }
 
+        @Override
+        public String getMetadataRootDir() {
+            return metadataRootDir;
+        }
 
         KeyedObjectHashMap<String, ColumnInstructions> copyColumnNameToInstructions() {
             // noinspection unchecked
@@ -471,6 +490,7 @@ public static class Builder {
         private int targetPageSize = defaultTargetPageSize;
         private boolean isRefreshing = DEFAULT_IS_REFRESHING;
         private Object specialInstructions;
+        private String metadataRootDir = DEFAULT_METADATA_ROOT_DIR;
 
         public Builder() {}
 
@@ -647,6 +667,17 @@ public Builder setSpecialInstructions(final Object specialInstructions) {
             return this;
         }
 
+        /**
+         * Set the default metadata root directory.
+         *
+         * @param metadataRootDir the root directory to store metadata files in. All the parquet destinations should be
+         *        inside this directory.
+         */
+        public Builder setMetadataRootDir(final String metadataRootDir) {
+            this.metadataRootDir = metadataRootDir;
+            return this;
+        }
+
         public ParquetInstructions build() {
             final KeyedObjectHashMap<String, ColumnInstructions> columnNameToInstructionsOut = columnNameToInstructions;
             columnNameToInstructions = null;
@@ -655,7 +686,7 @@ public ParquetInstructions build() {
             parquetColumnNameToInstructions = null;
             return new ReadOnly(columnNameToInstructionsOut, parquetColumnNameToColumnNameOut, compressionCodecName,
                     maximumDictionaryKeys, maximumDictionarySize, isLegacyParquet, targetPageSize, isRefreshing,
-                    specialInstructions);
+                    specialInstructions, metadataRootDir);
         }
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index ad733d6abbf..db2d01a6314 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -18,6 +18,8 @@
 import io.deephaven.engine.table.impl.select.SelectColumn;
 import io.deephaven.engine.table.impl.select.SourceColumn;
 import io.deephaven.parquet.base.ColumnWriter;
+import io.deephaven.parquet.base.MetadataFileWriterBase;
+import io.deephaven.parquet.base.NullMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.RowGroupWriter;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
@@ -91,7 +93,10 @@ public GroupingColumnWritingInfo(final String parquetColumnName, final File meta
      * @param t The table to write
      * @param definition Table definition
      * @param writeInstructions Write instructions for customizations while writing
-     * @param destPathName The destination path
+     * @param destFile The destination to write the parquet file
+     * @param metadataFilePath The destination path to store in the metadata files. This can be different from
+     *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
+     *        always hold the accurate path.
      * @param incomingMeta A map of metadata values to be stores in the file footer
      * @throws SchemaMappingException Error creating a parquet table schema for the given table (likely due to
      *         unsupported types)
@@ -101,30 +106,45 @@ public static void write(
             @NotNull final Table t,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
-            @NotNull final String destPathName,
+            @NotNull final File destFile,
+            @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> incomingMeta,
-            final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap)
+            final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap,
+            @NotNull final MetadataFileWriterBase metadataFileWriter)
             throws SchemaMappingException, IOException {
         final TableInfo.Builder tableInfoBuilder = TableInfo.builder();
         List<File> cleanupFiles = null;
         try {
             if (groupingColumnsWritingInfoMap != null) {
                 cleanupFiles = new ArrayList<>(groupingColumnsWritingInfoMap.size());
-                final Path destDirPath = Paths.get(destPathName).getParent();
-                for (Map.Entry<String, GroupingColumnWritingInfo> entry : groupingColumnsWritingInfoMap.entrySet()) {
+                final Path destDirPath = destFile.toPath().getParent();
+                for (final Map.Entry<String, GroupingColumnWritingInfo> entry : groupingColumnsWritingInfoMap
+                        .entrySet()) {
                     final String groupingColumnName = entry.getKey();
                     final Table auxiliaryTable = groupingAsTable(t, groupingColumnName);
                     final String parquetColumnName = entry.getValue().parquetColumnName;
-                    final File metadataFilePath = entry.getValue().metadataFilePath;
+                    final File metadataGroupingFilePath = entry.getValue().metadataFilePath;
                     final File groupingDestFile = entry.getValue().destFile;
                     cleanupFiles.add(groupingDestFile);
                     tableInfoBuilder.addGroupingColumns(GroupingColumnInfo.of(parquetColumnName,
-                            destDirPath.relativize(metadataFilePath.toPath()).toString()));
-                    write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions,
-                            groupingDestFile.getAbsolutePath(), Collections.emptyMap(), TableInfo.builder());
+                            destDirPath.relativize(metadataGroupingFilePath.toPath()).toString()));
+                    // We don't include schema and row group information from grouping files in the metadata files
+
+                    // TODO Remove this comment
+                    // If I pass the regular metadata writer instead of the null here, it would fail with the following
+                    // error:
+                    // java.lang.RuntimeException: could not merge metadata: key deephaven has conflicting values:
+                    // [{"version":"unknown"},
+                    // {"version":"unknown","groupingColumns":[{"columnName":"vvv","groupingTablePath":".dh_metadata/indexes/vvv/index_vvv_groupingColumnsWriteTests.parquet"}]}]
+                    // Basically, the metadata file for grouping file has a different key-value-metadata than the non
+                    // grouping file.
+                    write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions, groupingDestFile,
+                            metadataGroupingFilePath, Collections.emptyMap(), TableInfo.builder(),
+                            NullMetadataFileWriter.INSTANCE);
                 }
             }
-            write(t, definition, writeInstructions, destPathName, incomingMeta, tableInfoBuilder);
+            write(t, definition, writeInstructions, destFile, metadataFilePath, incomingMeta, tableInfoBuilder,
+                    metadataFileWriter);
         } catch (Exception e) {
             if (cleanupFiles != null) {
                 for (final File cleanupFile : cleanupFiles) {
@@ -145,9 +165,12 @@ public static void write(
      * @param table The table to write
      * @param definition The table definition
      * @param writeInstructions Write instructions for customizations while writing
-     * @param path The destination path
+     * @param destFile The destination path
+     * @param metadataFilePath The destination path to store in the metadata files. This can be different from
+     *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
+     *        always hold the accurate path.
      * @param tableMeta A map of metadata values to be stores in the file footer
-     * @param tableInfoBuilder A partially-constructed builder for the metadata object
+     * @param tableInfoBuilder A partially constructed builder for the metadata object
      * @throws SchemaMappingException Error creating a parquet table schema for the given table (likely due to
      *         unsupported types)
      * @throws IOException For file writing related errors
@@ -156,19 +179,22 @@ public static void write(
             @NotNull final Table table,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
-            @NotNull final String path,
+            @NotNull final File destFile,
+            @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> tableMeta,
-            @NotNull final TableInfo.Builder tableInfoBuilder) throws SchemaMappingException, IOException {
+            @NotNull final TableInfo.Builder tableInfoBuilder,
+            @NotNull final MetadataFileWriterBase metadataFileWriter) throws SchemaMappingException, IOException {
         try (final SafeCloseable ignored = LivenessScopeStack.open()) {
             final Table t = pretransformTable(table, definition);
             final TrackingRowSet tableRowSet = t.getRowSet();
             final Map<String, ? extends ColumnSource<?>> columnSourceMap = t.getColumnSourceMap();
             // When we need to perform some computation depending on column data to make a decision impacting both
             // schema and written data, we store results in computedCache to avoid having to calculate twice.
-            // An example is the necessary precision and scale for a BigDecimal column writen as decimal logical type.
+            // An example is the necessary precision and scale for a BigDecimal column written as decimal logical type.
             final Map<String, Map<ParquetCacheTags, Object>> computedCache = new HashMap<>();
             final ParquetFileWriter parquetFileWriter = getParquetFileWriter(computedCache, definition, tableRowSet,
-                    columnSourceMap, path, writeInstructions, tableMeta, tableInfoBuilder);
+                    columnSourceMap, destFile, metadataFilePath, writeInstructions, tableMeta, tableInfoBuilder,
+                    metadataFileWriter);
             // Given the transformation, do not use the original table's "definition" for writing
             write(t, writeInstructions, parquetFileWriter, computedCache);
         }
@@ -256,7 +282,10 @@ private static Table pretransformTable(@NotNull final Table table, @NotNull fina
      * @param definition the writable definition
      * @param tableRowSet the row set being written
      * @param columnSourceMap the columns of the table
-     * @param path the destination to write to
+     * @param destFile the destination to write to
+     * @param metadataFilePath The destination path to store in the metadata files. This can be different from
+     *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
+     *        always hold the accurate path.
      * @param writeInstructions write instructions for the file
      * @param tableMeta metadata to include in the parquet metadata
      * @param tableInfoBuilder a builder for accumulating per-column information to construct the deephaven metadata
@@ -269,10 +298,12 @@ private static ParquetFileWriter getParquetFileWriter(
             @NotNull final TableDefinition definition,
             @NotNull final RowSet tableRowSet,
             @NotNull final Map<String, ? extends ColumnSource<?>> columnSourceMap,
-            @NotNull final String path,
+            @NotNull final File destFile,
+            @NotNull final File metadataFilePath,
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final Map<String, String> tableMeta,
-            @NotNull final TableInfo.Builder tableInfoBuilder) throws IOException {
+            @NotNull final TableInfo.Builder tableInfoBuilder,
+            @NotNull final MetadataFileWriterBase metadataFileWriter) throws IOException {
 
         // First, map the TableDefinition to a parquet Schema
         final MappedSchema mappedSchema =
@@ -314,11 +345,11 @@ private static ParquetFileWriter getParquetFileWriter(
 
         final Map<String, String> extraMetaData = new HashMap<>(tableMeta);
         extraMetaData.put(METADATA_KEY, tableInfoBuilder.build().serializeToJSON());
-        return new ParquetFileWriter(path,
-                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(path), null),
+        return new ParquetFileWriter(destFile, metadataFilePath,
+                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(destFile.getPath()), null),
                 writeInstructions.getTargetPageSize(),
                 new HeapByteBufferAllocator(), mappedSchema.getParquetSchema(),
-                writeInstructions.getCompressionCodecName(), extraMetaData);
+                writeInstructions.getCompressionCodecName(), extraMetaData, metadataFileWriter);
     }
 
     @VisibleForTesting
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 5ea3e1ca9ba..28b6fa73dc7 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -14,6 +14,9 @@
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.engine.table.impl.locations.util.TableDataRefreshService;
 import io.deephaven.engine.updategraph.UpdateSourceRegistrar;
+import io.deephaven.parquet.base.MetadataFileWriter;
+import io.deephaven.parquet.base.MetadataFileWriterBase;
+import io.deephaven.parquet.base.NullMetadataFileWriter;
 import io.deephaven.util.channel.SeekableChannelsProvider;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
 import io.deephaven.util.channel.SeekableChannelsProviderPlugin;
@@ -44,6 +47,7 @@
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.apache.parquet.schema.MessageType;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.io.IOException;
@@ -51,6 +55,7 @@
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
 
@@ -474,7 +479,7 @@ public static void writeParquetTables(@NotNull final Table[] sources,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final File[] destinations,
-            @NotNull final String[] groupingColumns) {
+            @Nullable final String[] groupingColumns) {
         Require.eq(sources.length, "sources.length", destinations.length, "destinations.length");
         if (definition.numColumns() == 0) {
             throw new TableDataException("Cannot write a parquet table with zero columns");
@@ -488,20 +493,29 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         final File[] firstCreatedDirs =
                 Arrays.stream(shadowDestFiles).map(ParquetTools::prepareDestinationFileLocation).toArray(File[]::new);
 
+        final MetadataFileWriterBase metadataFileWriter;
+        final String metadataRootDir = writeInstructions.getMetadataRootDir();
+        if (metadataRootDir != null && !metadataRootDir.equals(ParquetInstructions.DEFAULT_METADATA_ROOT_DIR)) {
+            metadataFileWriter = new MetadataFileWriter(metadataRootDir, destinations);
+        } else {
+            metadataFileWriter = NullMetadataFileWriter.INSTANCE;
+        }
+
         // List of shadow files, to clean up in case of exceptions
         final List<File> shadowFiles = new ArrayList<>();
         // List of all destination files (including grouping files), to roll back in case of exceptions
         final List<File> destFiles = new ArrayList<>();
         try {
             final List<Map<String, ParquetTableWriter.GroupingColumnWritingInfo>> groupingColumnWritingInfoMaps;
-            if (groupingColumns.length == 0) {
+            if (groupingColumns == null || groupingColumns.length == 0) {
                 // Write the tables without any grouping info
                 groupingColumnWritingInfoMaps = null;
                 for (int tableIdx = 0; tableIdx < sources.length; tableIdx++) {
                     shadowFiles.add(shadowDestFiles[tableIdx]);
                     final Table source = sources[tableIdx];
-                    ParquetTableWriter.write(source, definition, writeInstructions, shadowDestFiles[tableIdx].getPath(),
-                            Collections.emptyMap(), (Map<String, ParquetTableWriter.GroupingColumnWritingInfo>) null);
+                    ParquetTableWriter.write(source, definition, writeInstructions, shadowDestFiles[tableIdx],
+                            destinations[tableIdx], Collections.emptyMap(),
+                            (Map<String, ParquetTableWriter.GroupingColumnWritingInfo>) null, metadataFileWriter);
                 }
             } else {
                 // Create grouping info for each table and write the table and grouping files to shadow path
@@ -522,12 +536,27 @@ public static void writeParquetTables(@NotNull final Table[] sources,
                     groupingColumnWritingInfoMap.values().forEach(gcwi -> shadowFiles.add(gcwi.destFile));
 
                     final Table sourceTable = sources[tableIdx];
-                    ParquetTableWriter.write(sourceTable, definition, writeInstructions,
-                            shadowDestFiles[tableIdx].getPath(),
-                            Collections.emptyMap(), groupingColumnWritingInfoMap);
+                    ParquetTableWriter.write(sourceTable, definition, writeInstructions, shadowDestFiles[tableIdx],
+                            tableDestination, Collections.emptyMap(), groupingColumnWritingInfoMap, metadataFileWriter);
                 }
             }
 
+            // TODO (Discuss with Ryan) Note that we are directly combining the schema information from all the tables
+            // into a single metadata file. So say if we are writing a partitioned table data using this method, like
+            // writing "Date=1-1-2024/data.parquet" and "Date=1-2-2024/data.parquet" in the same call, then the metadata
+            // file with contain schema information from the tables and not the "Date" column.
+            //
+            // Another challenge is that based on the checks by the hadoop code, all the metadata files have to be
+            // written in the provided metadataRootDir path and the metadata file names are also fixed. Therefore, I
+            // cannot write these files to shadow paths. I will have to write them directly to the destination path
+            // which can lead to overwriting exiting files and deleting existing data in case of failure.
+            // One way I can think of is to make a copy of existing metadata files and then write the new metadata files
+            // and then delete the copies at the time of success.
+            // Other way is to copy the metadata file logic outside of hadoop code and then tweak it to use our shadow
+            // file writing logic. Note that the parquet-hadoop method is deprecated, so we don't expect the code to
+            // change in the future.
+            metadataFileWriter.writeMetadataFiles();
+
             // Write to shadow files was successful
             for (int tableIdx = 0; tableIdx < sources.length; tableIdx++) {
                 destFiles.add(destinations[tableIdx]);
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index dae6baf0594..f616b1a84af 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -105,6 +105,7 @@
 import static io.deephaven.parquet.table.ParquetTools.readKeyValuePartitionedTable;
 import static io.deephaven.parquet.table.ParquetTools.readSingleFileTable;
 import static io.deephaven.parquet.table.ParquetTools.readTable;
+import static io.deephaven.parquet.table.ParquetTools.writeParquetTables;
 import static io.deephaven.parquet.table.ParquetTools.writeTable;
 import static io.deephaven.util.QueryConstants.*;
 import static org.junit.Assert.*;
@@ -468,11 +469,43 @@ private static void writeReadTableTest(final Table table, final File dest,
     }
 
     @Test
-    public void basicWriteReadTest() {
+    public void basicParquetWithMetadataTest() {
         final Table table = TableTools.emptyTable(5).update("A=(int)i", "B=(long)i", "C=(double)i");
-        final String filename = "basicWriteReadTest.parquet";
+        final String filename = "basicParquetWithMetadataTest.parquet";
         final File destFile = new File(rootFile, filename);
-        writeTable(table, destFile, EMPTY);
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(rootFile.getAbsolutePath())
+                .build();
+        writeTable(table, destFile, writeInstructions);
+
+        final File metadataFile = new File(rootFile, "_metadata");
+        assertTrue(metadataFile.exists());
+        assertTrue(new File(rootFile, "._metadata.crc").exists());
+        assertTrue(new File(rootFile, "_common_metadata").exists());
+        assertTrue(new File(rootFile, "._common_metadata.crc").exists());
+
+        final Table fromDisk = readTable(destFile);
+        assertTableEquals(table, fromDisk);
+
+        final Table fromDiskWithMetadata = readTable(metadataFile);
+        assertTableEquals(table, fromDiskWithMetadata);
+    }
+
+    @Test
+    public void parquetWithGroupingDataAndMetadataTest() {
+        final Integer data[] = new Integer[500 * 4];
+        for (int i = 0; i < data.length; i++) {
+            data[i] = i / 4;
+        }
+        final TableDefinition tableDefinition = TableDefinition.of(ColumnDefinition.ofInt("vvv").withGrouping());
+        final Table table = newTable(tableDefinition, TableTools.col("vvv", data));
+
+        final File destFile = new File(rootFile, "parquetWithGroupingDataAndMetadataTest.parquet");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(rootFile.getAbsolutePath())
+                .build();
+        writeTable(table, destFile, writeInstructions);
+
 
         final Table fromDisk = readTable(destFile);
         assertTableEquals(table, fromDisk);
@@ -482,6 +515,39 @@ public void basicWriteReadTest() {
         assertTableEquals(table, fromDiskWithMetadata);
     }
 
+    @Test
+    public void flatPartitionedParquetWithMetadataTest() {
+        // Create an empty parent directory
+        final File parentDir = new File(rootFile, "tempDir");
+        parentDir.mkdir();
+        assertTrue(parentDir.exists() && parentDir.isDirectory() && parentDir.list().length == 0);
+
+        final Table someTable = TableTools.emptyTable(5).update("A=(int)i");
+        final File firstDataFile = new File(parentDir, "data1.parquet");
+        final File secondDataFile = new File(parentDir, "data2.parquet");
+
+        // Write without any metadata files
+        writeParquetTables(new Table[] {someTable, someTable}, someTable.getDefinition(), ParquetInstructions.EMPTY,
+                new File[] {firstDataFile, secondDataFile}, null);
+        final Table source = readTable(parentDir);
+
+        // Now write with metadata files
+        parentDir.delete();
+        parentDir.mkdir();
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(rootFile.getAbsolutePath())
+                .build();
+        writeParquetTables(new Table[] {someTable, someTable}, someTable.getDefinition(), writeInstructions,
+                new File[] {firstDataFile, secondDataFile}, null);
+
+        final Table fromDisk = readTable(parentDir);
+        assertTableEquals(source, fromDisk);
+
+        final File metadataFile = new File(rootFile, "_metadata");
+        final Table fromDiskWithMetadata = readTable(metadataFile);
+        assertTableEquals(source, fromDiskWithMetadata);
+    }
+
     @Test
     public void testVectorColumns() {
         final Table table = getTableFlat(20000, true, false);

From 6c6b075dfc787d5f8a61626206e3268077f845ea Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 13 Feb 2024 23:30:27 +0530
Subject: [PATCH 04/45] Some more changes

---
 extensions/parquet/base/build.gradle          |  2 ++
 .../parquet/base/ColumnChunkReaderImpl.java   | 21 ++++---------------
 .../parquet/base/ParquetFileWriter.java       |  1 -
 .../parquet/base/RowGroupReaderImpl.java      |  3 +--
 4 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/extensions/parquet/base/build.gradle b/extensions/parquet/base/build.gradle
index 41e0904a67c..9aefec2fba6 100644
--- a/extensions/parquet/base/build.gradle
+++ b/extensions/parquet/base/build.gradle
@@ -11,6 +11,8 @@ dependencies {
     Classpaths.inheritParquetHadoop(project)
     Classpaths.inheritParquetHadoopConfiguration(project)
 
+// TODO These are the additional dependencies that are required for the "fs.file.impl.disable.cache", check
+//  MetadataFileWriter code for more details.
 //    implementation group: 'org.apache.commons', name: 'commons-configuration2', version: '2.0'
 //    implementation group: 'org.apache.hadoop', name: 'hadoop-auth', version: '3.3.3'
 
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
index 88de4065b1e..79dfd7de595 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -41,18 +41,13 @@ public class ColumnChunkReaderImpl implements ColumnChunkReader {
 
     private final ColumnChunk columnChunk;
     private final SeekableChannelsProvider channelsProvider;
-    /**
-     * If reading a single parquet file, root URI is the URI of the file, else the parent directory for a metadata file
-     */
-    private final URI rootURI;
     private final CompressorAdapter decompressor;
     private final ColumnDescriptor path;
     private final OffsetIndex offsetIndex;
     private final List<Type> fieldTypes;
     private final Function<SeekableChannelContext, Dictionary> dictionarySupplier;
     private final PageMaterializer.Factory nullMaterializerFactory;
-
-    private URI uri;
+    private final URI columnChunkURI;
     /**
      * Number of rows in the row group of this column chunk.
      */
@@ -62,12 +57,12 @@ public class ColumnChunkReaderImpl implements ColumnChunkReader {
      */
     private final String version;
 
-    ColumnChunkReaderImpl(ColumnChunk columnChunk, SeekableChannelsProvider channelsProvider, URI rootURI,
+    ColumnChunkReaderImpl(ColumnChunk columnChunk, SeekableChannelsProvider channelsProvider, URI columnChunkURI,
             MessageType type, OffsetIndex offsetIndex, List<Type> fieldTypes, final long numRows,
             final String version) {
         this.channelsProvider = channelsProvider;
         this.columnChunk = columnChunk;
-        this.rootURI = rootURI;
+        this.columnChunkURI = columnChunkURI;
         this.path = type
                 .getColumnDescription(columnChunk.meta_data.getPath_in_schema().toArray(new String[0]));
         if (columnChunk.getMeta_data().isSetCodec()) {
@@ -122,15 +117,7 @@ public final ColumnPageDirectAccessor getPageAccessor() {
     }
 
     private URI getURI() {
-        if (uri != null) {
-            return uri;
-        }
-        if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
-            return uri = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
-        } else {
-            // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
-            return uri = rootURI;
-        }
+        return columnChunkURI;
     }
 
     @Override
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 44f08ffbdbc..31da24dc79d 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -83,7 +83,6 @@ public void close() throws IOException {
         // Flush any buffered data and close the channel
         bufferedOutput.close();
         compressorAdapter.close();
-
     }
 
     private void serializeFooter(final ParquetMetadata footer) throws IOException {
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index 338c1489757..17f4f962264 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -82,7 +82,6 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path,
             return null;
         }
 
-        // TODO Pass this to lower layers
         final URI columnChunkURI;
         if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
             columnChunkURI = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
@@ -102,7 +101,7 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path,
                 throw new UncheckedIOException(e);
             }
         }
-        return new ColumnChunkReaderImpl(columnChunk, channelsProvider, rootURI, type, offsetIndex, fieldTypes,
+        return new ColumnChunkReaderImpl(columnChunk, channelsProvider, columnChunkURI, type, offsetIndex, fieldTypes,
                 numRows(), version);
     }
 

From 9263aa02d36d719083a2656d57791313be62504b Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Wed, 14 Feb 2024 23:44:41 +0530
Subject: [PATCH 05/45] WIP commit

---
 extensions/parquet/base/build.gradle                |  5 -----
 ...iter.java => NullParquetMetadataFileWriter.java} |  6 +++---
 .../deephaven/parquet/base/ParquetFileWriter.java   |  4 ++--
 ...iterBase.java => ParquetMetadataFileWriter.java} |  2 +-
 ...iter.java => ParquetMetadataFileWriterImpl.java} |  8 ++------
 .../deephaven/parquet/table/ParquetTableWriter.java | 13 ++++++-------
 .../io/deephaven/parquet/table/ParquetTools.java    | 13 ++++++-------
 7 files changed, 20 insertions(+), 31 deletions(-)
 rename extensions/parquet/base/src/main/java/io/deephaven/parquet/base/{NullMetadataFileWriter.java => NullParquetMetadataFileWriter.java} (63%)
 rename extensions/parquet/base/src/main/java/io/deephaven/parquet/base/{MetadataFileWriterBase.java => ParquetMetadataFileWriter.java} (94%)
 rename extensions/parquet/base/src/main/java/io/deephaven/parquet/base/{MetadataFileWriter.java => ParquetMetadataFileWriterImpl.java} (87%)

diff --git a/extensions/parquet/base/build.gradle b/extensions/parquet/base/build.gradle
index 9aefec2fba6..3dc24f29c84 100644
--- a/extensions/parquet/base/build.gradle
+++ b/extensions/parquet/base/build.gradle
@@ -11,11 +11,6 @@ dependencies {
     Classpaths.inheritParquetHadoop(project)
     Classpaths.inheritParquetHadoopConfiguration(project)
 
-// TODO These are the additional dependencies that are required for the "fs.file.impl.disable.cache", check
-//  MetadataFileWriter code for more details.
-//    implementation group: 'org.apache.commons', name: 'commons-configuration2', version: '2.0'
-//    implementation group: 'org.apache.hadoop', name: 'hadoop-auth', version: '3.3.3'
-
     implementation project(':extensions-parquet-compression')
     implementation project(':Base')
     implementation project(':Util')
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
similarity index 63%
rename from extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java
rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
index ef37065886b..552a27323ec 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
@@ -7,11 +7,11 @@
 /**
  * A no-op implementation of MetadataFileWriterBase when we don't want to write metadata files for Parquet files.
  */
-public final class NullMetadataFileWriter implements MetadataFileWriterBase {
+public final class NullParquetMetadataFileWriter implements ParquetMetadataFileWriter {
 
-    public static final NullMetadataFileWriter INSTANCE = new NullMetadataFileWriter();
+    public static final NullParquetMetadataFileWriter INSTANCE = new NullParquetMetadataFileWriter();
 
-    private NullMetadataFileWriter() {}
+    private NullParquetMetadataFileWriter() {}
 
     @Override
     public void addFooter(final File parquetFile, final ParquetMetadata parquetMetadata) {}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 31da24dc79d..43e24e3954c 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -41,7 +41,7 @@ public final class ParquetFileWriter {
     private final List<BlockMetaData> blocks = new ArrayList<>();
     private final List<List<OffsetIndex>> offsetIndexes = new ArrayList<>();
     private final File metadataFilePath;
-    private final MetadataFileWriterBase metadataFileWriter;
+    private final ParquetMetadataFileWriter metadataFileWriter;
 
     public ParquetFileWriter(
             final File destFile,
@@ -52,7 +52,7 @@ public ParquetFileWriter(
             final MessageType type,
             final String codecName,
             final Map<String, String> extraMetaData,
-            @NotNull final MetadataFileWriterBase metadataFileWriter) throws IOException {
+            @NotNull final ParquetMetadataFileWriter metadataFileWriter) throws IOException {
         this.targetPageSize = targetPageSize;
         this.allocator = allocator;
         this.extraMetaData = new HashMap<>(extraMetaData);
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
similarity index 94%
rename from extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java
rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
index 45155e378b4..bc33627747a 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriterBase.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
@@ -7,7 +7,7 @@
 /**
  * Used to write _metadata and _common_metadata files for Parquet.
  */
-public interface MetadataFileWriterBase {
+public interface ParquetMetadataFileWriter {
 
     /**
      * Add the parquet metadata for the provided parquet file to the list of metadata to be written to combined metadata
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java
similarity index 87%
rename from extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java
rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java
index c19c9f12c54..dabee68faf1 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/MetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java
@@ -17,16 +17,12 @@
  * Uses the hadoop implementation of {@link org.apache.parquet.hadoop.ParquetFileWriter} to generate metadata files for
  * provided Parquet files.
  */
-public final class MetadataFileWriter implements MetadataFileWriterBase {
-    // TODO Please suggest a better name for this class and the interface
+public final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
 
     private final String metadataRootDir;
     private final List<Footer> footers;
 
-    public MetadataFileWriter(final String metadataRootDir, final File[] destinations) {
-        // TODO Discuss with Ryan about this check
-        // This check exists inside the hadoop code but is done at the time of writing the metadata files
-        // I added this here so that we can do this check upfront to catch errors early.
+    public ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
         for (final File destination : destinations) {
             if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
                 throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index db2d01a6314..87796b4f0bd 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -18,8 +18,8 @@
 import io.deephaven.engine.table.impl.select.SelectColumn;
 import io.deephaven.engine.table.impl.select.SourceColumn;
 import io.deephaven.parquet.base.ColumnWriter;
-import io.deephaven.parquet.base.MetadataFileWriterBase;
-import io.deephaven.parquet.base.NullMetadataFileWriter;
+import io.deephaven.parquet.base.ParquetMetadataFileWriter;
+import io.deephaven.parquet.base.NullParquetMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.RowGroupWriter;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
@@ -44,7 +44,6 @@
 import java.io.IOException;
 import java.nio.*;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.*;
 
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
@@ -110,7 +109,7 @@ public static void write(
             @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> incomingMeta,
             final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap,
-            @NotNull final MetadataFileWriterBase metadataFileWriter)
+            @NotNull final ParquetMetadataFileWriter metadataFileWriter)
             throws SchemaMappingException, IOException {
         final TableInfo.Builder tableInfoBuilder = TableInfo.builder();
         List<File> cleanupFiles = null;
@@ -140,7 +139,7 @@ public static void write(
                     // grouping file.
                     write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions, groupingDestFile,
                             metadataGroupingFilePath, Collections.emptyMap(), TableInfo.builder(),
-                            NullMetadataFileWriter.INSTANCE);
+                            NullParquetMetadataFileWriter.INSTANCE);
                 }
             }
             write(t, definition, writeInstructions, destFile, metadataFilePath, incomingMeta, tableInfoBuilder,
@@ -183,7 +182,7 @@ public static void write(
             @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> tableMeta,
             @NotNull final TableInfo.Builder tableInfoBuilder,
-            @NotNull final MetadataFileWriterBase metadataFileWriter) throws SchemaMappingException, IOException {
+            @NotNull final ParquetMetadataFileWriter metadataFileWriter) throws SchemaMappingException, IOException {
         try (final SafeCloseable ignored = LivenessScopeStack.open()) {
             final Table t = pretransformTable(table, definition);
             final TrackingRowSet tableRowSet = t.getRowSet();
@@ -303,7 +302,7 @@ private static ParquetFileWriter getParquetFileWriter(
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final Map<String, String> tableMeta,
             @NotNull final TableInfo.Builder tableInfoBuilder,
-            @NotNull final MetadataFileWriterBase metadataFileWriter) throws IOException {
+            @NotNull final ParquetMetadataFileWriter metadataFileWriter) throws IOException {
 
         // First, map the TableDefinition to a parquet Schema
         final MappedSchema mappedSchema =
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 28b6fa73dc7..5db905b469f 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -14,9 +14,9 @@
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.engine.table.impl.locations.util.TableDataRefreshService;
 import io.deephaven.engine.updategraph.UpdateSourceRegistrar;
-import io.deephaven.parquet.base.MetadataFileWriter;
-import io.deephaven.parquet.base.MetadataFileWriterBase;
-import io.deephaven.parquet.base.NullMetadataFileWriter;
+import io.deephaven.parquet.base.ParquetMetadataFileWriterImpl;
+import io.deephaven.parquet.base.ParquetMetadataFileWriter;
+import io.deephaven.parquet.base.NullParquetMetadataFileWriter;
 import io.deephaven.util.channel.SeekableChannelsProvider;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
 import io.deephaven.util.channel.SeekableChannelsProviderPlugin;
@@ -55,7 +55,6 @@
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
 
@@ -493,12 +492,12 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         final File[] firstCreatedDirs =
                 Arrays.stream(shadowDestFiles).map(ParquetTools::prepareDestinationFileLocation).toArray(File[]::new);
 
-        final MetadataFileWriterBase metadataFileWriter;
+        final ParquetMetadataFileWriter metadataFileWriter;
         final String metadataRootDir = writeInstructions.getMetadataRootDir();
         if (metadataRootDir != null && !metadataRootDir.equals(ParquetInstructions.DEFAULT_METADATA_ROOT_DIR)) {
-            metadataFileWriter = new MetadataFileWriter(metadataRootDir, destinations);
+            metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations);
         } else {
-            metadataFileWriter = NullMetadataFileWriter.INSTANCE;
+            metadataFileWriter = NullParquetMetadataFileWriter.INSTANCE;
         }
 
         // List of shadow files, to clean up in case of exceptions

From 8c887c4f9f63ec2019619d0a757851b16fc420b3 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Fri, 16 Feb 2024 23:20:33 +0530
Subject: [PATCH 06/45] Added a custom metadata file writer

---
 extensions/parquet/base/build.gradle          |   1 -
 .../base/NullParquetMetadataFileWriter.java   |   4 +-
 .../parquet/base/ParquetFileReader.java       |   2 +-
 .../parquet/base/ParquetFileWriter.java       |   8 +-
 .../base/ParquetMetadataFileWriter.java       |  10 +-
 .../base/ParquetMetadataFileWriterImpl.java   |  76 --------
 .../base/PositionedBufferedOutputStream.java  |   9 +-
 .../table/ParquetMetadataFileWriterImpl.java  | 177 ++++++++++++++++++
 .../parquet/table/ParquetTableWriter.java     |  13 +-
 .../deephaven/parquet/table/ParquetTools.java |  48 ++---
 .../table/ParquetTableReadWriteTest.java      |   7 +-
 11 files changed, 231 insertions(+), 124 deletions(-)
 delete mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java
 create mode 100644 extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java

diff --git a/extensions/parquet/base/build.gradle b/extensions/parquet/base/build.gradle
index 3dc24f29c84..028731cd85a 100644
--- a/extensions/parquet/base/build.gradle
+++ b/extensions/parquet/base/build.gradle
@@ -9,7 +9,6 @@ dependencies {
     api project(':util-channel')
 
     Classpaths.inheritParquetHadoop(project)
-    Classpaths.inheritParquetHadoopConfiguration(project)
 
     implementation project(':extensions-parquet-compression')
     implementation project(':Base')
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
index 552a27323ec..940969a11ee 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
@@ -14,9 +14,9 @@ public final class NullParquetMetadataFileWriter implements ParquetMetadataFileW
     private NullParquetMetadataFileWriter() {}
 
     @Override
-    public void addFooter(final File parquetFile, final ParquetMetadata parquetMetadata) {}
+    public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata metadata) {}
 
     @Override
-    public void writeMetadataFiles() {}
+    public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) {}
 }
 
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java
index 5421d133a9b..2a8ae22f815 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java
@@ -30,7 +30,7 @@
 public class ParquetFileReader {
     private static final int FOOTER_LENGTH_SIZE = 4;
     private static final String MAGIC_STR = "PAR1";
-    static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII);
+    public static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII);
     public static final String FILE_URI_SCHEME = "file";
 
     public final FileMetaData fileMetaData;
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 43e24e3954c..32e390dff14 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -78,14 +78,16 @@ public void close() throws IOException {
         serializeOffsetIndexes();
         final ParquetMetadata footer =
                 new ParquetMetadata(new FileMetaData(type, extraMetaData, Version.FULL_VERSION), blocks);
-        serializeFooter(footer);
-        metadataFileWriter.addFooter(metadataFilePath, footer);
+        serializeFooter(footer, bufferedOutput);
+        metadataFileWriter.addParquetFileMetadata(metadataFilePath, footer);
         // Flush any buffered data and close the channel
         bufferedOutput.close();
         compressorAdapter.close();
     }
 
-    private void serializeFooter(final ParquetMetadata footer) throws IOException {
+    public static void serializeFooter(final ParquetMetadata footer,
+            final PositionedBufferedOutputStream bufferedOutput)
+            throws IOException {
         final long footerIndex = bufferedOutput.position();
         final org.apache.parquet.format.FileMetaData parquetMetadata =
                 metadataConverter.toParquetMetadata(VERSION, footer);
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
index bc33627747a..cb523ba7cf9 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
@@ -3,6 +3,7 @@
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 
 import java.io.File;
+import java.io.IOException;
 
 /**
  * Used to write _metadata and _common_metadata files for Parquet.
@@ -14,12 +15,15 @@ public interface ParquetMetadataFileWriter {
      * files.
      *
      * @param parquetFile The parquet file destination path
-     * @param parquetMetadata The parquet metadata corresponding to the parquet file
+     * @param metadata The parquet metadata corresponding to the parquet file
      */
-    void addFooter(File parquetFile, ParquetMetadata parquetMetadata);
+    void addParquetFileMetadata(File parquetFile, ParquetMetadata metadata);
 
     /**
      * Write the combined metadata files for all metadata accumulated so far and clear the list.
+     *
+     * @param metadataFile The destination file for the _metadata file
+     * @param commonMetadataFile The destination file for the _common_metadata file
      */
-    void writeMetadataFiles();
+    void writeMetadataFiles(File metadataFile, File commonMetadataFile) throws IOException;
 }
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java
deleted file mode 100644
index dabee68faf1..00000000000
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriterImpl.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package io.deephaven.parquet.base;
-
-import io.deephaven.UncheckedDeephavenException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.parquet.hadoop.Footer;
-import org.apache.parquet.hadoop.metadata.ParquetMetadata;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.parquet.hadoop.ParquetFileWriter.writeMetadataFile;
-
-/**
- * Uses the hadoop implementation of {@link org.apache.parquet.hadoop.ParquetFileWriter} to generate metadata files for
- * provided Parquet files.
- */
-public final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
-
-    private final String metadataRootDir;
-    private final List<Footer> footers;
-
-    public ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
-        for (final File destination : destinations) {
-            if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
-                throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
-                        + " directory, provided destination " + destination.getAbsolutePath() + " is not in " +
-                        metadataRootDir);
-            }
-        }
-        this.metadataRootDir = metadataRootDir;
-        this.footers = new ArrayList<>(destinations.length);
-    }
-
-    /**
-     * Added parquet metadata for provided parquet file.
-     *
-     * @param parquetFile The parquet file destination path
-     * @param parquetMetadata The parquet metadata
-     */
-    public void addFooter(final File parquetFile, final ParquetMetadata parquetMetadata) {
-        footers.add(new Footer(new Path(parquetFile.getAbsolutePath()), parquetMetadata));
-    }
-
-    public void writeMetadataFiles() {
-        final Configuration conf = new Configuration();
-        conf.setBoolean("fs.file.impl.disable.cache", true);
-
-        // TODO Discuss with Ryan about this setting
-        // Not setting it requires adding dependency on hadoop-auth and commons-configuration2
-        // Setting it means we will not cache the filesystem instance which leads to a small performance hit but that's
-        // okay since this is only used when writing the metadata files
-        // But this leads to additional warnings like
-        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
-        // WARNING: Cannot load filesystem: java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
-        // Provider org.apache.hadoop.fs.viewfs.ViewFileSystem could not be instantiated
-        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
-        // WARNING: java.lang.NoClassDefFoundError: org/apache/commons/configuration2/Configuration
-        // Jan 31, 2024 6:31:23 PM org.apache.hadoop.fs.FileSystem loadFileSystems
-        // WARNING: java.lang.ClassNotFoundException: org.apache.commons.configuration2.Configuration
-        //
-        // Other option is to move the metadata file writing code outside of parquet-hadoop and then remove the code
-        // for this caching and filesystem stuff. Note that the parquet-hadoop method is deprecated, so we don't expect
-        // the code to
-        // change in the future.
-
-        try {
-            writeMetadataFile(conf, new Path(metadataRootDir), footers);
-        } catch (final IOException e) {
-            throw new UncheckedDeephavenException("Failed to write metadata files at " + metadataRootDir, e);
-        }
-        footers.clear();
-    }
-}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java
index f343f0bcb2e..9c857e5f3a3 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java
@@ -5,17 +5,22 @@
 import java.nio.channels.Channels;
 import java.nio.channels.SeekableByteChannel;
 
-final class PositionedBufferedOutputStream extends BufferedOutputStream {
+public final class PositionedBufferedOutputStream extends BufferedOutputStream {
 
     private final SeekableByteChannel writeChannel;
 
+    public PositionedBufferedOutputStream(final SeekableByteChannel writeChannel) {
+        super(Channels.newOutputStream(writeChannel));
+        this.writeChannel = writeChannel;
+    }
+
     PositionedBufferedOutputStream(final SeekableByteChannel writeChannel, final int size) {
         super(Channels.newOutputStream(writeChannel), size);
         this.writeChannel = writeChannel;
     }
 
     /**
-     * Get total number of bytes written to this stream
+     * Get the total number of bytes written to this stream
      */
     long position() throws IOException {
         // Number of bytes buffered in the stream + bytes written to the underlying channel
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
new file mode 100644
index 00000000000..c2672bf424d
--- /dev/null
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -0,0 +1,177 @@
+package io.deephaven.parquet.table;
+
+import io.deephaven.UncheckedDeephavenException;
+import io.deephaven.parquet.base.ParquetFileReader;
+import io.deephaven.parquet.base.ParquetFileWriter;
+import io.deephaven.parquet.base.ParquetMetadataFileWriter;
+import io.deephaven.parquet.base.PositionedBufferedOutputStream;
+import io.deephaven.parquet.table.metadata.ColumnTypeInfo;
+import io.deephaven.parquet.table.metadata.TableInfo;
+import io.deephaven.util.channel.SeekableChannelsProvider;
+import io.deephaven.util.channel.SeekableChannelsProviderLoader;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.schema.MessageType;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
+
+/**
+ * Used to generate a combined _metadata and _common_metadata file for provided Parquet files.
+ */
+public final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
+
+    /**
+     * A class to hold the parquet file and its metadata.
+     */
+    private static class ParquetFileMetadata {
+        final File file;
+        final ParquetMetadata metadata;
+
+        ParquetFileMetadata(final File file, final ParquetMetadata metadata) {
+            this.file = file;
+            this.metadata = metadata;
+        }
+    }
+
+    /**
+     * A class to accumulate metadata from multiple parquet files.
+     */
+    private static final class AccumulatedMetadata {
+        private MessageType schema;
+        private final Map<String, String> keyValueMetaData;
+        private final Set<String> createdBy;
+        private List<ColumnTypeInfo> columnTypes;
+
+        AccumulatedMetadata() {
+            this.schema = null;
+            this.keyValueMetaData = new HashMap<>();
+            this.createdBy = new HashSet<>();
+            this.columnTypes = null;
+        }
+
+        void combineWith(final FileMetaData fileMetaData) throws IOException {
+            // Take union of the schema
+            if (schema == null) {
+                schema = fileMetaData.getSchema();
+            } else if (!schema.equals(fileMetaData.getSchema())) {
+                schema = schema.union(fileMetaData.getSchema(), true);
+            }
+
+            // Accumulate the createdBy metadata
+            createdBy.add(fileMetaData.getCreatedBy());
+
+            // Accumulate the key-value metadata
+            for (final Map.Entry<String, String> entry : fileMetaData.getKeyValueMetaData().entrySet()) {
+                if (!entry.getKey().equals(ParquetTableWriter.METADATA_KEY)) {
+                    // We should only have one value for each key
+                    if (!keyValueMetaData.containsKey(entry.getKey())) {
+                        keyValueMetaData.put(entry.getKey(), entry.getValue());
+                    } else if (!keyValueMetaData.get(entry.getKey()).equals(entry.getValue())) {
+                        throw new UncheckedDeephavenException("Could not merge metadata for key " + entry.getKey() +
+                                ", has conflicting values: " + entry.getValue() + " and "
+                                + keyValueMetaData.get(entry.getKey()));
+                    }
+                } else {
+                    // For merging deephaven-specific metadata,
+                    // - groupingColumns, dataIndexes should always be dropped
+                    // - version is optional, so we read it from the first file's metadata
+                    // - columnTypes must be the same for all partitions
+                    final TableInfo tableInfo = TableInfo.deserializeFromJSON(entry.getValue());
+                    if (!keyValueMetaData.containsKey(ParquetTableWriter.METADATA_KEY)) {
+                        columnTypes = tableInfo.columnTypes();
+                        keyValueMetaData.put(ParquetTableWriter.METADATA_KEY,
+                                TableInfo.builder().addAllColumnTypes(columnTypes)
+                                        .version(tableInfo.version())
+                                        .build().serializeToJSON());
+                    } else if (!columnTypes.equals(tableInfo.columnTypes())) {
+                        throw new UncheckedDeephavenException("Could not merge metadata for key " +
+                                ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
+                                entry.getValue() + " and " + keyValueMetaData.get(entry.getKey()));
+                    }
+                }
+            }
+        }
+
+        FileMetaData getMerged() {
+            final String createdByString = createdBy.size() == 1 ? createdBy.iterator().next() : createdBy.toString();
+            return new FileMetaData(schema, keyValueMetaData, createdByString);
+        }
+    }
+
+    private final String metadataRootDirAbsPath;
+    private final List<ParquetFileMetadata> parquetFileMetadataList;
+    private final SeekableChannelsProvider channelsProvider;
+
+    public ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
+        for (final File destination : destinations) {
+            if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
+                throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
+                        + " directory, provided destination " + destination.getAbsolutePath() + " is not in " +
+                        metadataRootDir);
+            }
+        }
+        final File rootDir = new File(metadataRootDir);
+        this.metadataRootDirAbsPath = rootDir.getAbsolutePath();
+        this.parquetFileMetadataList = new ArrayList<>(destinations.length);
+        this.channelsProvider =
+                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(rootDir.getPath()), null);
+    }
+
+    /**
+     * Added parquet metadata for provided parquet file.
+     *
+     * @param parquetFile The parquet file destination path
+     * @param metadata The parquet metadata
+     */
+    public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata metadata) {
+        parquetFileMetadataList.add(new ParquetFileMetadata(parquetFile, metadata));
+    }
+
+    public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) throws IOException {
+        final ParquetMetadata metadataFooter = mergeMetadata();
+        writeMetadataFile(metadataFooter, metadataFile.getAbsolutePath());
+
+        metadataFooter.getBlocks().clear();
+        writeMetadataFile(metadataFooter, commonMetadataFile.toString());
+        parquetFileMetadataList.clear();
+    }
+
+    private ParquetMetadata mergeMetadata() throws IOException {
+        final AccumulatedMetadata accumulatedMetadata = new AccumulatedMetadata();
+        final List<BlockMetaData> blocks = new ArrayList<>();
+        for (final ParquetFileMetadata parquetFileMetadata : parquetFileMetadataList) {
+            accumulatedMetadata.combineWith(parquetFileMetadata.metadata.getFileMetaData());
+            final String fileAbsolutePath = parquetFileMetadata.file.getAbsolutePath();
+            String fileRelativePath = fileAbsolutePath.substring(metadataRootDirAbsPath.length());
+            // Remove leading slashes from the relative path
+            int pos = 0;
+            while (pos < fileRelativePath.length() && fileRelativePath.charAt(pos) == '/') {
+                pos++;
+            }
+            fileRelativePath = fileRelativePath.substring(pos);
+            for (final BlockMetaData block : parquetFileMetadata.metadata.getBlocks()) {
+                block.setPath(fileRelativePath);
+                blocks.add(block);
+            }
+        }
+        return new ParquetMetadata(accumulatedMetadata.getMerged(), blocks);
+    }
+
+    private void writeMetadataFile(final ParquetMetadata metadataFooter, final String outputPath) throws IOException {
+        final PositionedBufferedOutputStream metadataOutputStream =
+                new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(outputPath, false));
+        metadataOutputStream.write(ParquetFileReader.MAGIC);
+        ParquetFileWriter.serializeFooter(metadataFooter, metadataOutputStream);
+        metadataOutputStream.close();
+    }
+}
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 87796b4f0bd..0e27c297922 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -18,8 +18,8 @@
 import io.deephaven.engine.table.impl.select.SelectColumn;
 import io.deephaven.engine.table.impl.select.SourceColumn;
 import io.deephaven.parquet.base.ColumnWriter;
-import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.NullParquetMetadataFileWriter;
+import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.RowGroupWriter;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
@@ -127,16 +127,7 @@ public static void write(
                     cleanupFiles.add(groupingDestFile);
                     tableInfoBuilder.addGroupingColumns(GroupingColumnInfo.of(parquetColumnName,
                             destDirPath.relativize(metadataGroupingFilePath.toPath()).toString()));
-                    // We don't include schema and row group information from grouping files in the metadata files
-
-                    // TODO Remove this comment
-                    // If I pass the regular metadata writer instead of the null here, it would fail with the following
-                    // error:
-                    // java.lang.RuntimeException: could not merge metadata: key deephaven has conflicting values:
-                    // [{"version":"unknown"},
-                    // {"version":"unknown","groupingColumns":[{"columnName":"vvv","groupingTablePath":".dh_metadata/indexes/vvv/index_vvv_groupingColumnsWriteTests.parquet"}]}]
-                    // Basically, the metadata file for grouping file has a different key-value-metadata than the non
-                    // grouping file.
+                    // We don't accumulate metadata from grouping files into the main metadata file
                     write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions, groupingDestFile,
                             metadataGroupingFilePath, Collections.emptyMap(), TableInfo.builder(),
                             NullParquetMetadataFileWriter.INSTANCE);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 5db905b469f..0e14af40ff9 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -14,7 +14,6 @@
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.engine.table.impl.locations.util.TableDataRefreshService;
 import io.deephaven.engine.updategraph.UpdateSourceRegistrar;
-import io.deephaven.parquet.base.ParquetMetadataFileWriterImpl;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.NullParquetMetadataFileWriter;
 import io.deephaven.util.channel.SeekableChannelsProvider;
@@ -70,6 +69,8 @@
 public class ParquetTools {
 
     private static final int MAX_PARTITIONING_LEVELS_INFERENCE = 32;
+    private static final String PARQUET_METADATA_FILE = "_metadata";
+    private static final String PARQUET_COMMON_METADATA_FILE = "_common_metadata";
 
     private ParquetTools() {}
 
@@ -485,8 +486,8 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         }
         Arrays.stream(destinations).forEach(ParquetTools::deleteBackupFile);
 
-        // Write tables and index files at temporary shadow file paths in the same directory to prevent overwriting
-        // any existing files
+        // Write all files at temporary shadow file paths in the same directory to prevent overwriting any existing
+        // data in case of failure
         final File[] shadowDestFiles =
                 Arrays.stream(destinations).map(ParquetTools::getShadowFile).toArray(File[]::new);
         final File[] firstCreatedDirs =
@@ -494,7 +495,8 @@ public static void writeParquetTables(@NotNull final Table[] sources,
 
         final ParquetMetadataFileWriter metadataFileWriter;
         final String metadataRootDir = writeInstructions.getMetadataRootDir();
-        if (metadataRootDir != null && !metadataRootDir.equals(ParquetInstructions.DEFAULT_METADATA_ROOT_DIR)) {
+        final boolean writeMetadataFiles = !ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(metadataRootDir);
+        if (writeMetadataFiles) {
             metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations);
         } else {
             metadataFileWriter = NullParquetMetadataFileWriter.INSTANCE;
@@ -540,23 +542,21 @@ public static void writeParquetTables(@NotNull final Table[] sources,
                 }
             }
 
-            // TODO (Discuss with Ryan) Note that we are directly combining the schema information from all the tables
-            // into a single metadata file. So say if we are writing a partitioned table data using this method, like
-            // writing "Date=1-1-2024/data.parquet" and "Date=1-2-2024/data.parquet" in the same call, then the metadata
-            // file with contain schema information from the tables and not the "Date" column.
-            //
-            // Another challenge is that based on the checks by the hadoop code, all the metadata files have to be
-            // written in the provided metadataRootDir path and the metadata file names are also fixed. Therefore, I
-            // cannot write these files to shadow paths. I will have to write them directly to the destination path
-            // which can lead to overwriting exiting files and deleting existing data in case of failure.
-            // One way I can think of is to make a copy of existing metadata files and then write the new metadata files
-            // and then delete the copies at the time of success.
-            // Other way is to copy the metadata file logic outside of hadoop code and then tweak it to use our shadow
-            // file writing logic. Note that the parquet-hadoop method is deprecated, so we don't expect the code to
-            // change in the future.
-            metadataFileWriter.writeMetadataFiles();
-
-            // Write to shadow files was successful
+            // Write the combined metadata files to shadow destinations
+            final File metadataDestFile, shadowMetadataFile, commonMetadataDestFile, shadowCommonMetadataFile;
+            if (writeMetadataFiles) {
+                metadataDestFile = new File(metadataRootDir, PARQUET_METADATA_FILE);
+                shadowMetadataFile = ParquetTools.getShadowFile(metadataDestFile);
+                shadowFiles.add(shadowMetadataFile);
+                commonMetadataDestFile = new File(metadataRootDir, PARQUET_COMMON_METADATA_FILE);
+                shadowCommonMetadataFile = ParquetTools.getShadowFile(commonMetadataDestFile);
+                shadowFiles.add(shadowCommonMetadataFile);
+                metadataFileWriter.writeMetadataFiles(shadowMetadataFile, shadowCommonMetadataFile);
+            } else {
+                metadataDestFile = shadowMetadataFile = commonMetadataDestFile = shadowCommonMetadataFile = null;
+            }
+
+            // Write to shadow files was successful, now replace the original files with the shadow files
             for (int tableIdx = 0; tableIdx < sources.length; tableIdx++) {
                 destFiles.add(destinations[tableIdx]);
                 installShadowFile(destinations[tableIdx], shadowDestFiles[tableIdx]);
@@ -571,6 +571,12 @@ public static void writeParquetTables(@NotNull final Table[] sources,
                     }
                 }
             }
+            if (writeMetadataFiles) {
+                destFiles.add(metadataDestFile);
+                installShadowFile(metadataDestFile, shadowMetadataFile);
+                destFiles.add(commonMetadataDestFile);
+                installShadowFile(commonMetadataDestFile, shadowCommonMetadataFile);
+            }
         } catch (Exception e) {
             for (final File file : destFiles) {
                 rollbackFile(file);
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 1989d40b103..b6dfa933028 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -480,9 +480,7 @@ public void basicParquetWithMetadataTest() {
 
         final File metadataFile = new File(rootFile, "_metadata");
         assertTrue(metadataFile.exists());
-        assertTrue(new File(rootFile, "._metadata.crc").exists());
         assertTrue(new File(rootFile, "_common_metadata").exists());
-        assertTrue(new File(rootFile, "._common_metadata.crc").exists());
 
         final Table fromDisk = readTable(destFile);
         assertTableEquals(table, fromDisk);
@@ -493,6 +491,7 @@ public void basicParquetWithMetadataTest() {
 
     @Test
     public void parquetWithGroupingDataAndMetadataTest() {
+        final File rootFile = new File("/Users/shivammalhotra/Documents/tempDir");
         final Integer data[] = new Integer[500 * 4];
         for (int i = 0; i < data.length; i++) {
             data[i] = i / 4;
@@ -535,7 +534,7 @@ public void flatPartitionedParquetWithMetadataTest() {
         parentDir.delete();
         parentDir.mkdir();
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(rootFile.getAbsolutePath())
+                .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
         writeParquetTables(new Table[] {someTable, someTable}, someTable.getDefinition(), writeInstructions,
                 new File[] {firstDataFile, secondDataFile}, null);
@@ -543,7 +542,7 @@ public void flatPartitionedParquetWithMetadataTest() {
         final Table fromDisk = readTable(parentDir);
         assertTableEquals(source, fromDisk);
 
-        final File metadataFile = new File(rootFile, "_metadata");
+        final File metadataFile = new File(parentDir, "_metadata");
         final Table fromDiskWithMetadata = readTable(metadataFile);
         assertTableEquals(source, fromDiskWithMetadata);
     }

From b696649b6b9e4da89f82018bf192730cbf0f0c43 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Sat, 17 Feb 2024 00:26:05 +0530
Subject: [PATCH 07/45] Minor fix

---
 .../main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index 420d15e072f..e56eac9f807 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -29,7 +29,7 @@
 
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 
-public final class RowGroupReaderImpl implements RowGroupReader {
+final class RowGroupReaderImpl implements RowGroupReader {
     private final RowGroup rowGroup;
     private final SeekableChannelsProvider channelsProvider;
     private final MessageType type;

From 797b3dcf320054177cbee858364219fab1b6c0b2 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Sat, 17 Feb 2024 19:02:32 +0530
Subject: [PATCH 08/45] Fixed failing test

---
 .../parquet/table/ParquetTableReadWriteTest.java          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index b6dfa933028..789cdc65294 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -491,7 +491,7 @@ public void basicParquetWithMetadataTest() {
 
     @Test
     public void parquetWithGroupingDataAndMetadataTest() {
-        final File rootFile = new File("/Users/shivammalhotra/Documents/tempDir");
+        final File parentDir = new File(rootFile, "tempDir");
         final Integer data[] = new Integer[500 * 4];
         for (int i = 0; i < data.length; i++) {
             data[i] = i / 4;
@@ -499,9 +499,9 @@ public void parquetWithGroupingDataAndMetadataTest() {
         final TableDefinition tableDefinition = TableDefinition.of(ColumnDefinition.ofInt("vvv").withGrouping());
         final Table table = newTable(tableDefinition, TableTools.col("vvv", data));
 
-        final File destFile = new File(rootFile, "parquetWithGroupingDataAndMetadataTest.parquet");
+        final File destFile = new File(parentDir, "parquetWithGroupingDataAndMetadataTest.parquet");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(rootFile.getAbsolutePath())
+                .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
         writeTable(table, destFile, writeInstructions);
 
@@ -509,7 +509,7 @@ public void parquetWithGroupingDataAndMetadataTest() {
         final Table fromDisk = readTable(destFile);
         assertTableEquals(table, fromDisk);
 
-        final File metadataFile = new File(rootFile, "_metadata");
+        final File metadataFile = new File(parentDir, "_metadata");
         final Table fromDiskWithMetadata = readTable(metadataFile);
         assertTableEquals(table, fromDiskWithMetadata);
     }

From d987a06a48635525481e527fd3f38651f0685fe6 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 20 Feb 2024 06:47:42 -0600
Subject: [PATCH 09/45] Moved some code around

---
 .../base/NullParquetMetadataFileWriter.java   |   3 +
 .../base/ParquetMetadataFileWriter.java       |   5 +
 .../table/ParquetMetadataFileWriterImpl.java  | 164 +++++++++---------
 3 files changed, 91 insertions(+), 81 deletions(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
index 940969a11ee..95c91ca8f6e 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
@@ -18,5 +18,8 @@ public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata
 
     @Override
     public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) {}
+
+    @Override
+    public void clear() {}
 }
 
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
index cb523ba7cf9..a8f3fb9a0b3 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
@@ -26,4 +26,9 @@ public interface ParquetMetadataFileWriter {
      * @param commonMetadataFile The destination file for the _common_metadata file
      */
     void writeMetadataFiles(File metadataFile, File commonMetadataFile) throws IOException;
+
+    /**
+     * Clear the list of metadata accumulated so far.
+     */
+    void clear();
 }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index c2672bf424d..494fbf7e92c 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -1,6 +1,7 @@
 package io.deephaven.parquet.table;
 
 import io.deephaven.UncheckedDeephavenException;
+import io.deephaven.base.verify.Assert;
 import io.deephaven.parquet.base.ParquetFileReader;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
@@ -17,11 +18,11 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
@@ -43,74 +44,10 @@ private static class ParquetFileMetadata {
         }
     }
 
-    /**
-     * A class to accumulate metadata from multiple parquet files.
-     */
-    private static final class AccumulatedMetadata {
-        private MessageType schema;
-        private final Map<String, String> keyValueMetaData;
-        private final Set<String> createdBy;
-        private List<ColumnTypeInfo> columnTypes;
-
-        AccumulatedMetadata() {
-            this.schema = null;
-            this.keyValueMetaData = new HashMap<>();
-            this.createdBy = new HashSet<>();
-            this.columnTypes = null;
-        }
-
-        void combineWith(final FileMetaData fileMetaData) throws IOException {
-            // Take union of the schema
-            if (schema == null) {
-                schema = fileMetaData.getSchema();
-            } else if (!schema.equals(fileMetaData.getSchema())) {
-                schema = schema.union(fileMetaData.getSchema(), true);
-            }
-
-            // Accumulate the createdBy metadata
-            createdBy.add(fileMetaData.getCreatedBy());
-
-            // Accumulate the key-value metadata
-            for (final Map.Entry<String, String> entry : fileMetaData.getKeyValueMetaData().entrySet()) {
-                if (!entry.getKey().equals(ParquetTableWriter.METADATA_KEY)) {
-                    // We should only have one value for each key
-                    if (!keyValueMetaData.containsKey(entry.getKey())) {
-                        keyValueMetaData.put(entry.getKey(), entry.getValue());
-                    } else if (!keyValueMetaData.get(entry.getKey()).equals(entry.getValue())) {
-                        throw new UncheckedDeephavenException("Could not merge metadata for key " + entry.getKey() +
-                                ", has conflicting values: " + entry.getValue() + " and "
-                                + keyValueMetaData.get(entry.getKey()));
-                    }
-                } else {
-                    // For merging deephaven-specific metadata,
-                    // - groupingColumns, dataIndexes should always be dropped
-                    // - version is optional, so we read it from the first file's metadata
-                    // - columnTypes must be the same for all partitions
-                    final TableInfo tableInfo = TableInfo.deserializeFromJSON(entry.getValue());
-                    if (!keyValueMetaData.containsKey(ParquetTableWriter.METADATA_KEY)) {
-                        columnTypes = tableInfo.columnTypes();
-                        keyValueMetaData.put(ParquetTableWriter.METADATA_KEY,
-                                TableInfo.builder().addAllColumnTypes(columnTypes)
-                                        .version(tableInfo.version())
-                                        .build().serializeToJSON());
-                    } else if (!columnTypes.equals(tableInfo.columnTypes())) {
-                        throw new UncheckedDeephavenException("Could not merge metadata for key " +
-                                ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
-                                entry.getValue() + " and " + keyValueMetaData.get(entry.getKey()));
-                    }
-                }
-            }
-        }
-
-        FileMetaData getMerged() {
-            final String createdByString = createdBy.size() == 1 ? createdBy.iterator().next() : createdBy.toString();
-            return new FileMetaData(schema, keyValueMetaData, createdByString);
-        }
-    }
-
     private final String metadataRootDirAbsPath;
     private final List<ParquetFileMetadata> parquetFileMetadataList;
     private final SeekableChannelsProvider channelsProvider;
+    private List<ColumnTypeInfo> columnTypes; // Useful when merging deephaven specific metadata
 
     public ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
         for (final File destination : destinations) {
@@ -125,6 +62,7 @@ public ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[]
         this.parquetFileMetadataList = new ArrayList<>(destinations.length);
         this.channelsProvider =
                 SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(rootDir.getPath()), null);
+        this.columnTypes = null;
     }
 
     /**
@@ -147,24 +85,83 @@ public void writeMetadataFiles(final File metadataFile, final File commonMetadat
     }
 
     private ParquetMetadata mergeMetadata() throws IOException {
-        final AccumulatedMetadata accumulatedMetadata = new AccumulatedMetadata();
-        final List<BlockMetaData> blocks = new ArrayList<>();
+        MessageType mergedSchema = null;
+        final Map<String, String> mergedKeyValueMetaData = new HashMap<>();
+        final List<BlockMetaData> mergedBlocks = new ArrayList<>();
+        final Collection<String> mergedCreatedBy = new HashSet<>();
         for (final ParquetFileMetadata parquetFileMetadata : parquetFileMetadataList) {
-            accumulatedMetadata.combineWith(parquetFileMetadata.metadata.getFileMetaData());
-            final String fileAbsolutePath = parquetFileMetadata.file.getAbsolutePath();
-            String fileRelativePath = fileAbsolutePath.substring(metadataRootDirAbsPath.length());
-            // Remove leading slashes from the relative path
-            int pos = 0;
-            while (pos < fileRelativePath.length() && fileRelativePath.charAt(pos) == '/') {
-                pos++;
-            }
-            fileRelativePath = fileRelativePath.substring(pos);
-            for (final BlockMetaData block : parquetFileMetadata.metadata.getBlocks()) {
-                block.setPath(fileRelativePath);
-                blocks.add(block);
+            final FileMetaData fileMetaData = parquetFileMetadata.metadata.getFileMetaData();
+            mergedSchema = mergeSchemaInto(fileMetaData.getSchema(), mergedSchema);
+            mergeKeyValueMetaDataInto(fileMetaData.getKeyValueMetaData(), mergedKeyValueMetaData);
+            mergeBlocksInto(parquetFileMetadata, metadataRootDirAbsPath, mergedBlocks);
+            mergedCreatedBy.add(fileMetaData.getCreatedBy());
+        }
+        final String createdByString =
+                mergedCreatedBy.size() == 1 ? mergedCreatedBy.iterator().next() : mergedCreatedBy.toString();
+        return new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, createdByString),
+                mergedBlocks);
+    }
+
+    private static MessageType mergeSchemaInto(final MessageType schema, final MessageType mergedSchema) {
+        if (mergedSchema == null) {
+            return schema;
+        }
+        if (mergedSchema.equals(schema)) {
+            return mergedSchema;
+        }
+        return mergedSchema.union(schema, true);
+    }
+
+    private void mergeKeyValueMetaDataInto(final Map<String, String> keyValueMetaData,
+            final Map<String, String> mergedKeyValueMetaData) throws IOException {
+        for (final Map.Entry<String, String> entry : keyValueMetaData.entrySet()) {
+            if (!entry.getKey().equals(ParquetTableWriter.METADATA_KEY)) {
+                // We should only have one value for each key
+                if (!mergedKeyValueMetaData.containsKey(entry.getKey())) {
+                    mergedKeyValueMetaData.put(entry.getKey(), entry.getValue());
+                } else if (!mergedKeyValueMetaData.get(entry.getKey()).equals(entry.getValue())) {
+                    throw new UncheckedDeephavenException("Could not merge metadata for key " + entry.getKey() +
+                            ", has conflicting values: " + entry.getValue() + " and "
+                            + mergedKeyValueMetaData.get(entry.getKey()));
+                }
+            } else {
+                // For merging deephaven-specific metadata,
+                // - groupingColumns, dataIndexes should always be dropped
+                // - version is optional, so we read it from the first file's metadata
+                // - columnTypes must be the same for all partitions
+                final TableInfo tableInfo = TableInfo.deserializeFromJSON(entry.getValue());
+                if (!mergedKeyValueMetaData.containsKey(ParquetTableWriter.METADATA_KEY)) {
+                    // First time we've seen deephaven specific metadata
+                    Assert.eqNull(columnTypes, "columnTypes");
+                    columnTypes = tableInfo.columnTypes();
+                    mergedKeyValueMetaData.put(ParquetTableWriter.METADATA_KEY,
+                            TableInfo.builder().addAllColumnTypes(columnTypes)
+                                    .version(tableInfo.version())
+                                    .build().serializeToJSON());
+                } else if (!columnTypes.equals(tableInfo.columnTypes())) {
+                    throw new UncheckedDeephavenException("Could not merge metadata for key " +
+                            ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
+                            entry.getValue() + " and " + mergedKeyValueMetaData.get(entry.getKey()));
+                }
             }
         }
-        return new ParquetMetadata(accumulatedMetadata.getMerged(), blocks);
+    }
+
+    private static void mergeBlocksInto(final ParquetFileMetadata parquetFileMetadata,
+            final String metadataRootDirAbsPath,
+            final List<BlockMetaData> mergedBlocks) {
+        final String fileAbsolutePath = parquetFileMetadata.file.getAbsolutePath();
+        String fileRelativePath = fileAbsolutePath.substring(metadataRootDirAbsPath.length());
+        // Remove leading slashes from the relative path
+        int pos = 0;
+        while (pos < fileRelativePath.length() && fileRelativePath.charAt(pos) == '/') {
+            pos++;
+        }
+        fileRelativePath = fileRelativePath.substring(pos);
+        for (final BlockMetaData block : parquetFileMetadata.metadata.getBlocks()) {
+            block.setPath(fileRelativePath);
+            mergedBlocks.add(block);
+        }
     }
 
     private void writeMetadataFile(final ParquetMetadata metadataFooter, final String outputPath) throws IOException {
@@ -174,4 +171,9 @@ private void writeMetadataFile(final ParquetMetadata metadataFooter, final Strin
         ParquetFileWriter.serializeFooter(metadataFooter, metadataOutputStream);
         metadataOutputStream.close();
     }
+
+    public void clear() {
+        parquetFileMetadataList.clear();
+        columnTypes = null;
+    }
 }

From d4da1750becf819e8f587ea34c99ffd38f7b1409 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 20 Feb 2024 06:57:45 -0600
Subject: [PATCH 10/45] Minor change

---
 .../parquet/table/ParquetMetadataFileWriterImpl.java          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index 494fbf7e92c..2ce81e4ff28 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -29,7 +29,7 @@
 /**
  * Used to generate a combined _metadata and _common_metadata file for provided Parquet files.
  */
-public final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
+final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
 
     /**
      * A class to hold the parquet file and its metadata.
@@ -49,7 +49,7 @@ private static class ParquetFileMetadata {
     private final SeekableChannelsProvider channelsProvider;
     private List<ColumnTypeInfo> columnTypes; // Useful when merging deephaven specific metadata
 
-    public ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
+    ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
         for (final File destination : destinations) {
             if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
                 throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"

From f33d712ec22e37ef51acf2029274ffb46321b14a Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Thu, 22 Feb 2024 09:37:45 -0600
Subject: [PATCH 11/45] Review comments

---
 .../parquet/table/ParquetTableWriter.java       | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 0e27c297922..c864e3bfb3e 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -61,25 +61,24 @@ public class ParquetTableWriter {
     /**
      * Helper struct used to pass information about where to write the grouping files for each grouping column
      */
-    public static class GroupingColumnWritingInfo {
+    static class GroupingColumnWritingInfo {
         /**
          * Parquet name of this grouping column
          */
-        public final String parquetColumnName;
+        final String parquetColumnName;
         /**
          * File path to be added in the grouping metadata of main parquet file
          */
-        public final File metadataFilePath;
+        final File metadataFilePath;
 
         /**
          * Destination path for writing the grouping file. The two filenames can differ because we write grouping files
          * to shadow file paths first and then place them at the final path once the write is complete. But the metadata
          * should always hold the accurate path.
          */
-        public final File destFile;
+        final File destFile;
 
-        public GroupingColumnWritingInfo(final String parquetColumnName, final File metadataFilePath,
-                final File destFile) {
+        GroupingColumnWritingInfo(final String parquetColumnName, final File metadataFilePath, final File destFile) {
             this.parquetColumnName = parquetColumnName;
             this.metadataFilePath = metadataFilePath;
             this.destFile = destFile;
@@ -97,6 +96,10 @@ public GroupingColumnWritingInfo(final String parquetColumnName, final File meta
      *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
      *        always hold the accurate path.
      * @param incomingMeta A map of metadata values to be stores in the file footer
+     * @param groupingColumnsWritingInfoMap A map of grouping column names to their respective info used for writing
+     *        grouping files
+     * @param metadataFileWriter The writer for the _metadata and _common_metadata files
+     *
      * @throws SchemaMappingException Error creating a parquet table schema for the given table (likely due to
      *         unsupported types)
      * @throws IOException For file writing related errors
@@ -161,6 +164,7 @@ public static void write(
      *        always hold the accurate path.
      * @param tableMeta A map of metadata values to be stores in the file footer
      * @param tableInfoBuilder A partially constructed builder for the metadata object
+     * @param metadataFileWriter The writer for the _metadata and _common_metadata files
      * @throws SchemaMappingException Error creating a parquet table schema for the given table (likely due to
      *         unsupported types)
      * @throws IOException For file writing related errors
@@ -279,6 +283,7 @@ private static Table pretransformTable(@NotNull final Table table, @NotNull fina
      * @param writeInstructions write instructions for the file
      * @param tableMeta metadata to include in the parquet metadata
      * @param tableInfoBuilder a builder for accumulating per-column information to construct the deephaven metadata
+     * @param metadataFileWriter The writer for the _metadata and _common_metadata files
      *
      * @return a new file writer
      */

From 238f5f56c56aaaecad1a2a402b6700798e94124f Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 26 Feb 2024 11:42:38 -0600
Subject: [PATCH 12/45] Read offset index from column chunk on demand

---
 .../parquet/base/ColumnChunkReader.java       | 17 ++++--
 .../parquet/base/ColumnChunkReaderImpl.java   | 33 +++++++----
 .../parquet/base/OffsetIndexReader.java       | 22 ++++++++
 .../parquet/base/OffsetIndexReaderImpl.java   | 55 +++++++++++++++++++
 .../parquet/base/RowGroupReader.java          |  3 +-
 .../parquet/base/RowGroupReaderImpl.java      | 33 ++---------
 .../table/location/ParquetColumnLocation.java | 12 ++--
 .../table/location/ParquetTableLocation.java  |  7 +--
 .../table/pagestore/ColumnChunkPageStore.java |  2 +-
 .../OffsetIndexBasedColumnChunkPageStore.java |  8 ++-
 10 files changed, 132 insertions(+), 60 deletions(-)
 create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java
 create mode 100644 extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java
index 2f13ec64802..4302f2b36bb 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java
@@ -31,10 +31,17 @@ public interface ColumnChunkReader {
     int getMaxRl();
 
     /**
-     * @return The offset index for this column chunk, or null if it not found in the metadata.
+     * @return Whether the column chunk has offset index information set in the metadata or not.
      */
-    @Nullable
-    OffsetIndex getOffsetIndex();
+    boolean hasOffsetIndex();
+
+    /**
+     * Get the offset index for a column chunk.
+     *
+     * @param context The channel context to use for reading the offset index.
+     *
+     */
+    OffsetIndex getOffsetIndex(final SeekableChannelContext context);
 
     /**
      * Used to iterate over column page readers for each page with the capability to set channel context to for reading
@@ -69,9 +76,9 @@ interface ColumnPageDirectAccessor {
     }
 
     /**
-     * @return An accessor for individual parquet pages.
+     * @return An accessor for individual parquet pages which uses the provided offset index.
      */
-    ColumnPageDirectAccessor getPageAccessor();
+    ColumnPageDirectAccessor getPageAccessor(OffsetIndex offsetIndex);
 
     /**
      * @return Whether this column chunk uses a dictionary-based encoding on every page.
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
index 799ce823ce0..d87b9360666 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -27,12 +27,10 @@
 import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.channels.SeekableByteChannel;
-import java.nio.file.Path;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.function.Function;
 
-import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 import static org.apache.parquet.format.Encoding.PLAIN_DICTIONARY;
 import static org.apache.parquet.format.Encoding.RLE_DICTIONARY;
 
@@ -42,7 +40,7 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
     private final SeekableChannelsProvider channelsProvider;
     private final CompressorAdapter decompressor;
     private final ColumnDescriptor path;
-    private final OffsetIndex offsetIndex;
+    private OffsetIndexReader offsetIndexReader;
     private final List<Type> fieldTypes;
     private final Function<SeekableChannelContext, Dictionary> dictionarySupplier;
     private final PageMaterializerFactory nullMaterializerFactory;
@@ -57,8 +55,7 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
     private final String version;
 
     ColumnChunkReaderImpl(ColumnChunk columnChunk, SeekableChannelsProvider channelsProvider, URI columnChunkURI,
-            MessageType type, OffsetIndex offsetIndex, List<Type> fieldTypes, final long numRows,
-            final String version) {
+            MessageType type, List<Type> fieldTypes, final long numRows, final String version) {
         this.channelsProvider = channelsProvider;
         this.columnChunk = columnChunk;
         this.columnChunkURI = columnChunkURI;
@@ -70,12 +67,15 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
         } else {
             decompressor = CompressorAdapter.PASSTHRU;
         }
-        this.offsetIndex = offsetIndex;
         this.fieldTypes = fieldTypes;
         this.dictionarySupplier = new LazyCachingFunction<>(this::getDictionary);
         this.nullMaterializerFactory = PageMaterializer.factoryForType(path.getPrimitiveType().getPrimitiveTypeName());
         this.numRows = numRows;
         this.version = version;
+        // Construct the reader object but don't read the offset index yet
+        this.offsetIndexReader = (columnChunk.isSetOffset_index_offset())
+                ? new OffsetIndexReaderImpl(channelsProvider, columnChunk, columnChunkURI)
+                : OffsetIndexReader.NULL;
     }
 
     @Override
@@ -93,8 +93,15 @@ public int getMaxRl() {
         return path.getMaxRepetitionLevel();
     }
 
-    public OffsetIndex getOffsetIndex() {
-        return offsetIndex;
+    @Override
+    public boolean hasOffsetIndex() {
+        return columnChunk.isSetOffset_index_offset();
+    }
+
+    @Override
+    public OffsetIndex getOffsetIndex(final SeekableChannelContext context) {
+        // Read the offset index if it hasn't been read yet
+        return offsetIndexReader.getOffsetIndex(context);
     }
 
     @Override
@@ -103,11 +110,11 @@ public ColumnPageReaderIterator getPageIterator() {
     }
 
     @Override
-    public final ColumnPageDirectAccessor getPageAccessor() {
+    public ColumnPageDirectAccessor getPageAccessor(final OffsetIndex offsetIndex) {
         if (offsetIndex == null) {
             throw new UnsupportedOperationException("Cannot use direct accessor without offset index");
         }
-        return new ColumnPageDirectAccessorImpl();
+        return new ColumnPageDirectAccessorImpl(offsetIndex);
     }
 
     private URI getURI() {
@@ -294,7 +301,11 @@ private static int getNumValues(PageHeader pageHeader) {
 
     private final class ColumnPageDirectAccessorImpl implements ColumnPageDirectAccessor {
 
-        ColumnPageDirectAccessorImpl() {}
+        private final OffsetIndex offsetIndex;
+
+        ColumnPageDirectAccessorImpl(final OffsetIndex offsetIndex) {
+            this.offsetIndex = offsetIndex;
+        }
 
         @Override
         public ColumnPageReader getPageReader(final int pageNum, final SeekableChannelContext channelContext) {
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java
new file mode 100644
index 00000000000..dbbe7264869
--- /dev/null
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java
@@ -0,0 +1,22 @@
+package io.deephaven.parquet.base;
+
+import io.deephaven.util.channel.SeekableChannelContext;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+
+/**
+ * Interface for reading the offset index for a column chunk.
+ */
+public interface OffsetIndexReader {
+
+    /**
+     * Get the offset index for a column chunk.
+     *
+     * @param context The channel context to use for reading the offset index.
+     */
+    OffsetIndex getOffsetIndex(SeekableChannelContext context);
+
+    /**
+     * A null implementation of the offset index reader.
+     */
+    OffsetIndexReader NULL = context -> null;
+}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java
new file mode 100644
index 00000000000..ecaeba34664
--- /dev/null
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java
@@ -0,0 +1,55 @@
+package io.deephaven.parquet.base;
+
+import io.deephaven.util.channel.SeekableChannelContext;
+import io.deephaven.util.channel.SeekableChannelsProvider;
+import org.apache.parquet.format.ColumnChunk;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.format.Util;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.nio.channels.SeekableByteChannel;
+
+/**
+ * Implementation of {@link OffsetIndexReader}, which reads the offset index for a column chunk on demand.
+ */
+final class OffsetIndexReaderImpl implements OffsetIndexReader {
+
+    private final SeekableChannelsProvider channelsProvider;
+    private final ColumnChunk chunk;
+    private final URI columnChunkURI;
+    private OffsetIndex offsetIndex;
+
+    OffsetIndexReaderImpl(final SeekableChannelsProvider channelsProvider, final ColumnChunk chunk,
+            final URI columnChunkURI) {
+        this.channelsProvider = channelsProvider;
+        this.chunk = chunk;
+        this.columnChunkURI = columnChunkURI;
+        this.offsetIndex = null;
+    }
+
+    @Override
+    public OffsetIndex getOffsetIndex(@NotNull final SeekableChannelContext context) {
+        if (offsetIndex != null) {
+            return offsetIndex;
+        }
+        return readOffsetIndex(context);
+    }
+
+    private OffsetIndex readOffsetIndex(@NotNull final SeekableChannelContext channelContext) {
+        try (
+                final SeekableChannelContext.ContextHolder holder =
+                        SeekableChannelContext.ensureContext(channelsProvider, channelContext);
+                final SeekableByteChannel readChannel = channelsProvider.getReadChannel(holder.get(), columnChunkURI);
+                final InputStream in =
+                        channelsProvider.getInputStream(readChannel.position(chunk.getOffset_index_offset()))) {
+            return (offsetIndex = ParquetMetadataConverter.fromParquetOffsetIndex(Util.readOffsetIndex(in)));
+        } catch (final IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java
index 71f02961ca3..cf340d148dd 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java
@@ -17,10 +17,9 @@ public interface RowGroupReader {
      * Returns the accessor to a given Column Chunk
      * 
      * @param path the full column path
-     * @param channelContext the channel context to use while reading the parquet file
      * @return the accessor to a given Column Chunk
      */
-    ColumnChunkReader getColumnChunk(@NotNull List<String> path, @NotNull final SeekableChannelContext channelContext);
+    ColumnChunkReader getColumnChunk(@NotNull List<String> path);
 
     long numRows();
 
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index e56eac9f807..9798a42f2a7 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -71,11 +71,10 @@ final class RowGroupReaderImpl implements RowGroupReader {
     }
 
     @Override
-    public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path,
-            @NotNull final SeekableChannelContext channelContext) {
-        String key = path.toString();
-        ColumnChunk columnChunk = chunkMap.get(key);
-        List<Type> fieldTypes = schemaMap.get(key);
+    public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path) {
+        final String key = path.toString();
+        final ColumnChunk columnChunk = chunkMap.get(key);
+        final List<Type> fieldTypes = schemaMap.get(key);
         if (columnChunk == null) {
             return null;
         }
@@ -87,31 +86,11 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path,
             // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
             columnChunkURI = rootURI;
         }
-
-        final OffsetIndex offsetIndex = offsetIndex(columnChunk, columnChunkURI, channelContext);
-        return new ColumnChunkReaderImpl(columnChunk, channelsProvider, columnChunkURI, type, offsetIndex, fieldTypes,
-                numRows(), version);
+        return new ColumnChunkReaderImpl(columnChunk, channelsProvider, columnChunkURI, type, fieldTypes, numRows(),
+                version);
     }
 
-    private OffsetIndex offsetIndex(ColumnChunk chunk, URI columnChunkURI, @NotNull SeekableChannelContext context) {
-        if (!chunk.isSetOffset_index_offset()) {
-            return null;
-        }
-        return ParquetMetadataConverter.fromParquetOffsetIndex(readOffsetIndex(chunk, columnChunkURI, context));
-    }
 
-    private org.apache.parquet.format.OffsetIndex readOffsetIndex(ColumnChunk chunk, URI columnChunkURI,
-            @NotNull SeekableChannelContext channelContext) {
-        try (
-                final ContextHolder holder = SeekableChannelContext.ensureContext(channelsProvider, channelContext);
-                final SeekableByteChannel readChannel = channelsProvider.getReadChannel(holder.get(), columnChunkURI);
-                final InputStream in =
-                        channelsProvider.getInputStream(readChannel.position(chunk.getOffset_index_offset()))) {
-            return Util.readOffsetIndex(in);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
 
     @Override
     public long numRows() {
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java
index d5182938617..e077cbada01 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java
@@ -213,13 +213,11 @@ public <METADATA_TYPE> METADATA_TYPE getMetadata(@NotNull final ColumnDefinition
             final String version = tableInfo.map(TableInfo::version).orElse(null);
 
             final RowGroupReader rowGroupReader = parquetFileReader.getRowGroup(0, version);
-            final ColumnChunkReader groupingKeyReader, beginPosReader, endPosReader;
-            try (final SeekableChannelContext channelContext = channelsProvider.makeSingleUseContext()) {
-                groupingKeyReader =
-                        rowGroupReader.getColumnChunk(Collections.singletonList(GROUPING_KEY), channelContext);
-                beginPosReader = rowGroupReader.getColumnChunk(Collections.singletonList(BEGIN_POS), channelContext);
-                endPosReader = rowGroupReader.getColumnChunk(Collections.singletonList(END_POS), channelContext);
-            }
+            final ColumnChunkReader groupingKeyReader =
+                    rowGroupReader.getColumnChunk(Collections.singletonList(GROUPING_KEY));
+            final ColumnChunkReader beginPosReader =
+                    rowGroupReader.getColumnChunk(Collections.singletonList(BEGIN_POS));
+            final ColumnChunkReader endPosReader = rowGroupReader.getColumnChunk(Collections.singletonList(END_POS));
             if (groupingKeyReader == null || beginPosReader == null || endPosReader == null) {
                 log.warn().append("Index file ").append(indexFilePath)
                         .append(" is missing one or more expected columns for table location ")
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java
index d3652947bc3..7e664b000ac 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java
@@ -154,11 +154,8 @@ protected ParquetColumnLocation<Values> makeColumnLocation(@NotNull final String
         final String[] columnPath = parquetColumnNameToPath.get(parquetColumnName);
         final List<String> nameList =
                 columnPath == null ? Collections.singletonList(parquetColumnName) : Arrays.asList(columnPath);
-        final ColumnChunkReader[] columnChunkReaders;
-        try (final SeekableChannelContext channelContext = getChannelProvider().makeSingleUseContext()) {
-            columnChunkReaders = Arrays.stream(getRowGroupReaders())
-                    .map(rgr -> rgr.getColumnChunk(nameList, channelContext)).toArray(ColumnChunkReader[]::new);
-        }
+        final ColumnChunkReader[] columnChunkReaders = Arrays.stream(getRowGroupReaders())
+                .map(rgr -> rgr.getColumnChunk(nameList)).toArray(ColumnChunkReader[]::new);
         final boolean exists = Arrays.stream(columnChunkReaders).anyMatch(ccr -> ccr != null && ccr.numRows() > 0);
         return new ParquetColumnLocation<>(this, columnName, parquetColumnName,
                 exists ? columnChunkReaders : null,
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java
index f8ad63fe011..5301676e3b0 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java
@@ -62,7 +62,7 @@ private CreatorResult(
     private static boolean canUseOffsetIndexBasedPageStore(
             @NotNull final ColumnChunkReader columnChunkReader,
             @NotNull final ColumnDefinition<?> columnDefinition) {
-        if (columnChunkReader.getOffsetIndex() == null) {
+        if (!columnChunkReader.hasOffsetIndex()) {
             return false;
         }
         final String version = columnChunkReader.getVersion();
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
index 807837d13aa..d8881962d88 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
@@ -11,6 +11,7 @@
 import io.deephaven.parquet.table.pagestore.topage.ToPage;
 import io.deephaven.parquet.base.ColumnChunkReader;
 import io.deephaven.parquet.base.ColumnPageReader;
+import io.deephaven.util.channel.SeekableChannelContext;
 import io.deephaven.util.channel.SeekableChannelContext.ContextHolder;
 import org.apache.parquet.internal.column.columnindex.OffsetIndex;
 import org.jetbrains.annotations.NotNull;
@@ -56,12 +57,15 @@ private static final class PageState<ATTR extends Any> {
             final long mask,
             @NotNull final ToPage<ATTR, ?> toPage) throws IOException {
         super(pageCache, columnChunkReader, mask, toPage);
-        offsetIndex = columnChunkReader.getOffsetIndex();
+        try (final SeekableChannelContext channelContext =
+                columnChunkReader.getChannelsProvider().makeSingleUseContext()) {
+            offsetIndex = columnChunkReader.getOffsetIndex(channelContext);
+        }
         Assert.neqNull(offsetIndex, "offsetIndex");
         numPages = offsetIndex.getPageCount();
         Assert.gtZero(numPages, "numPages");
         pageStates = new AtomicReferenceArray<>(numPages);
-        columnPageDirectAccessor = columnChunkReader.getPageAccessor();
+        columnPageDirectAccessor = columnChunkReader.getPageAccessor(offsetIndex);
 
         if (numPages == 1) {
             fixedPageSize = numRows();

From 7a3652df97e20277fafa9b6eabad78823651cc83 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 26 Feb 2024 13:16:13 -0600
Subject: [PATCH 13/45] Fixed failing test

---
 .../parquet/table/ParquetTableReadWriteTest.java          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 0fc564d838c..818c793a559 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -1419,7 +1419,7 @@ public void readChangedUnderlyingFileTests() {
         readChangedUnderlyingFileTestsImpl(MULTI_WRITER);
     }
 
-    public void readChangedUnderlyingFileTestsImpl(TestParquetTableWriter writer) {
+    private void readChangedUnderlyingFileTestsImpl(TestParquetTableWriter writer) {
         // Write a table to parquet file and read it back
         final Table tableToSave = TableTools.emptyTable(5).update("A=(int)i", "B=(long)i", "C=(double)i");
         final String filename = "readChangedUnderlyingFileTests.parquet";
@@ -1439,9 +1439,9 @@ public void readChangedUnderlyingFileTestsImpl(TestParquetTableWriter writer) {
 
         // Read back fromDisk. Since the underlying file has changed, we expect this to fail.
         try {
-            fromDisk.coalesce();
-            TestCase.fail("Expected TableDataException");
-        } catch (TableDataException ignored) {
+            fromDisk.select();
+            TestCase.fail("Expected exception");
+        } catch (RuntimeException ignored) {
             // expected
         }
     }

From 1f41eefa83f597450c1383d6858a670d73605360 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 27 Feb 2024 13:56:35 -0600
Subject: [PATCH 14/45] Added support for partitioned parquet writing

---
 .../parquet/table/ParquetInstructions.java    | 61 ++++++++++++-
 .../table/ParquetMetadataFileWriterImpl.java  | 85 +++++++++++++++----
 .../parquet/table/ParquetTableWriter.java     | 17 ++++
 .../deephaven/parquet/table/ParquetTools.java | 81 +++++++++++++++++-
 .../table/ParquetTableReadWriteTest.java      | 67 ++++++++++++++-
 5 files changed, 287 insertions(+), 24 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
index 2cc99bad99d..ffe2e45b888 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
@@ -10,6 +10,7 @@
 import io.deephaven.hash.KeyedObjectKey;
 import io.deephaven.util.annotations.VisibleForTesting;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.schema.MessageType;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -117,8 +118,46 @@ public static int getDefaultTargetPageSize() {
 
     static final String DEFAULT_METADATA_ROOT_DIR = ""; // Empty = No metadata files written
 
+    private static final MessageType DEFAULT_COMMON_SCHEMA = null; // No common schema
+
     public ParquetInstructions() {}
 
+    /**
+     * Create a new ParquetInstructions object by copying all fields from the base instructions, and then adding the
+     * provided common schema. This method is intended for use internally and therefore is package-private.
+     *
+     * @param baseInstructions The base instructions to copy from
+     * @param commonSchema The common schema to add
+     * @return A new ParquetInstructions object with the common schema added
+     */
+    static ParquetInstructions createWithCommonSchema(@NotNull final ParquetInstructions baseInstructions,
+            @NotNull final MessageType commonSchema) {
+        if (!(baseInstructions instanceof ReadOnly)) {
+            throw new UnsupportedOperationException("Cannot add common schema to non-ReadOnly ParquetInstructions");
+        }
+        if (baseInstructions.getCommonSchema() != DEFAULT_COMMON_SCHEMA) {
+            throw new UnsupportedOperationException("Cannot add common schema to ParquetInstructions with existing " +
+                    "common schema");
+        }
+        if (DEFAULT_METADATA_ROOT_DIR.equals(baseInstructions.getMetadataRootDir())) {
+            throw new UnsupportedOperationException("Cannot add common schema to ParquetInstructions without any" +
+                    " metadata root directory");
+        }
+        final ReadOnly readOnly = (ReadOnly) baseInstructions;
+        return new ReadOnly(
+                readOnly.copyColumnNameToInstructions(),
+                readOnly.copyParquetColumnNameToInstructions(),
+                readOnly.getCompressionCodecName(),
+                readOnly.getMaximumDictionaryKeys(),
+                readOnly.getMaximumDictionarySize(),
+                readOnly.isLegacyParquet(),
+                readOnly.getTargetPageSize(),
+                readOnly.isRefreshing(),
+                readOnly.getSpecialInstructions(),
+                readOnly.getMetadataRootDir(),
+                commonSchema);
+    }
+
     public final String getColumnNameFromParquetColumnNameOrDefault(final String parquetColumnName) {
         final String mapped = getColumnNameFromParquetColumnName(parquetColumnName);
         return (mapped != null) ? mapped : parquetColumnName;
@@ -171,6 +210,11 @@ public final String getColumnNameFromParquetColumnNameOrDefault(final String par
      */
     public abstract String getMetadataRootDir();
 
+    /**
+     * @return the common schema used for writing _common_metadata file.
+     */
+    abstract MessageType getCommonSchema();
+
     @VisibleForTesting
     public static boolean sameColumnNamesAndCodecMappings(final ParquetInstructions i1, final ParquetInstructions i2) {
         if (i1 == EMPTY) {
@@ -250,6 +294,11 @@ public boolean isRefreshing() {
         public String getMetadataRootDir() {
             return DEFAULT_METADATA_ROOT_DIR;
         }
+
+        @Override
+        public MessageType getCommonSchema() {
+            return DEFAULT_COMMON_SCHEMA;
+        }
     };
 
     private static class ColumnInstructions {
@@ -319,6 +368,7 @@ private static final class ReadOnly extends ParquetInstructions {
         private final boolean isRefreshing;
         private final Object specialInstructions;
         private final String metadataRootDir;
+        private final MessageType commonSchema;
 
         private ReadOnly(
                 final KeyedObjectHashMap<String, ColumnInstructions> columnNameToInstructions,
@@ -330,7 +380,8 @@ private ReadOnly(
                 final int targetPageSize,
                 final boolean isRefreshing,
                 final Object specialInstructions,
-                final String metadataRootDir) {
+                final String metadataRootDir,
+                final MessageType commonSchema) {
             this.columnNameToInstructions = columnNameToInstructions;
             this.parquetColumnNameToInstructions = parquetColumnNameToColumnName;
             this.compressionCodecName = compressionCodecName;
@@ -341,6 +392,7 @@ private ReadOnly(
             this.isRefreshing = isRefreshing;
             this.specialInstructions = specialInstructions;
             this.metadataRootDir = metadataRootDir;
+            this.commonSchema = commonSchema;
         }
 
         private String getOrDefault(final String columnName, final String defaultValue,
@@ -439,6 +491,11 @@ public String getMetadataRootDir() {
             return metadataRootDir;
         }
 
+        @Override
+        public MessageType getCommonSchema() {
+            return commonSchema;
+        }
+
         KeyedObjectHashMap<String, ColumnInstructions> copyColumnNameToInstructions() {
             // noinspection unchecked
             return (columnNameToInstructions == null)
@@ -686,7 +743,7 @@ public ParquetInstructions build() {
             parquetColumnNameToInstructions = null;
             return new ReadOnly(columnNameToInstructionsOut, parquetColumnNameToColumnNameOut, compressionCodecName,
                     maximumDictionaryKeys, maximumDictionarySize, isLegacyParquet, targetPageSize, isRefreshing,
-                    specialInstructions, metadataRootDir);
+                    specialInstructions, metadataRootDir, DEFAULT_COMMON_SCHEMA);
         }
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index 2ce81e4ff28..7a39be5308a 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -14,6 +14,8 @@
 import org.apache.parquet.hadoop.metadata.FileMetaData;
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.apache.parquet.schema.MessageType;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.io.IOException;
@@ -47,9 +49,26 @@ private static class ParquetFileMetadata {
     private final String metadataRootDirAbsPath;
     private final List<ParquetFileMetadata> parquetFileMetadataList;
     private final SeekableChannelsProvider channelsProvider;
-    private List<ColumnTypeInfo> columnTypes; // Useful when merging deephaven specific metadata
+    private final MessageType commonSchema;
 
-    ParquetMetadataFileWriterImpl(final String metadataRootDir, final File[] destinations) {
+    // The following fields are used to accumulate metadata for all parquet files
+    private MessageType mergedSchema;
+    private String mergedCreatedByString;
+    private final Map<String, String> mergedKeyValueMetaData;
+    private final List<BlockMetaData> mergedBlocks;
+
+    /**
+     * Per-column type information stored in key-value metadata
+     */
+    private List<ColumnTypeInfo> columnTypes;
+
+    /**
+     * @param metadataRootDir The root directory for the metadata files
+     * @param destinations The indivdual parquet file destinations, all of which must be contained in the metadata root
+     * @param commonSchema The common schema to be included for writing the _common_metadata file.
+     */
+    ParquetMetadataFileWriterImpl(@NotNull final String metadataRootDir, @NotNull final File[] destinations,
+            @Nullable final MessageType commonSchema) {
         for (final File destination : destinations) {
             if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
                 throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
@@ -63,6 +82,12 @@ private static class ParquetFileMetadata {
         this.channelsProvider =
                 SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(rootDir.getPath()), null);
         this.columnTypes = null;
+        this.commonSchema = commonSchema;
+
+        this.mergedSchema = null;
+        this.mergedCreatedByString = null;
+        this.mergedKeyValueMetaData = new HashMap<>();
+        this.mergedBlocks = new ArrayList<>();
     }
 
     /**
@@ -75,19 +100,39 @@ public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata
         parquetFileMetadataList.add(new ParquetFileMetadata(parquetFile, metadata));
     }
 
+    /**
+     * Write the accumulated metadata to the provided files and clear the metadata accumulated so far.
+     *
+     * @param metadataFile The destination file for the _metadata file
+     * @param commonMetadataFile The destination file for the _common_metadata file
+     */
     public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) throws IOException {
-        final ParquetMetadata metadataFooter = mergeMetadata();
+        if (parquetFileMetadataList.isEmpty()) {
+            throw new UncheckedDeephavenException("No parquet files to write metadata for");
+        }
+        mergeMetadata();
+        final ParquetMetadata metadataFooter =
+                new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, mergedCreatedByString),
+                        mergedBlocks);
         writeMetadataFile(metadataFooter, metadataFile.getAbsolutePath());
 
-        metadataFooter.getBlocks().clear();
-        writeMetadataFile(metadataFooter, commonMetadataFile.toString());
-        parquetFileMetadataList.clear();
+        // Skip the blocks data and merge schema with the common schema to write the common metadata file
+        // The ordering of arguments in method call is important because we want common schema to determine the overall
+        // ordering of the schema fields.
+        mergedSchema = mergeSchemaInto(mergedSchema, commonSchema);
+        final ParquetMetadata commonMetadataFooter =
+                new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, mergedCreatedByString),
+                        new ArrayList<>());
+        writeMetadataFile(commonMetadataFooter, commonMetadataFile.toString());
+
+        // Clear the accumulated metadata
+        clear();
     }
 
-    private ParquetMetadata mergeMetadata() throws IOException {
-        MessageType mergedSchema = null;
-        final Map<String, String> mergedKeyValueMetaData = new HashMap<>();
-        final List<BlockMetaData> mergedBlocks = new ArrayList<>();
+    /**
+     * Merge all the accumulated metadata for the parquet files.
+     */
+    private void mergeMetadata() throws IOException {
         final Collection<String> mergedCreatedBy = new HashSet<>();
         for (final ParquetFileMetadata parquetFileMetadata : parquetFileMetadataList) {
             final FileMetaData fileMetaData = parquetFileMetadata.metadata.getFileMetaData();
@@ -96,12 +141,14 @@ private ParquetMetadata mergeMetadata() throws IOException {
             mergeBlocksInto(parquetFileMetadata, metadataRootDirAbsPath, mergedBlocks);
             mergedCreatedBy.add(fileMetaData.getCreatedBy());
         }
-        final String createdByString =
+        mergedCreatedByString =
                 mergedCreatedBy.size() == 1 ? mergedCreatedBy.iterator().next() : mergedCreatedBy.toString();
-        return new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, createdByString),
-                mergedBlocks);
     }
 
+    /**
+     * Merge the provided schema into the merged schema. Note that if there are common fields between the two schemas,
+     * the output schema will have the fields in the order they appear in the merged schema.
+     */
     private static MessageType mergeSchemaInto(final MessageType schema, final MessageType mergedSchema) {
         if (mergedSchema == null) {
             return schema;
@@ -126,7 +173,7 @@ private void mergeKeyValueMetaDataInto(final Map<String, String> keyValueMetaDat
                 }
             } else {
                 // For merging deephaven-specific metadata,
-                // - groupingColumns, dataIndexes should always be dropped
+                // - groupingColumns, dataIndexes are skipped
                 // - version is optional, so we read it from the first file's metadata
                 // - columnTypes must be the same for all partitions
                 final TableInfo tableInfo = TableInfo.deserializeFromJSON(entry.getValue());
@@ -135,9 +182,11 @@ private void mergeKeyValueMetaDataInto(final Map<String, String> keyValueMetaDat
                     Assert.eqNull(columnTypes, "columnTypes");
                     columnTypes = tableInfo.columnTypes();
                     mergedKeyValueMetaData.put(ParquetTableWriter.METADATA_KEY,
-                            TableInfo.builder().addAllColumnTypes(columnTypes)
+                            TableInfo.builder()
+                                    .addAllColumnTypes(columnTypes)
                                     .version(tableInfo.version())
-                                    .build().serializeToJSON());
+                                    .build()
+                                    .serializeToJSON());
                 } else if (!columnTypes.equals(tableInfo.columnTypes())) {
                     throw new UncheckedDeephavenException("Could not merge metadata for key " +
                             ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
@@ -174,6 +223,10 @@ private void writeMetadataFile(final ParquetMetadata metadataFooter, final Strin
 
     public void clear() {
         parquetFileMetadataList.clear();
+        mergedKeyValueMetaData.clear();
+        mergedBlocks.clear();
         columnTypes = null;
+        mergedSchema = null;
+        mergedCreatedByString = null;
     }
 }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index d92ca09d19b..5f1243483d3 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -36,6 +36,7 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.parquet.bytes.HeapByteBufferAllocator;
 import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Types;
 import org.jetbrains.annotations.NotNull;
@@ -229,6 +230,22 @@ private static void write(
         parquetFileWriter.close();
     }
 
+    /**
+     * Get the parquet schema for a table
+     *
+     * @param table the input table
+     * @param definition the table definition
+     * @param instructions write instructions for the file
+     * @return the parquet schema
+     */
+    static MessageType getSchemaForTable(@NotNull final Table table,
+            @NotNull final TableDefinition definition,
+            @NotNull final ParquetInstructions instructions) {
+        final Table pretransformTable = pretransformTable(table, definition);
+        return MappedSchema.create(new HashMap<>(), definition, pretransformTable.getRowSet(),
+                pretransformTable.getColumnSourceMap(), instructions).getParquetSchema();
+    }
+
     /**
      * Detect any missing or StringSet columns and convert them to arrays / null values as appropriate to prepare the
      * input table to be written to the parquet file.
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 0e14af40ff9..5642f990057 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -10,6 +10,7 @@
 import io.deephaven.base.verify.Require;
 import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.ColumnDefinition;
+import io.deephaven.engine.table.PartitionedTable;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.engine.table.impl.locations.util.TableDataRefreshService;
@@ -58,6 +59,7 @@
 import java.util.*;
 
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
+import static io.deephaven.parquet.table.ParquetTableWriter.getSchemaForTable;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 import static io.deephaven.parquet.table.ParquetTableWriter.PARQUET_FILE_EXTENSION;
 import static io.deephaven.util.type.TypeUtils.getUnboxedTypeIfBoxed;
@@ -461,15 +463,87 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
         return gcwim;
     }
 
+    /**
+     * Write tables to disk in parquet format with partitioning columns written as "key=value" format in a nested
+     * directory structure. To generate these individual partitions, this method will call
+     * {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns.
+     *
+     * @param sourceTable The table to partition and write
+     * @param destinationDir The destination directory to store partitioned data in. Non-existing directories are
+     *        created.
+     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
+     *        result in files named "partition1/table.parquet", "partition2/table.parquet", etc.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final File destinationDir,
+            @NotNull final String baseName,
+            @NotNull final ParquetInstructions writeInstructions) {
+        // TODO Should I take an additional write instruction for partitioning column names in case they are different
+        // from the table's partitioning columns? Also, I couldn't find a doc explaining how a user can convert a
+        // regular
+        // column to a partitioning column. So would be easier for users to just pass the partitioning column names.
+
+        // Also, how should I get the baseName, should I move it to the write instructions.
+        // Pyarrow has this optional parameter "basename_template" used to generate basenames of written data files.
+        // The token ‘{i}’ will be replaced with an automatically incremented integer for files in the same folder.
+        // If not specified, it defaults to “someHash-{i}.parquet”.
+        // pyspark has names of the form "part.{i}.parquet" and allows passing a naming function which takes an integer
+        // and generates a file name.
+
+        final TableDefinition sourceTableDefinition = sourceTable.getDefinition();
+        final List<ColumnDefinition<?>> partitioningColumns = sourceTableDefinition.getPartitioningColumns();
+        if (partitioningColumns.isEmpty()) {
+            throw new IllegalArgumentException("Table must have partitioning columns to write partitioned data");
+        }
+        final String[] partitioningColNames = partitioningColumns.stream()
+                .map(ColumnDefinition::getName)
+                .toArray(String[]::new);
+        final PartitionedTable partitionedData = sourceTable.partitionBy(partitioningColNames);
+        final Table keyTable = sourceTable.selectDistinct(partitioningColNames);
+        final Collection<Table> partitionedTables = new ArrayList<>();
+        final Collection<File> destinations = new ArrayList<>();
+        keyTable.getRowSet().forAllRowKeys(key -> {
+            final Object[] keyValues = Arrays.stream(partitioningColNames)
+                    .map(keyTable::getColumnSource)
+                    .map(colSource -> colSource.get(key))
+                    .toArray();
+            final StringBuilder partitionTableRelativePath = new StringBuilder();
+            for (int i = 0; i < partitioningColNames.length; i++) {
+                partitionTableRelativePath.append(File.separator)
+                        .append(partitioningColNames[i]).append("=").append(keyValues[i]);
+            }
+            partitionTableRelativePath.append(File.separator).append(baseName).append(".parquet");
+            destinations.add(new File(destinationDir, partitionTableRelativePath.toString()));
+            partitionedTables.add(partitionedData.constituentFor(keyValues));
+        });
+
+        // If needed, generate schema for _common_metadata file from source table
+        final ParquetInstructions updatedWriteInstructions;
+        if (!ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(writeInstructions.getMetadataRootDir())) {
+            final MessageType commonSchema = getSchemaForTable(sourceTable, sourceTableDefinition, writeInstructions);
+            updatedWriteInstructions = ParquetInstructions.createWithCommonSchema(writeInstructions, commonSchema);
+        } else {
+            updatedWriteInstructions = writeInstructions;
+        }
+        final TableDefinition partitionedTableDefinition = sourceTableDefinition.getWritable(false);
+        ParquetTools.writeParquetTables(
+                partitionedTables.toArray(Table[]::new),
+                partitionedTableDefinition,
+                updatedWriteInstructions,
+                destinations.toArray(File[]::new),
+                partitionedTableDefinition.getGroupingColumnNamesArray());
+    }
+
     /**
      * Writes tables to disk in parquet format to a supplied set of destinations. If you specify grouping columns, there
      * must already be grouping information for those columns in the sources. This can be accomplished with
      * {@code .groupBy(<grouping columns>).ungroup()} or {@code .sort(<grouping column>)}.
      *
      * @param sources The tables to write
-     * @param definition The common schema for all the tables to write
+     * @param definition The common definition for all the tables to write
      * @param writeInstructions Write instructions for customizations while writing
-     * @param destinations The destinations paths. Any non-existing directories in the paths provided are created. If
+     * @param destinations The destination paths. Any non-existing directories in the paths provided are created. If
      *        there is an error any intermediate directories previously created are removed; note this makes this method
      *        unsafe for concurrent use
      * @param groupingColumns List of columns the tables are grouped by (the write operation will store the grouping
@@ -497,7 +571,8 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         final String metadataRootDir = writeInstructions.getMetadataRootDir();
         final boolean writeMetadataFiles = !ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(metadataRootDir);
         if (writeMetadataFiles) {
-            metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations);
+            metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations,
+                    writeInstructions.getCommonSchema());
         } else {
             metadataFileWriter = NullParquetMetadataFileWriter.INSTANCE;
         }
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 818c793a559..ad9f4ee6bc4 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -16,7 +16,6 @@
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.SourceTable;
-import io.deephaven.engine.table.impl.locations.TableDataException;
 import io.deephaven.engine.table.impl.select.FunctionalColumn;
 import io.deephaven.engine.table.impl.select.SelectColumn;
 import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
@@ -107,6 +106,7 @@
 import static io.deephaven.parquet.table.ParquetTools.readSingleFileTable;
 import static io.deephaven.parquet.table.ParquetTools.readTable;
 import static io.deephaven.parquet.table.ParquetTools.writeParquetTables;
+import static io.deephaven.parquet.table.ParquetTools.writeKeyValuePartitionedTable;
 import static io.deephaven.parquet.table.ParquetTools.writeTable;
 import static io.deephaven.util.QueryConstants.*;
 import static org.junit.Assert.*;
@@ -516,7 +516,7 @@ public void parquetWithGroupingDataAndMetadataTest() {
     }
 
     @Test
-    public void flatPartitionedParquetWithMetadataTest() {
+    public void flatPartitionedParquetWithMetadataTest() throws IOException {
         // Create an empty parent directory
         final File parentDir = new File(rootFile, "tempDir");
         parentDir.mkdir();
@@ -529,7 +529,7 @@ public void flatPartitionedParquetWithMetadataTest() {
         // Write without any metadata files
         writeParquetTables(new Table[] {someTable, someTable}, someTable.getDefinition(), ParquetInstructions.EMPTY,
                 new File[] {firstDataFile, secondDataFile}, null);
-        final Table source = readTable(parentDir);
+        final Table source = readTable(parentDir).select();
 
         // Now write with metadata files
         parentDir.delete();
@@ -546,6 +546,67 @@ public void flatPartitionedParquetWithMetadataTest() {
         final File metadataFile = new File(parentDir, "_metadata");
         final Table fromDiskWithMetadata = readTable(metadataFile);
         assertTableEquals(source, fromDiskWithMetadata);
+
+        // Now replace the underlying data files with empty files and read the size from metadata file verifying that
+        // we can read the size without touching the data
+        firstDataFile.delete();
+        firstDataFile.createNewFile();
+        secondDataFile.delete();
+        secondDataFile.createNewFile();
+        final Table fromDiskWithMetadataWithoutData = readTable(metadataFile);
+        assertEquals(source.size(), fromDiskWithMetadataWithoutData.size());
+    }
+
+    @Test
+    public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofInt("PC1").withPartitioning(),
+                ColumnDefinition.ofInt("PC2").withPartitioning(),
+                ColumnDefinition.ofLong("I"));
+        final Table inputData = ((QueryTable) TableTools.emptyTable(20)
+                .updateView("PC1 = (int)(ii%3)",
+                        "PC2 = (int)(ii%2)",
+                        "I = ii"))
+                .withDefinitionUnsafe(definition);
+
+        final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
+
+        final File commonMetadata = new File(parentDir, "_common_metadata");
+        final Table fromDiskWithMetadata = readTable(commonMetadata);
+        assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
+    }
+
+    @Test
+    public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofInt("PC1").withPartitioning(),
+                ColumnDefinition.ofChar("PC2").withPartitioning(),
+                ColumnDefinition.ofString("PC3").withPartitioning(),
+                ColumnDefinition.ofLong("I"));
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10)
+                .updateView("PC1 = (int)(ii%3)",
+                        "PC2 = (char)(65 + (ii % 2))",
+                        "PC3 = java.time.LocalDate.ofEpochDay(i%5).toString()",
+                        "I = ii"))
+                .withDefinitionUnsafe(definition);
+
+        final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
+
+        final File commonMetadata = new File(parentDir, "_common_metadata");
+        final Table fromDiskWithMetadata = readTable(commonMetadata);
+        assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
     }
 
     @Test

From c8aa76462d32fcda0eb8fb5675aaec00b3beee4a Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 27 Feb 2024 15:23:38 -0600
Subject: [PATCH 15/45] Added some more tests

---
 .../parquet/table/ParquetTableReadWriteTest.java         | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index ad9f4ee6bc4..ae9721f51cd 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -601,12 +601,19 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
         writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
-        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
         final File commonMetadata = new File(parentDir, "_common_metadata");
         final Table fromDiskWithMetadata = readTable(commonMetadata);
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
+
+        // Delete some files from the partitioned data and read the required rows to verify that we only read the
+        // required partitions
+        FileUtils.deleteRecursivelyOnNFS(new File(parentDir, "PC1=0"));
+        FileUtils.deleteRecursivelyOnNFS(new File(parentDir, "PC1=1"));
+        fromDisk = readTable(commonMetadata).where("PC1 == 2");
+        assertTableEquals(inputData.where("PC1 == 2"), fromDisk);
     }
 
     @Test

From ff99a36387efa992459ea8342ebe290c05c24f7f Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 27 Feb 2024 15:53:59 -0600
Subject: [PATCH 16/45] Added some more tests

---
 .../table/ParquetTableReadWriteTest.java      | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index ae9721f51cd..fbe3c24bfde 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -616,6 +616,35 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         assertTableEquals(inputData.where("PC1 == 2"), fromDisk);
     }
 
+    @Test
+    public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofString("symbol").withPartitioning(),
+                ColumnDefinition.ofString("epic_collection_id").withPartitioning(),
+                ColumnDefinition.ofString("epic_request_id").withPartitioning(),
+                ColumnDefinition.ofLong("I"));
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10)
+                .updateView("symbol = (i % 2 == 0) ? `AA` : `BB`",
+                        "epic_collection_id = (i % 2 == 0) ? `fss_tick%1234%4321` : `fss_tick%5678%8765`",
+                        "epic_request_id = (i % 2 == 0) ? `223ea-asd43` : `98dce-oiu23`",
+                        "I = ii"))
+                .withDefinitionUnsafe(definition);
+
+        final File parentDir = new File(rootFile, "someTest");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        assertTableEquals(inputData.sort("symbol", "epic_collection_id"),
+                fromDisk.sort("symbol", "epic_collection_id"));
+
+        final File commonMetadata = new File(parentDir, "_common_metadata");
+        final Table fromDiskWithMetadata = readTable(commonMetadata);
+        assertTableEquals(inputData.sort("symbol", "epic_collection_id"),
+                fromDiskWithMetadata.sort("symbol", "epic_collection_id"));
+    }
+
     @Test
     public void testVectorColumns() {
         final Table table = getTableFlat(20000, true, false);

From 3e48937018e81c0c40714e982d6c6c03e382a339 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Wed, 28 Feb 2024 13:27:10 -0600
Subject: [PATCH 17/45] Added a new API for writing a partitioned table
 directly

---
 .../parquet/table/ParquetTableWriter.java     |  5 +-
 .../deephaven/parquet/table/ParquetTools.java | 82 ++++++++++++++-----
 .../table/ParquetTableReadWriteTest.java      |  9 +-
 3 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 5f1243483d3..f23e7304b63 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -239,10 +239,9 @@ private static void write(
      * @return the parquet schema
      */
     static MessageType getSchemaForTable(@NotNull final Table table,
-            @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions instructions) {
-        final Table pretransformTable = pretransformTable(table, definition);
-        return MappedSchema.create(new HashMap<>(), definition, pretransformTable.getRowSet(),
+        final Table pretransformTable = pretransformTable(table, table.getDefinition());
+        return MappedSchema.create(new HashMap<>(), table.getDefinition(), pretransformTable.getRowSet(),
                 pretransformTable.getColumnSourceMap(), instructions).getParquetSchema();
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 5642f990057..69f9e4c986e 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -57,6 +57,7 @@
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
+import java.util.stream.Collectors;
 
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 import static io.deephaven.parquet.table.ParquetTableWriter.getSchemaForTable;
@@ -464,9 +465,9 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
     }
 
     /**
-     * Write tables to disk in parquet format with partitioning columns written as "key=value" format in a nested
-     * directory structure. To generate these individual partitions, this method will call
-     * {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns.
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns.
      *
      * @param sourceTable The table to partition and write
      * @param destinationDir The destination directory to store partitioned data in. Non-existing directories are
@@ -480,9 +481,8 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         // TODO Should I take an additional write instruction for partitioning column names in case they are different
-        // from the table's partitioning columns? Also, I couldn't find a doc explaining how a user can convert a
-        // regular
-        // column to a partitioning column. So would be easier for users to just pass the partitioning column names.
+        // from the table's partitioning columns? Also, should I take partitioning column names from the user as an
+        // argument to this method?
 
         // Also, how should I get the baseName, should I move it to the write instructions.
         // Pyarrow has this optional parameter "basename_template" used to generate basenames of written data files.
@@ -490,49 +490,87 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
         // If not specified, it defaults to “someHash-{i}.parquet”.
         // pyspark has names of the form "part.{i}.parquet" and allows passing a naming function which takes an integer
         // and generates a file name.
-
-        final TableDefinition sourceTableDefinition = sourceTable.getDefinition();
-        final List<ColumnDefinition<?>> partitioningColumns = sourceTableDefinition.getPartitioningColumns();
+        final List<ColumnDefinition<?>> partitioningColumns = sourceTable.getDefinition().getPartitioningColumns();
         if (partitioningColumns.isEmpty()) {
             throw new IllegalArgumentException("Table must have partitioning columns to write partitioned data");
         }
         final String[] partitioningColNames = partitioningColumns.stream()
                 .map(ColumnDefinition::getName)
                 .toArray(String[]::new);
-        final PartitionedTable partitionedData = sourceTable.partitionBy(partitioningColNames);
-        final Table keyTable = sourceTable.selectDistinct(partitioningColNames);
-        final Collection<Table> partitionedTables = new ArrayList<>();
+        final PartitionedTable partitionedTable = sourceTable.partitionBy(partitioningColNames);
+        writeKeyValuePartitionedTable(partitionedTable, destinationDir, baseName, writeInstructions);
+    }
+
+    /**
+     * Write a partitioned tables to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param destinationDir The destination directory to store partitioned data in. Non-existing directories are
+     *        created.
+     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
+     *        result in files named "partition1/table.parquet", "partition2/table.parquet", etc.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final File destinationDir,
+            @NotNull final String baseName,
+            @NotNull final ParquetInstructions writeInstructions) {
+        final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
+        final Table keyTable;
+        if (partitionedTable.uniqueKeys()) {
+            keyTable = partitionedTable.table().view(partitioningColumnNames);
+        } else {
+            keyTable = partitionedTable.table().selectDistinct(partitioningColumnNames);
+        }
+        final Collection<Table> constituentTables = new ArrayList<>();
         final Collection<File> destinations = new ArrayList<>();
         keyTable.getRowSet().forAllRowKeys(key -> {
-            final Object[] keyValues = Arrays.stream(partitioningColNames)
+            final Object[] keyValues = Arrays.stream(partitioningColumnNames)
                     .map(keyTable::getColumnSource)
                     .map(colSource -> colSource.get(key))
                     .toArray();
             final StringBuilder partitionTableRelativePath = new StringBuilder();
-            for (int i = 0; i < partitioningColNames.length; i++) {
+            for (int i = 0; i < partitioningColumnNames.length; i++) {
                 partitionTableRelativePath.append(File.separator)
-                        .append(partitioningColNames[i]).append("=").append(keyValues[i]);
+                        .append(partitioningColumnNames[i]).append("=").append(keyValues[i]);
             }
             partitionTableRelativePath.append(File.separator).append(baseName).append(".parquet");
             destinations.add(new File(destinationDir, partitionTableRelativePath.toString()));
-            partitionedTables.add(partitionedData.constituentFor(keyValues));
+            constituentTables.add(partitionedTable.constituentFor(keyValues));
         });
 
-        // If needed, generate schema for _common_metadata file from source table
+        // If needed, generate schema for _common_metadata file from key table
         final ParquetInstructions updatedWriteInstructions;
         if (!ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(writeInstructions.getMetadataRootDir())) {
-            final MessageType commonSchema = getSchemaForTable(sourceTable, sourceTableDefinition, writeInstructions);
+            final MessageType commonSchema = getSchemaForTable(keyTable, writeInstructions);
             updatedWriteInstructions = ParquetInstructions.createWithCommonSchema(writeInstructions, commonSchema);
         } else {
             updatedWriteInstructions = writeInstructions;
         }
-        final TableDefinition partitionedTableDefinition = sourceTableDefinition.getWritable(false);
+        final TableDefinition constituentDefinition = getNonKeyTableDefiniton(partitionedTable);
         ParquetTools.writeParquetTables(
-                partitionedTables.toArray(Table[]::new),
-                partitionedTableDefinition,
+                constituentTables.toArray(Table[]::new),
+                constituentDefinition,
                 updatedWriteInstructions,
                 destinations.toArray(File[]::new),
-                partitionedTableDefinition.getGroupingColumnNamesArray());
+                constituentDefinition.getGroupingColumnNamesArray());
+    }
+
+    /**
+     * Create a table definition for the non-key columns of a partitioned table
+     */
+    private static TableDefinition getNonKeyTableDefiniton(@NotNull final PartitionedTable partitionedTable) {
+        final Collection<String> keyColumnNames = partitionedTable.keyColumnNames();
+        final List<ColumnDefinition<?>> nonKeyColumnDefinition =
+                partitionedTable.constituentDefinition()
+                        .getColumns().stream()
+                        .filter(columnDefinition -> {
+                            final String columnName = columnDefinition.getName();
+                            return !keyColumnNames.contains(columnName);
+                        }).collect(Collectors.toList());
+        return TableDefinition.of(nonKeyColumnDefinition);
     }
 
     /**
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index fbe3c24bfde..b8405605482 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -15,6 +15,7 @@
 import io.deephaven.engine.primitive.iterator.CloseableIterator;
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.ColumnSource;
+import io.deephaven.engine.table.PartitionedTable;
 import io.deephaven.engine.table.impl.SourceTable;
 import io.deephaven.engine.table.impl.select.FunctionalColumn;
 import io.deephaven.engine.table.impl.select.SelectColumn;
@@ -620,8 +621,8 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
     public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
         final TableDefinition definition = TableDefinition.of(
                 ColumnDefinition.ofString("symbol").withPartitioning(),
-                ColumnDefinition.ofString("epic_collection_id").withPartitioning(),
-                ColumnDefinition.ofString("epic_request_id").withPartitioning(),
+                ColumnDefinition.ofString("epic_collection_id"),
+                ColumnDefinition.ofString("epic_request_id"),
                 ColumnDefinition.ofLong("I"));
         final Table inputData = ((QueryTable) TableTools.emptyTable(10)
                 .updateView("symbol = (i % 2 == 0) ? `AA` : `BB`",
@@ -634,7 +635,9 @@ public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        final PartitionedTable partitionedTable =
+                inputData.partitionBy("symbol", "epic_collection_id", "epic_request_id");
+        writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         assertTableEquals(inputData.sort("symbol", "epic_collection_id"),
                 fromDisk.sort("symbol", "epic_collection_id"));

From e58434367de0400b31b6c505db33119ef477cf5a Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Thu, 29 Feb 2024 09:52:10 -0600
Subject: [PATCH 18/45] Improved the tests

---
 .../table/ParquetTableReadWriteTest.java      | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index b8405605482..e10c0ee2816 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -575,6 +575,17 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
         writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+
+        // Verify that the partitioned data exists
+        for (int PC1 = 0; PC1 <= 2; PC1++) {
+            for (int PC2 = 0; PC2 <= 1; PC2++) {
+                final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
+                assertTrue(dir.exists() && dir.isDirectory());
+                final File dataFile = new File(dir, "data.parquet");
+                assertTrue(dataFile.exists() && dataFile.isFile());
+            }
+        }
+
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
@@ -593,7 +604,7 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         final Table inputData = ((QueryTable) TableTools.emptyTable(10)
                 .updateView("PC1 = (int)(ii%3)",
                         "PC2 = (char)(65 + (ii % 2))",
-                        "PC3 = java.time.LocalDate.ofEpochDay(i%5).toString()",
+                        "PC3 = java.time.LocalDate.ofEpochDay(i%2).toString()",
                         "I = ii"))
                 .withDefinitionUnsafe(definition);
 
@@ -602,6 +613,19 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
         writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+
+        // Verify that the partitioned data exists
+        for (int PC1 = 0; PC1 <= 2; PC1++) {
+            for (int idx = 0; idx <= 1; idx++) {
+                final char PC2 = (char) ('A' + idx);
+                final String PC3 = java.time.LocalDate.ofEpochDay(idx).toString();
+                final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2 +
+                        File.separator + "PC3=" + PC3);
+                assertTrue(dir.exists() && dir.isDirectory());
+                final File dataFile = new File(dir, "data.parquet");
+                assertTrue(dataFile.exists() && dataFile.isFile());
+            }
+        }
         Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
@@ -613,8 +637,8 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         // required partitions
         FileUtils.deleteRecursivelyOnNFS(new File(parentDir, "PC1=0"));
         FileUtils.deleteRecursivelyOnNFS(new File(parentDir, "PC1=1"));
-        fromDisk = readTable(commonMetadata).where("PC1 == 2");
-        assertTableEquals(inputData.where("PC1 == 2"), fromDisk);
+        assertTableEquals(inputData.where("PC1 == 2").sort("PC1", "PC2", "PC3"),
+                readTable(commonMetadata).where("PC1 == 2").sort("PC1", "PC2", "PC3"));
     }
 
     @Test
@@ -638,6 +662,11 @@ public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
         final PartitionedTable partitionedTable =
                 inputData.partitionBy("symbol", "epic_collection_id", "epic_request_id");
         writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
+        assertTrue(new File(parentDir,
+                "symbol=AA/epic_collection_id=fss_tick%1234%4321/epic_request_id=223ea-asd43/data.parquet").exists());
+        assertTrue(new File(parentDir,
+                "symbol=BB/epic_collection_id=fss_tick%5678%8765/epic_request_id=98dce-oiu23/data.parquet").exists());
+
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         assertTableEquals(inputData.sort("symbol", "epic_collection_id"),
                 fromDisk.sort("symbol", "epic_collection_id"));
@@ -1025,7 +1054,7 @@ private interface TestParquetTableWriter {
      */
     private static void verifyFilesInDir(final File parentDir, final String[] expectedDataFiles,
             @Nullable final Map<String, String[]> indexingColumnToFileMap) {
-        final List filesInParentDir = Arrays.asList(parentDir.list());
+        final List<String> filesInParentDir = Arrays.asList(parentDir.list());
         for (String expectedFile : expectedDataFiles) {
             assertTrue(filesInParentDir.contains(expectedFile));
         }

From f659f3cbd9b07b6b5f9cb8f52dfaf1f5da63d317 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Sun, 3 Mar 2024 18:34:33 -0600
Subject: [PATCH 19/45] Review with Ryan part 1

---
 .../runner/TableBenchmarkState.java           |   5 +-
 .../table/impl/QueryTableAggregationTest.java |   4 +-
 .../parquet/table/ParquetTableWriter.java     |   8 +-
 .../deephaven/parquet/table/ParquetTools.java | 177 +++++++++++++-----
 .../deephaven/parquet/table/ParquetUtils.java |  18 ++
 .../parquet/table/PartitionFormatter.java     | 134 +++++++++++++
 .../table/layout/ParquetFileHelper.java       |  15 --
 .../layout/ParquetFlatPartitionedLayout.java  |   3 +-
 .../ParquetKeyValuePartitionedLayout.java     |   5 +-
 .../layout/ParquetMetadataFileLayout.java     |   5 +-
 .../location/ParquetTableLocationKey.java     |   7 +-
 .../table/ParquetTableReadWriteTest.java      |  78 +++++++-
 12 files changed, 369 insertions(+), 90 deletions(-)
 create mode 100644 extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java
 create mode 100644 extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
 delete mode 100644 extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFileHelper.java

diff --git a/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java b/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java
index b6bf36b1a5b..93a982c275d 100644
--- a/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java
+++ b/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java
@@ -8,11 +8,12 @@
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.parquet.table.ParquetTools;
 import io.deephaven.engine.util.TableTools;
-import io.deephaven.parquet.table.ParquetTableWriter;
 import io.deephaven.engine.table.impl.util.TableBuilder;
 import io.deephaven.benchmarking.BenchmarkTools;
 import org.openjdk.jmh.infra.BenchmarkParams;
 
+import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
+
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -44,7 +45,7 @@ public void init() {
 
     public void logOutput() throws IOException {
         final Path outputPath = BenchmarkTools.dataDir()
-                .resolve(BenchmarkTools.getDetailOutputPath(benchmarkName) + ParquetTableWriter.PARQUET_FILE_EXTENSION);
+                .resolve(BenchmarkTools.getDetailOutputPath(benchmarkName) + PARQUET_FILE_EXTENSION);
 
         final Table output = outputBuilder.build();
         ParquetTools.writeTable(output, outputPath.toFile(), RESULT_DEF);
diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
index 6b4a2d9d278..00636c04aad 100644
--- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
+++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
@@ -37,7 +37,6 @@
 import io.deephaven.engine.util.TableTools;
 import io.deephaven.engine.util.systemicmarking.SystemicObjectTracker;
 import io.deephaven.parquet.table.ParquetInstructions;
-import io.deephaven.parquet.table.ParquetTableWriter;
 import io.deephaven.parquet.table.ParquetTools;
 import io.deephaven.parquet.table.layout.ParquetKeyValuePartitionedLayout;
 import io.deephaven.test.types.OutOfBandTest;
@@ -75,6 +74,7 @@
 import static io.deephaven.util.QueryConstants.*;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.*;
+import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
 
 @Category(OutOfBandTest.class)
 public class QueryTableAggregationTest {
@@ -3957,7 +3957,7 @@ private Table makeDiskTable(File directory) throws IOException {
         final TableDefaults result = testTable(stringCol("Symbol", syms),
                 intCol("Value", values));
 
-        final File outputFile = new File(directory, "disk_table" + ParquetTableWriter.PARQUET_FILE_EXTENSION);
+        final File outputFile = new File(directory, "disk_table" + PARQUET_FILE_EXTENSION);
 
         ParquetTools.writeTable(result, outputFile, result.getDefinition());
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index f23e7304b63..e8fa8b8107e 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -57,7 +57,6 @@ public class ParquetTableWriter {
     public static final String BEGIN_POS = "dh_begin_pos";
     public static final String END_POS = "dh_end_pos";
     public static final String GROUPING_KEY = "dh_key";
-    public static final String PARQUET_FILE_EXTENSION = ".parquet";
 
     /**
      * Helper struct used to pass information about where to write the grouping files for each grouping column
@@ -234,14 +233,15 @@ private static void write(
      * Get the parquet schema for a table
      *
      * @param table the input table
-     * @param definition the table definition
+     * @param definition the definition to use for creating the schema
      * @param instructions write instructions for the file
      * @return the parquet schema
      */
     static MessageType getSchemaForTable(@NotNull final Table table,
+            @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions instructions) {
-        final Table pretransformTable = pretransformTable(table, table.getDefinition());
-        return MappedSchema.create(new HashMap<>(), table.getDefinition(), pretransformTable.getRowSet(),
+        final Table pretransformTable = pretransformTable(table, definition);
+        return MappedSchema.create(new HashMap<>(), definition, pretransformTable.getRowSet(),
                 pretransformTable.getColumnSourceMap(), instructions).getParquetSchema();
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 69f9e4c986e..b05e53d5a6b 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -10,6 +10,7 @@
 import io.deephaven.base.verify.Require;
 import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.ColumnDefinition;
+import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.PartitionedTable;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
@@ -59,10 +60,13 @@
 import java.util.*;
 import java.util.stream.Collectors;
 
+import static io.deephaven.engine.table.impl.partitioned.PartitionedTableCreatorImpl.CONSTITUENT;
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 import static io.deephaven.parquet.table.ParquetTableWriter.getSchemaForTable;
+import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
+import static io.deephaven.parquet.table.ParquetUtils.COMMON_METADATA_FILE_NAME;
+import static io.deephaven.parquet.table.ParquetUtils.METADATA_FILE_NAME;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
-import static io.deephaven.parquet.table.ParquetTableWriter.PARQUET_FILE_EXTENSION;
 import static io.deephaven.util.type.TypeUtils.getUnboxedTypeIfBoxed;
 
 /**
@@ -72,8 +76,6 @@
 public class ParquetTools {
 
     private static final int MAX_PARTITIONING_LEVELS_INFERENCE = 32;
-    private static final String PARQUET_METADATA_FILE = "_metadata";
-    private static final String PARQUET_COMMON_METADATA_FILE = "_common_metadata";
 
     private ParquetTools() {}
 
@@ -470,27 +472,48 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns.
      *
      * @param sourceTable The table to partition and write
-     * @param destinationDir The destination directory to store partitioned data in. Non-existing directories are
-     *        created.
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "partition1/table.parquet", "partition2/table.parquet", etc.
+     *        result in files named "PartitioningColumn=partition1/table.parquet",
+     *        "PartitioningColumn=partition2/table.parquet", etc.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
             @NotNull final File destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
-        // TODO Should I take an additional write instruction for partitioning column names in case they are different
-        // from the table's partitioning columns? Also, should I take partitioning column names from the user as an
-        // argument to this method?
+        writeKeyValuePartitionedTable(sourceTable, sourceTable.getDefinition(), destinationDir, baseName,
+                writeInstructions);
+    }
 
-        // Also, how should I get the baseName, should I move it to the write instructions.
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
+     * table definition.
+     *
+     * @param sourceTable The table to partition and write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
+     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
+     *        result in files named "PartitioningColumn=partition1/table.parquet",
+     *        "PartitioningColumn=partition2/table.parquet", etc.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final File destinationDir,
+            @NotNull final String baseName,
+            @NotNull final ParquetInstructions writeInstructions) {
+        // TODO Also, how should I get the baseName, should I move it to the write instructions.
         // Pyarrow has this optional parameter "basename_template" used to generate basenames of written data files.
         // The token ‘{i}’ will be replaced with an automatically incremented integer for files in the same folder.
         // If not specified, it defaults to “someHash-{i}.parquet”.
         // pyspark has names of the form "part.{i}.parquet" and allows passing a naming function which takes an integer
         // and generates a file name.
-        final List<ColumnDefinition<?>> partitioningColumns = sourceTable.getDefinition().getPartitioningColumns();
+        final List<ColumnDefinition<?>> partitioningColumns = definition.getPartitioningColumns();
         if (partitioningColumns.isEmpty()) {
             throw new IllegalArgumentException("Table must have partitioning columns to write partitioned data");
         }
@@ -498,74 +521,130 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
                 .map(ColumnDefinition::getName)
                 .toArray(String[]::new);
         final PartitionedTable partitionedTable = sourceTable.partitionBy(partitioningColNames);
-        writeKeyValuePartitionedTable(partitionedTable, destinationDir, baseName, writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, baseName, writeInstructions);
     }
 
     /**
-     * Write a partitioned tables to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
      * {@link Table#partitionBy(String...) partitionBy} on the required columns.
      *
      * @param partitionedTable The partitioned table to write
-     * @param destinationDir The destination directory to store partitioned data in. Non-existing directories are
-     *        created.
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "partition1/table.parquet", "partition2/table.parquet", etc.
+     *        result in files named "PartitioningColumn=partition1/table.parquet",
+     *        "PartitioningColumn=partition2/table.parquet", etc.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final File destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
+        // Get key column definitions from the partitioned table definition and non-key column definitions from the
+        // constituent table definition, and combine them to build the overall table definition
+        final Set<String> keyColumnNames = partitionedTable.keyColumnNames();
+        final Collection<ColumnDefinition<?>> columnDefinitions = new ArrayList<>(keyColumnNames.size() +
+                partitionedTable.constituentDefinition().numColumns());
+        partitionedTable.table().getDefinition().getColumns().stream()
+                .filter(columnDefinition -> keyColumnNames.contains(columnDefinition.getName()))
+                .forEach(columnDefinitions::add);
+        partitionedTable.constituentDefinition().getColumns().stream()
+                .filter(columnDefinition -> !keyColumnNames.contains(columnDefinition.getName()))
+                .forEach(columnDefinitions::add);
+        final TableDefinition definition = TableDefinition.of(columnDefinitions);
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, baseName, writeInstructions);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
+     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
+     *        result in files named "PartitioningColumn=partition1/table.parquet",
+     *        "PartitioningColumn=partition2/table.parquet", etc.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final File destinationDir,
+            @NotNull final String baseName,
+            @NotNull final ParquetInstructions writeInstructions) {
         final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
-        final Table keyTable;
-        if (partitionedTable.uniqueKeys()) {
-            keyTable = partitionedTable.table().view(partitioningColumnNames);
-        } else {
-            keyTable = partitionedTable.table().selectDistinct(partitioningColumnNames);
+        if (partitionedTable.table().numColumns() == partitioningColumnNames.length) {
+            throw new IllegalArgumentException(
+                    "Cannot write a partitioned parquet table with no non-partitioning columns");
         }
-        final Collection<Table> constituentTables = new ArrayList<>();
+        // Note that there can be multiple constituents with the same key values, so cannot directly use the
+        // partitionedTable.constituentFor(keyValues) method, and we need to group them together
+        final Table withGroupConstituents = partitionedTable.table().groupBy(partitioningColumnNames);
+        final ColumnSource<ObjectVector<Table>> consituentVectorColumnSource =
+                withGroupConstituents.getColumnSource(CONSTITUENT.name());
+        if (consituentVectorColumnSource == null) {
+            throw new IllegalStateException("Partitioned table must have a constituent column");
+        }
+        final Collection<Table> partitionedData = new ArrayList<>();
         final Collection<File> destinations = new ArrayList<>();
-        keyTable.getRowSet().forAllRowKeys(key -> {
-            final Object[] keyValues = Arrays.stream(partitioningColumnNames)
-                    .map(keyTable::getColumnSource)
-                    .map(colSource -> colSource.get(key))
-                    .toArray();
-            final StringBuilder partitionTableRelativePath = new StringBuilder();
-            for (int i = 0; i < partitioningColumnNames.length; i++) {
-                partitionTableRelativePath.append(File.separator)
-                        .append(partitioningColumnNames[i]).append("=").append(keyValues[i]);
+        withGroupConstituents.getRowSet().forAllRowKeys(key -> {
+            final StringBuilder relativePathBuilder = new StringBuilder();
+            for (final String partitioningColumnName : partitioningColumnNames) {
+                final ColumnSource<?> partitioningColSource =
+                        withGroupConstituents.getColumnSource(partitioningColumnName);
+                final String partitioningValue = PartitionFormatter.formatToString(partitioningColSource.get(key));
+                relativePathBuilder.append(partitioningColumnName).append("=").append(partitioningValue)
+                        .append(File.separator);
+            }
+            final String relativePath = relativePathBuilder.toString();
+            final ObjectVector<? extends Table> constituentVector = consituentVectorColumnSource.get(key);
+            if (constituentVector == null) {
+                throw new IllegalStateException("Grouped partitioned table must have a vector constituent column for" +
+                        " key = " + key);
+            }
+            int count = 0;
+            final long numConstituents = constituentVector.size();
+            for (final Table constituent : constituentVector) {
+                final File destination;
+                if (numConstituents == 1) {
+                    destination = new File(destinationDir, relativePath + baseName + ".parquet");
+                } else {
+                    destination = new File(destinationDir, relativePath + baseName + "-part-" + count + ".parquet");
+                }
+                destinations.add(destination);
+                partitionedData.add(constituent);
+                count++;
             }
-            partitionTableRelativePath.append(File.separator).append(baseName).append(".parquet");
-            destinations.add(new File(destinationDir, partitionTableRelativePath.toString()));
-            constituentTables.add(partitionedTable.constituentFor(keyValues));
         });
 
         // If needed, generate schema for _common_metadata file from key table
         final ParquetInstructions updatedWriteInstructions;
         if (!ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(writeInstructions.getMetadataRootDir())) {
-            final MessageType commonSchema = getSchemaForTable(keyTable, writeInstructions);
+            final MessageType commonSchema = getSchemaForTable(partitionedTable.table(), definition, writeInstructions);
             updatedWriteInstructions = ParquetInstructions.createWithCommonSchema(writeInstructions, commonSchema);
         } else {
             updatedWriteInstructions = writeInstructions;
         }
-        final TableDefinition constituentDefinition = getNonKeyTableDefiniton(partitionedTable);
+        final TableDefinition partitionedTablesDefinition = getNonKeyTableDefiniton(partitionedTable.keyColumnNames(),
+                definition);
         ParquetTools.writeParquetTables(
-                constituentTables.toArray(Table[]::new),
-                constituentDefinition,
+                partitionedData.toArray(Table[]::new),
+                partitionedTablesDefinition,
                 updatedWriteInstructions,
                 destinations.toArray(File[]::new),
-                constituentDefinition.getGroupingColumnNamesArray());
+                partitionedTablesDefinition.getGroupingColumnNamesArray());
     }
 
     /**
-     * Create a table definition for the non-key columns of a partitioned table
+     * Using the provided definition and key column names, create a sub table definition for the non-key columns.
      */
-    private static TableDefinition getNonKeyTableDefiniton(@NotNull final PartitionedTable partitionedTable) {
-        final Collection<String> keyColumnNames = partitionedTable.keyColumnNames();
+    private static TableDefinition getNonKeyTableDefiniton(@NotNull final Collection<String> keyColumnNames,
+            @NotNull final TableDefinition definition) {
         final List<ColumnDefinition<?>> nonKeyColumnDefinition =
-                partitionedTable.constituentDefinition()
-                        .getColumns().stream()
+                definition.getColumns().stream()
                         .filter(columnDefinition -> {
                             final String columnName = columnDefinition.getName();
                             return !keyColumnNames.contains(columnName);
@@ -658,10 +737,10 @@ public static void writeParquetTables(@NotNull final Table[] sources,
             // Write the combined metadata files to shadow destinations
             final File metadataDestFile, shadowMetadataFile, commonMetadataDestFile, shadowCommonMetadataFile;
             if (writeMetadataFiles) {
-                metadataDestFile = new File(metadataRootDir, PARQUET_METADATA_FILE);
+                metadataDestFile = new File(metadataRootDir, METADATA_FILE_NAME);
                 shadowMetadataFile = ParquetTools.getShadowFile(metadataDestFile);
                 shadowFiles.add(shadowMetadataFile);
-                commonMetadataDestFile = new File(metadataRootDir, PARQUET_COMMON_METADATA_FILE);
+                commonMetadataDestFile = new File(metadataRootDir, COMMON_METADATA_FILE_NAME);
                 shadowCommonMetadataFile = ParquetTools.getShadowFile(commonMetadataDestFile);
                 shadowFiles.add(shadowCommonMetadataFile);
                 metadataFileWriter.writeMetadataFiles(shadowMetadataFile, shadowCommonMetadataFile);
@@ -781,17 +860,17 @@ private static Table readTableInternal(
             if (sourceFileName.endsWith(PARQUET_FILE_EXTENSION)) {
                 return readSingleFileTable(source, instructions);
             }
-            if (sourceFileName.equals(ParquetMetadataFileLayout.METADATA_FILE_NAME)) {
+            if (sourceFileName.equals(METADATA_FILE_NAME)) {
                 return readPartitionedTableWithMetadata(sourceFile.getParentFile(), instructions);
             }
-            if (sourceFileName.equals(ParquetMetadataFileLayout.COMMON_METADATA_FILE_NAME)) {
+            if (sourceFileName.equals(COMMON_METADATA_FILE_NAME)) {
                 return readPartitionedTableWithMetadata(sourceFile.getParentFile(), instructions);
             }
             throw new TableDataException(
                     "Source file " + sourceFile + " does not appear to be a parquet file or metadata file");
         }
         if (sourceAttr.isDirectory()) {
-            final Path metadataPath = sourcePath.resolve(ParquetMetadataFileLayout.METADATA_FILE_NAME);
+            final Path metadataPath = sourcePath.resolve(METADATA_FILE_NAME);
             if (Files.exists(metadataPath)) {
                 return readPartitionedTableWithMetadata(sourceFile, instructions);
             }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java
new file mode 100644
index 00000000000..87932f18b54
--- /dev/null
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java
@@ -0,0 +1,18 @@
+package io.deephaven.parquet.table;
+
+import java.nio.file.Path;
+
+public final class ParquetUtils {
+
+    public static final String METADATA_FILE_NAME = "_metadata";
+    public static final String COMMON_METADATA_FILE_NAME = "_common_metadata";
+    public static final String PARQUET_FILE_EXTENSION = ".parquet";
+
+    /**
+     * Used as a filter to select relevant parquet files while reading all files in a directory.
+     */
+    public static boolean fileNameMatches(final Path path) {
+        final String fileName = path.getFileName().toString();
+        return fileName.endsWith(PARQUET_FILE_EXTENSION) && fileName.charAt(0) != '.';
+    }
+}
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
new file mode 100644
index 00000000000..ef50f74f0ba
--- /dev/null
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
@@ -0,0 +1,134 @@
+package io.deephaven.parquet.table;
+
+import io.deephaven.time.DateTimeUtils;
+import org.jetbrains.annotations.NotNull;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This class takes an object read from a parquet file and formats it to a string, only for the supported types.
+ */
+enum PartitionFormatter {
+    ForString {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return (String) obj;
+        }
+    },
+    ForBoolean {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Boolean) obj).toString();
+        }
+    },
+    ForChar {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Character) obj).toString();
+        }
+    },
+    ForByte {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Byte) obj).toString();
+        }
+    },
+    ForShort {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Short) obj).toString();
+        }
+    },
+    ForInt {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Integer) obj).toString();
+        }
+    },
+    ForLong {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Long) obj).toString();
+        }
+    },
+    ForFloat {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Float) obj).toString();
+        }
+    },
+    ForDouble {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Double) obj).toString();
+        }
+    },
+    ForBigInteger {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((BigInteger) obj).toString();
+        }
+    },
+    ForBigDecimal {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((BigDecimal) obj).toString();
+        }
+    },
+    ForInstant {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((Instant) obj).toString();
+        }
+    },
+    ForLocalDate {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return DateTimeUtils.formatDate((LocalDate) obj);
+        }
+    },
+    ForLocalTime {
+        @Override
+        public String format(@NotNull final Object obj) {
+            return ((LocalTime) obj).toString();
+        }
+    };
+
+    private static final Map<Class<?>, PartitionFormatter> typeMap = new HashMap<>();
+    static {
+        typeMap.put(String.class, ForString);
+        typeMap.put(Boolean.class, ForBoolean);
+        typeMap.put(Character.class, ForChar);
+        typeMap.put(Byte.class, ForByte);
+        typeMap.put(Short.class, ForShort);
+        typeMap.put(Integer.class, ForInt);
+        typeMap.put(Long.class, ForLong);
+        typeMap.put(Float.class, ForFloat);
+        typeMap.put(Double.class, ForDouble);
+        typeMap.put(BigInteger.class, ForBigInteger);
+        typeMap.put(BigDecimal.class, ForBigDecimal);
+        typeMap.put(Instant.class, ForInstant);
+        typeMap.put(LocalDate.class, ForLocalDate);
+        typeMap.put(LocalTime.class, ForLocalTime);
+    }
+
+    abstract String format(@NotNull Object obj);
+
+    static String formatToString(final Object obj) {
+        if (obj == null) {
+            return "null";
+        }
+        final PartitionFormatter formatter = typeMap.get(obj.getClass());
+        if (formatter != null) {
+            return formatter.format(obj);
+        } else {
+            throw new UnsupportedOperationException("Unsupported type: " + obj.getClass().getSimpleName());
+        }
+    }
+}
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFileHelper.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFileHelper.java
deleted file mode 100644
index 86dbc5ad3fa..00000000000
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFileHelper.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package io.deephaven.parquet.table.layout;
-
-import io.deephaven.parquet.table.ParquetTableWriter;
-
-import java.nio.file.Path;
-
-final class ParquetFileHelper {
-    /**
-     * Used as a filter to select relevant parquet files while reading all files in a directory.
-     */
-    static boolean fileNameMatches(final Path path) {
-        final String fileName = path.getFileName().toString();
-        return fileName.endsWith(ParquetTableWriter.PARQUET_FILE_EXTENSION) && fileName.charAt(0) != '.';
-    }
-}
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java
index bbaefc5d971..6e2ef6c000c 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java
@@ -7,6 +7,7 @@
 import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder;
 import io.deephaven.parquet.table.ParquetInstructions;
 import io.deephaven.parquet.table.location.ParquetTableLocationKey;
+import io.deephaven.parquet.table.ParquetUtils;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
@@ -49,7 +50,7 @@ public String toString() {
     @Override
     public synchronized void findKeys(@NotNull final Consumer<ParquetTableLocationKey> locationKeyObserver) {
         try (final DirectoryStream<Path> parquetFileStream =
-                Files.newDirectoryStream(tableRootDirectory.toPath(), ParquetFileHelper::fileNameMatches)) {
+                Files.newDirectoryStream(tableRootDirectory.toPath(), ParquetUtils::fileNameMatches)) {
             for (final Path parquetFilePath : parquetFileStream) {
                 ParquetTableLocationKey locationKey = cache.get(parquetFilePath);
                 if (locationKey == null) {
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java
index b1ea92a4b70..2404a69fcd2 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java
@@ -10,6 +10,7 @@
 import io.deephaven.engine.table.impl.locations.local.LocationTableBuilderDefinition;
 import io.deephaven.parquet.table.ParquetInstructions;
 import io.deephaven.parquet.table.location.ParquetTableLocationKey;
+import io.deephaven.parquet.table.ParquetUtils;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
@@ -27,7 +28,7 @@ public ParquetKeyValuePartitionedLayout(
             @NotNull final TableDefinition tableDefinition,
             @NotNull final ParquetInstructions readInstructions) {
         super(tableRootDirectory,
-                ParquetFileHelper::fileNameMatches,
+                ParquetUtils::fileNameMatches,
                 () -> new LocationTableBuilderDefinition(tableDefinition),
                 (path, partitions) -> new ParquetTableLocationKey(path.toFile(), 0, partitions, readInstructions),
                 Math.toIntExact(tableDefinition.getColumnStream().filter(ColumnDefinition::isPartitioning).count()));
@@ -38,7 +39,7 @@ public ParquetKeyValuePartitionedLayout(
             final int maxPartitioningLevels,
             @NotNull final ParquetInstructions readInstructions) {
         super(tableRootDirectory,
-                ParquetFileHelper::fileNameMatches,
+                ParquetUtils::fileNameMatches,
                 () -> new LocationTableBuilderCsv(tableRootDirectory),
                 (path, partitions) -> new ParquetTableLocationKey(path.toFile(), 0, partitions, readInstructions),
                 maxPartitioningLevels);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
index a7ed9a754c3..75ffb95913e 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
@@ -36,6 +36,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static io.deephaven.parquet.table.ParquetUtils.COMMON_METADATA_FILE_NAME;
+import static io.deephaven.parquet.table.ParquetUtils.METADATA_FILE_NAME;
 import static java.util.stream.Collectors.toMap;
 
 /**
@@ -55,9 +57,6 @@
  */
 public class ParquetMetadataFileLayout implements TableLocationKeyFinder<ParquetTableLocationKey> {
 
-    public static final String METADATA_FILE_NAME = "_metadata";
-    public static final String COMMON_METADATA_FILE_NAME = "_common_metadata";
-
     private final File metadataFile;
     private final File commonMetadataFile;
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
index f8f4a24556e..cdc6d3b5a02 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
@@ -8,7 +8,6 @@
 import io.deephaven.parquet.table.ParquetTools;
 import io.deephaven.engine.table.impl.locations.TableDataException;
 import io.deephaven.engine.table.impl.locations.TableLocationKey;
-import io.deephaven.parquet.table.ParquetTableWriter;
 import io.deephaven.parquet.base.ParquetFileReader;
 import org.apache.parquet.format.converter.ParquetMetadataConverter;
 import org.apache.parquet.format.RowGroup;
@@ -23,6 +22,8 @@
 import java.util.Map;
 import java.util.stream.IntStream;
 
+import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
+
 /**
  * {@link TableLocationKey} implementation for use with data stored in the parquet format.
  */
@@ -74,8 +75,8 @@ private static URI validateParquetFile(@NotNull final File file) {
     }
 
     private static URI validateParquetFile(@NotNull final URI parquetFileUri) {
-        if (!parquetFileUri.getRawPath().endsWith(ParquetTableWriter.PARQUET_FILE_EXTENSION)) {
-            throw new IllegalArgumentException("Parquet file must end in " + ParquetTableWriter.PARQUET_FILE_EXTENSION);
+        if (!parquetFileUri.getRawPath().endsWith(PARQUET_FILE_EXTENSION)) {
+            throw new IllegalArgumentException("Parquet file must end in " + PARQUET_FILE_EXTENSION);
         }
         return parquetFileUri;
     }
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index e10c0ee2816..98f59d81b9b 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -34,6 +34,7 @@
 import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore;
 import io.deephaven.parquet.table.transfer.StringDictionary;
 import io.deephaven.extensions.s3.S3Instructions;
+import io.deephaven.qst.type.Type;
 import io.deephaven.stringset.ArrayStringSet;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
@@ -71,6 +72,8 @@
 import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -598,21 +601,31 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
     public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         final TableDefinition definition = TableDefinition.of(
                 ColumnDefinition.ofInt("PC1").withPartitioning(),
-                ColumnDefinition.ofChar("PC2").withPartitioning(),
-                ColumnDefinition.ofString("PC3").withPartitioning(),
-                ColumnDefinition.ofLong("I"));
+                ColumnDefinition.ofChar("PC2"),
+                ColumnDefinition.ofString("PC3"),
+                ColumnDefinition.ofLong("II"),
+                ColumnDefinition.ofInt("I"));
         final Table inputData = ((QueryTable) TableTools.emptyTable(10)
                 .updateView("PC1 = (int)(ii%3)",
                         "PC2 = (char)(65 + (ii % 2))",
                         "PC3 = java.time.LocalDate.ofEpochDay(i%2).toString()",
-                        "I = ii"))
+                        "II = ii",
+                        "I = i"))
                 .withDefinitionUnsafe(definition);
 
+        // We skip one column in the definition, and add some more partitioning and non-partitioning columns
+        final TableDefinition tableDefinitionToWrite = TableDefinition.of(
+                ColumnDefinition.ofInt("PC1").withPartitioning(),
+                ColumnDefinition.ofChar("PC2").withPartitioning(),
+                ColumnDefinition.ofString("PC3").withPartitioning(),
+                ColumnDefinition.ofInt("I"),
+                ColumnDefinition.ofInt("J"));
+
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, tableDefinitionToWrite, parentDir, "data", writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
@@ -626,18 +639,21 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
                 assertTrue(dataFile.exists() && dataFile.isFile());
             }
         }
-        Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
-        assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
+
+        // Give then updated table definition used to write the data, we drop the column "II" and add a new column "J"
+        final Table expected = inputData.dropColumns("II").updateView("J = (int)null");
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        assertTableEquals(expected.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
         final File commonMetadata = new File(parentDir, "_common_metadata");
         final Table fromDiskWithMetadata = readTable(commonMetadata);
-        assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
+        assertTableEquals(expected.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
 
         // Delete some files from the partitioned data and read the required rows to verify that we only read the
         // required partitions
         FileUtils.deleteRecursivelyOnNFS(new File(parentDir, "PC1=0"));
         FileUtils.deleteRecursivelyOnNFS(new File(parentDir, "PC1=1"));
-        assertTableEquals(inputData.where("PC1 == 2").sort("PC1", "PC2", "PC3"),
+        assertTableEquals(expected.where("PC1 == 2").sort("PC1", "PC2", "PC3"),
                 readTable(commonMetadata).where("PC1 == 2").sort("PC1", "PC2", "PC3"));
     }
 
@@ -677,6 +693,50 @@ public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
                 fromDiskWithMetadata.sort("symbol", "epic_collection_id"));
     }
 
+    @Test
+    public void testAllPartitioningColumnTypes() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofString("PC1").withPartitioning(),
+                ColumnDefinition.ofBoolean("PC2").withPartitioning(),
+                ColumnDefinition.ofChar("PC3").withPartitioning(),
+                ColumnDefinition.ofByte("PC4").withPartitioning(),
+                ColumnDefinition.ofShort("PC5").withPartitioning(),
+                ColumnDefinition.ofInt("PC6").withPartitioning(),
+                ColumnDefinition.ofLong("PC7").withPartitioning(),
+                ColumnDefinition.ofFloat("PC8").withPartitioning(),
+                ColumnDefinition.ofDouble("PC9").withPartitioning(),
+                ColumnDefinition.of("PC10", Type.find(BigInteger.class)).withPartitioning(),
+                ColumnDefinition.of("PC11", Type.find(BigDecimal.class)).withPartitioning(),
+                ColumnDefinition.of("PC12", Type.find(Instant.class)).withPartitioning(),
+                ColumnDefinition.of("PC13", Type.find(LocalDate.class)).withPartitioning(),
+                ColumnDefinition.of("PC14", Type.find(LocalTime.class)).withPartitioning(),
+                ColumnDefinition.ofInt("data"));
+
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView("PC1 =  (ii%2 == 0) ? `AA` : `BB`",
+                "PC2 = (ii % 2 == 0)",
+                "PC3 = (char)(65 + (ii % 2))",
+                "PC4 = (byte)(ii % 2)",
+                "PC5 = (short)(ii % 2)",
+                "PC6 = (int)(ii%3)",
+                "PC7 = (long)(ii%2)",
+                "PC8 = (float)(ii % 2)",
+                "PC9 = (double)(ii % 2)",
+                "PC10 = java.math.BigInteger.valueOf(ii)",
+                "PC11 = java.math.BigDecimal.valueOf(ii)",
+                "PC12 = java.time.Instant.ofEpochSecond(ii)",
+                "PC13 = java.time.LocalDate.ofEpochDay(ii)",
+                "PC14 = java.time.LocalTime.of(i%24, i%60, (i+10)%60)",
+                "data = (int)(ii)"))
+                .withDefinitionUnsafe(definition);
+
+        final File parentDir = new File(rootFile, "testAllPartitioningColumnTypes");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        readKeyValuePartitionedTable(parentDir, EMPTY).select();
+    }
+
     @Test
     public void testVectorColumns() {
         final Table table = getTableFlat(20000, true, false);

From 3651e5f915e7a355a28ca34e182a766b4080edfc Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Sun, 3 Mar 2024 20:05:58 -0600
Subject: [PATCH 20/45] Added more tests

---
 .../deephaven/parquet/table/ParquetTools.java | 15 +++--
 .../parquet/table/PartitionFormatter.java     |  5 +-
 .../table/ParquetTableReadWriteTest.java      | 61 +++++++++++++++++++
 3 files changed, 72 insertions(+), 9 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index b05e53d5a6b..26e2026e35a 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -584,9 +584,10 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
         // partitionedTable.constituentFor(keyValues) method, and we need to group them together
         final Table withGroupConstituents = partitionedTable.table().groupBy(partitioningColumnNames);
         final ColumnSource<ObjectVector<Table>> consituentVectorColumnSource =
-                withGroupConstituents.getColumnSource(CONSTITUENT.name());
+                withGroupConstituents.getColumnSource(partitionedTable.constituentColumnName());
         if (consituentVectorColumnSource == null) {
-            throw new IllegalStateException("Partitioned table must have a constituent column");
+            throw new IllegalStateException("Grouped partitioned table must have a constituent column of type " +
+                    "ObjectVector");
         }
         final Collection<Table> partitionedData = new ArrayList<>();
         final Collection<File> destinations = new ArrayList<>();
@@ -595,7 +596,12 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             for (final String partitioningColumnName : partitioningColumnNames) {
                 final ColumnSource<?> partitioningColSource =
                         withGroupConstituents.getColumnSource(partitioningColumnName);
-                final String partitioningValue = PartitionFormatter.formatToString(partitioningColSource.get(key));
+                final Object partitioningKey = partitioningColSource.get(key);
+                if (partitioningKey == null) {
+                    throw new IllegalStateException("Partitioning column values must be non-null, found null " +
+                            "value for column " + partitioningColumnName);
+                }
+                final String partitioningValue = PartitionFormatter.formatToString(partitioningKey);
                 relativePathBuilder.append(partitioningColumnName).append("=").append(partitioningValue)
                         .append(File.separator);
             }
@@ -606,10 +612,9 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
                         " key = " + key);
             }
             int count = 0;
-            final long numConstituents = constituentVector.size();
             for (final Table constituent : constituentVector) {
                 final File destination;
-                if (numConstituents == 1) {
+                if (partitionedTable.uniqueKeys()) {
                     destination = new File(destinationDir, relativePath + baseName + ".parquet");
                 } else {
                     destination = new File(destinationDir, relativePath + baseName + "-part-" + count + ".parquet");
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
index ef50f74f0ba..2de2d426a0d 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
@@ -120,10 +120,7 @@ public String format(@NotNull final Object obj) {
 
     abstract String format(@NotNull Object obj);
 
-    static String formatToString(final Object obj) {
-        if (obj == null) {
-            return "null";
-        }
+    static String formatToString(@NotNull final Object obj) {
         final PartitionFormatter formatter = typeMap.get(obj.getClass());
         if (formatter != null) {
             return formatter.format(obj);
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 98f59d81b9b..77112ba7dd2 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -16,6 +16,7 @@
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.PartitionedTable;
+import io.deephaven.engine.table.PartitionedTableFactory;
 import io.deephaven.engine.table.impl.SourceTable;
 import io.deephaven.engine.table.impl.select.FunctionalColumn;
 import io.deephaven.engine.table.impl.select.SelectColumn;
@@ -597,6 +598,66 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
     }
 
+    @Test
+    public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofInt("PC1").withPartitioning(),
+                ColumnDefinition.ofLong("I"));
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10)
+                .updateView("PC1 = (int)(ii%3)",
+                        "I = ii"))
+                .withDefinitionUnsafe(definition);
+        final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
+        final Table internalTable = partitionedTable.table();
+        final Table internalTableDuplicated = merge(internalTable, internalTable);
+        final PartitionedTable partitionedTableWithDuplicatedKeys = PartitionedTableFactory.of(internalTableDuplicated);
+        assertFalse(partitionedTableWithDuplicatedKeys.uniqueKeys());
+        final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataWithNonUniqueKeys");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data", writeInstructions);
+
+        // Verify that the partitioned data exists
+        for (int PC1 = 0; PC1 <= 2; PC1++) {
+            final File dir = new File(parentDir, "PC1=" + PC1 + File.separator);
+            assertTrue(dir.exists() && dir.isDirectory());
+            final File dataFile1 = new File(dir, "data-part-0.parquet");
+            assertTrue(dataFile1.exists() && dataFile1.isFile());
+            final File dataFile2 = new File(dir, "data-part-1.parquet");
+            assertTrue(dataFile2.exists() && dataFile2.isFile());
+        }
+
+        final Table expected = merge(inputData, inputData);
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+
+        final File commonMetadata = new File(parentDir, "_common_metadata");
+        final Table fromDiskWithMetadata = readTable(commonMetadata);
+        assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+    }
+
+    @Test
+    public void writeKeyValuePartitionedDataWithNullKeys() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofInt("PC1").withPartitioning(),
+                ColumnDefinition.ofLong("I"));
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10)
+                .updateView("PC1 = (ii%2==1)? null : (int)(ii%3)",
+                        "I = ii"))
+                .withDefinitionUnsafe(definition);
+        final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
+        final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataWithNullKeys");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        try {
+            writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
+            fail("Expected exception when writing the partitioned table with null keys");
+        } catch (final RuntimeException expected) {
+        }
+    }
+
     @Test
     public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         final TableDefinition definition = TableDefinition.of(

From 51eefe10a888aa1c26e429e83909627b781dfa77 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 4 Mar 2024 11:50:58 -0600
Subject: [PATCH 21/45] Iterating using chunked iterators

---
 .../deephaven/parquet/table/ParquetTools.java | 94 +++++++++----------
 .../parquet/table/PartitionFormatter.java     |  3 +-
 .../table/ParquetTableReadWriteTest.java      | 35 ++++++-
 3 files changed, 81 insertions(+), 51 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 26e2026e35a..c3e64ac097a 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -9,8 +9,8 @@
 import io.deephaven.base.Pair;
 import io.deephaven.base.verify.Require;
 import io.deephaven.engine.context.ExecutionContext;
+import io.deephaven.engine.primitive.iterator.CloseableIterator;
 import io.deephaven.engine.table.ColumnDefinition;
-import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.PartitionedTable;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
@@ -60,7 +60,6 @@
 import java.util.*;
 import java.util.stream.Collectors;
 
-import static io.deephaven.engine.table.impl.partitioned.PartitionedTableCreatorImpl.CONSTITUENT;
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 import static io.deephaven.parquet.table.ParquetTableWriter.getSchemaForTable;
 import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
@@ -546,8 +545,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
         final Set<String> keyColumnNames = partitionedTable.keyColumnNames();
         final Collection<ColumnDefinition<?>> columnDefinitions = new ArrayList<>(keyColumnNames.size() +
                 partitionedTable.constituentDefinition().numColumns());
-        partitionedTable.table().getDefinition().getColumns().stream()
-                .filter(columnDefinition -> keyColumnNames.contains(columnDefinition.getName()))
+        keyColumnNames.stream().map(keyColName -> partitionedTable.table().getDefinition().getColumn(keyColName))
                 .forEach(columnDefinitions::add);
         partitionedTable.constituentDefinition().getColumns().stream()
                 .filter(columnDefinition -> !keyColumnNames.contains(columnDefinition.getName()))
@@ -576,55 +574,57 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
-        if (partitionedTable.table().numColumns() == partitioningColumnNames.length) {
-            throw new IllegalArgumentException(
-                    "Cannot write a partitioned parquet table with no non-partitioning columns");
+        final TableDefinition partitionedTableDefinition = getNonKeyTableDefiniton(partitionedTable.keyColumnNames(),
+                definition);
+        if (partitionedTableDefinition.numColumns() == 0) {
+            throw new IllegalArgumentException("Cannot write a partitioned parquet table without any non-partitioning "
+                    + "columns");
         }
         // Note that there can be multiple constituents with the same key values, so cannot directly use the
         // partitionedTable.constituentFor(keyValues) method, and we need to group them together
         final Table withGroupConstituents = partitionedTable.table().groupBy(partitioningColumnNames);
-        final ColumnSource<ObjectVector<Table>> consituentVectorColumnSource =
-                withGroupConstituents.getColumnSource(partitionedTable.constituentColumnName());
-        if (consituentVectorColumnSource == null) {
-            throw new IllegalStateException("Grouped partitioned table must have a constituent column of type " +
-                    "ObjectVector");
+        final List<StringBuilder> relativePathBuilders = new ArrayList<>();
+        final long numRows = withGroupConstituents.size();
+        for (long i = 0; i < numRows; i++) {
+            relativePathBuilders.add(new StringBuilder());
         }
-        final Collection<Table> partitionedData = new ArrayList<>();
-        final Collection<File> destinations = new ArrayList<>();
-        withGroupConstituents.getRowSet().forAllRowKeys(key -> {
-            final StringBuilder relativePathBuilder = new StringBuilder();
-            for (final String partitioningColumnName : partitioningColumnNames) {
-                final ColumnSource<?> partitioningColSource =
-                        withGroupConstituents.getColumnSource(partitioningColumnName);
-                final Object partitioningKey = partitioningColSource.get(key);
-                if (partitioningKey == null) {
-                    throw new IllegalStateException("Partitioning column values must be non-null, found null " +
-                            "value for column " + partitioningColumnName);
-                }
-                final String partitioningValue = PartitionFormatter.formatToString(partitioningKey);
-                relativePathBuilder.append(partitioningColumnName).append("=").append(partitioningValue)
-                        .append(File.separator);
-            }
-            final String relativePath = relativePathBuilder.toString();
-            final ObjectVector<? extends Table> constituentVector = consituentVectorColumnSource.get(key);
-            if (constituentVector == null) {
-                throw new IllegalStateException("Grouped partitioned table must have a vector constituent column for" +
-                        " key = " + key);
-            }
-            int count = 0;
-            for (final Table constituent : constituentVector) {
-                final File destination;
-                if (partitionedTable.uniqueKeys()) {
-                    destination = new File(destinationDir, relativePath + baseName + ".parquet");
-                } else {
-                    destination = new File(destinationDir, relativePath + baseName + "-part-" + count + ".parquet");
+        // For partitioning column for each row, accumulate the values in a key=value format
+        Arrays.stream(partitioningColumnNames).forEach(columnName -> {
+            try (final CloseableIterator<?> valueIterator = withGroupConstituents.columnIterator(columnName)) {
+                int row = 0;
+                while (valueIterator.hasNext()) {
+                    final String partitioningValue = PartitionFormatter.formatToString(valueIterator.next());
+                    relativePathBuilders.get(row).append(columnName).append("=").append(partitioningValue)
+                            .append(File.separator);
+                    row++;
                 }
-                destinations.add(destination);
-                partitionedData.add(constituent);
-                count++;
             }
         });
 
+        // For constituent column for each row, build final file paths
+        final Collection<Table> partitionedData = new ArrayList<>();
+        final Collection<File> destinations = new ArrayList<>();
+        try (final CloseableIterator<ObjectVector<? extends Table>> constituentIterator =
+                withGroupConstituents.objectColumnIterator(partitionedTable.constituentColumnName())) {
+            int row = 0;
+            while (constituentIterator.hasNext()) {
+                final ObjectVector<? extends Table> constituentVector = constituentIterator.next();
+                final String relativePath = relativePathBuilders.get(row).toString();
+                int count = 0;
+                for (final Table constituent : constituentVector) {
+                    final File destination;
+                    if (partitionedTable.uniqueKeys()) {
+                        destination = new File(destinationDir, relativePath + baseName + ".parquet");
+                    } else {
+                        destination = new File(destinationDir, relativePath + baseName + "-part-" + count + ".parquet");
+                    }
+                    destinations.add(destination);
+                    partitionedData.add(constituent);
+                    count++;
+                }
+                row++;
+            }
+        }
         // If needed, generate schema for _common_metadata file from key table
         final ParquetInstructions updatedWriteInstructions;
         if (!ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(writeInstructions.getMetadataRootDir())) {
@@ -633,14 +633,12 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
         } else {
             updatedWriteInstructions = writeInstructions;
         }
-        final TableDefinition partitionedTablesDefinition = getNonKeyTableDefiniton(partitionedTable.keyColumnNames(),
-                definition);
         ParquetTools.writeParquetTables(
                 partitionedData.toArray(Table[]::new),
-                partitionedTablesDefinition,
+                partitionedTableDefinition,
                 updatedWriteInstructions,
                 destinations.toArray(File[]::new),
-                partitionedTablesDefinition.getGroupingColumnNamesArray());
+                partitionedTableDefinition.getGroupingColumnNamesArray());
     }
 
     /**
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
index 2de2d426a0d..fef23a696a1 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
@@ -12,7 +12,8 @@
 import java.util.Map;
 
 /**
- * This class takes an object read from a parquet file and formats it to a string, only for the supported types.
+ * This class takes an object read from a parquet file and formats it to a string used for generate partitioning column
+ * values.
  */
 enum PartitionFormatter {
     ForString {
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 77112ba7dd2..b4a91f0a07e 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -598,6 +598,27 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
     }
 
+    @Test
+    public void writeKeyValuePartitionedDataWithNoNonPartitioningColumnsTest() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofInt("PC1").withPartitioning(),
+                ColumnDefinition.ofInt("PC2").withPartitioning());
+        final Table inputData = ((QueryTable) TableTools.emptyTable(20)
+                .updateView("PC1 = (int)(ii%3)",
+                        "PC2 = (int)(ii%2)"))
+                .withDefinitionUnsafe(definition);
+
+        final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+        try {
+            writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+            fail("Expected exception when writing the partitioned table with no non-partitioning columns");
+        } catch (final RuntimeException expected) {
+        }
+    }
+
     @Test
     public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         final TableDefinition definition = TableDefinition.of(
@@ -783,7 +804,7 @@ public void testAllPartitioningColumnTypes() {
                 "PC8 = (float)(ii % 2)",
                 "PC9 = (double)(ii % 2)",
                 "PC10 = java.math.BigInteger.valueOf(ii)",
-                "PC11 = java.math.BigDecimal.valueOf(ii)",
+                "PC11 = java.math.BigDecimal.valueOf((double)ii)",
                 "PC12 = java.time.Instant.ofEpochSecond(ii)",
                 "PC13 = java.time.LocalDate.ofEpochDay(ii)",
                 "PC14 = java.time.LocalTime.of(i%24, i%60, (i+10)%60)",
@@ -795,7 +816,17 @@ public void testAllPartitioningColumnTypes() {
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
         writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
-        readKeyValuePartitionedTable(parentDir, EMPTY).select();
+
+        // Verify that we can read the partition values, but types like LocalDate or LocalTime will be read as strings
+        // Therefore, we cannot compare the tables directly
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+
+        // Reading with metadata file should deduce the correct type, so we can compare the tables
+        final File commonMetadata = new File(parentDir, "_common_metadata");
+        final Table fromDiskWithMetadata = readTable(commonMetadata);
+        final String[] partitioningColumns = definition.getPartitioningColumns().stream()
+                .map(ColumnDefinition::getName).toArray(String[]::new);
+        assertTableEquals(inputData.sort(partitioningColumns), fromDiskWithMetadata.sort(partitioningColumns));
     }
 
     @Test

From 60ee1f57a6466e24f988cdfdb043625242ec4a28 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 4 Mar 2024 11:57:45 -0600
Subject: [PATCH 22/45] Removed some unnecessary includes

---
 .../io/deephaven/parquet/base/RowGroupReaderImpl.java | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index 9798a42f2a7..8212fc2458b 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -4,23 +4,14 @@
 package io.deephaven.parquet.base;
 
 import io.deephaven.util.channel.SeekableChannelsProvider;
-import io.deephaven.util.channel.SeekableChannelContext.ContextHolder;
-import io.deephaven.util.channel.SeekableChannelContext;
 import org.apache.parquet.format.ColumnChunk;
 import org.apache.parquet.format.RowGroup;
-import org.apache.parquet.format.Util;
-import org.apache.parquet.format.converter.ParquetMetadataConverter;
-import org.apache.parquet.internal.column.columnindex.OffsetIndex;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.Type;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.net.URI;
-import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -90,8 +81,6 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path) {
                 version);
     }
 
-
-
     @Override
     public long numRows() {
         return rowGroup.num_rows;

From 5c7353f6ca10f108673e6e0ae173bd629f1cad43 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 4 Mar 2024 13:50:43 -0600
Subject: [PATCH 23/45] Added support for {index} and {partition} in file
 basename

---
 .../deephaven/parquet/table/ParquetTools.java | 101 ++++++++++++------
 .../table/ParquetTableReadWriteTest.java      |  13 ++-
 2 files changed, 80 insertions(+), 34 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index c3e64ac097a..b69e5113622 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -468,14 +468,24 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
     /**
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns.
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
+     * table.
      *
      * @param sourceTable The table to partition and write
      * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PartitioningColumn=partition1/table.parquet",
-     *        "PartitioningColumn=partition2/table.parquet", etc.
+     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
+     *        partitioning column.
+     *        <p>
+     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
+     *        a directory. For example, a base name of "table-{i}" will result in files named
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
+     *        <p>
+     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
+     *        example, a base name of "{partitions}-table" will result in files
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
@@ -497,8 +507,17 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PartitioningColumn=partition1/table.parquet",
-     *        "PartitioningColumn=partition2/table.parquet", etc.
+     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
+     *        partitioning column.
+     *        <p>
+     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
+     *        a directory. For example, a base name of "table-{i}" will result in files named
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
+     *        <p>
+     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
+     *        example, a base name of "{partitions}-table" will result in files
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
@@ -506,12 +525,6 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
             @NotNull final File destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
-        // TODO Also, how should I get the baseName, should I move it to the write instructions.
-        // Pyarrow has this optional parameter "basename_template" used to generate basenames of written data files.
-        // The token ‘{i}’ will be replaced with an automatically incremented integer for files in the same folder.
-        // If not specified, it defaults to “someHash-{i}.parquet”.
-        // pyspark has names of the form "part.{i}.parquet" and allows passing a naming function which takes an integer
-        // and generates a file name.
         final List<ColumnDefinition<?>> partitioningColumns = definition.getPartitioningColumns();
         if (partitioningColumns.isEmpty()) {
             throw new IllegalArgumentException("Table must have partitioning columns to write partitioned data");
@@ -532,8 +545,17 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PartitioningColumn=partition1/table.parquet",
-     *        "PartitioningColumn=partition2/table.parquet", etc.
+     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
+     *        partitioning column.
+     *        <p>
+     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
+     *        a directory. For example, a base name of "table-{i}" will result in files named
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
+     *        <p>
+     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
+     *        example, a base name of "{partitions}-table" will result in files
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
@@ -541,7 +563,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         // Get key column definitions from the partitioned table definition and non-key column definitions from the
-        // constituent table definition, and combine them to build the overall table definition
+        // constituent table definition, and combine them to build the overall table definition to be written
         final Set<String> keyColumnNames = partitionedTable.keyColumnNames();
         final Collection<ColumnDefinition<?>> columnDefinitions = new ArrayList<>(keyColumnNames.size() +
                 partitionedTable.constituentDefinition().numColumns());
@@ -564,8 +586,17 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PartitioningColumn=partition1/table.parquet",
-     *        "PartitioningColumn=partition2/table.parquet", etc.
+     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
+     *        partitioning column.
+     *        <p>
+     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
+     *        a directory. For example, a base name of "table-{i}" will result in files named
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
+     *        <p>
+     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
+     *        example, a base name of "{partitions}-table" will result in files
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
@@ -573,35 +604,40 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             @NotNull final File destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
-        final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
         final TableDefinition partitionedTableDefinition = getNonKeyTableDefiniton(partitionedTable.keyColumnNames(),
                 definition);
         if (partitionedTableDefinition.numColumns() == 0) {
             throw new IllegalArgumentException("Cannot write a partitioned parquet table without any non-partitioning "
                     + "columns");
         }
+        final boolean hasPartitionInName = baseName.contains("{partitions}");
+        final boolean hasIndexInName = baseName.contains("{i}");
+        if (!partitionedTable.uniqueKeys() && !hasIndexInName) {
+            throw new IllegalArgumentException(
+                    "Cannot write a partitioned parquet table with non-unique keys without " +
+                            "{i} in the base name because there can be multiple partitions with the same key values");
+        }
         // Note that there can be multiple constituents with the same key values, so cannot directly use the
         // partitionedTable.constituentFor(keyValues) method, and we need to group them together
+        final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
         final Table withGroupConstituents = partitionedTable.table().groupBy(partitioningColumnNames);
-        final List<StringBuilder> relativePathBuilders = new ArrayList<>();
+        // For each row, accumulate the partition values in a key=value format
+        final List<List<String>> partitionStringsList = new ArrayList<>();
         final long numRows = withGroupConstituents.size();
         for (long i = 0; i < numRows; i++) {
-            relativePathBuilders.add(new StringBuilder());
+            partitionStringsList.add(new ArrayList<>(partitioningColumnNames.length));
         }
-        // For partitioning column for each row, accumulate the values in a key=value format
         Arrays.stream(partitioningColumnNames).forEach(columnName -> {
             try (final CloseableIterator<?> valueIterator = withGroupConstituents.columnIterator(columnName)) {
                 int row = 0;
                 while (valueIterator.hasNext()) {
                     final String partitioningValue = PartitionFormatter.formatToString(valueIterator.next());
-                    relativePathBuilders.get(row).append(columnName).append("=").append(partitioningValue)
-                            .append(File.separator);
+                    partitionStringsList.get(row).add(columnName + "=" + partitioningValue);
                     row++;
                 }
             }
         });
-
-        // For constituent column for each row, build final file paths
+        // For the constituent column for each row, accumulate the constituent tables and build the final file paths
         final Collection<Table> partitionedData = new ArrayList<>();
         final Collection<File> destinations = new ArrayList<>();
         try (final CloseableIterator<ObjectVector<? extends Table>> constituentIterator =
@@ -609,16 +645,19 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             int row = 0;
             while (constituentIterator.hasNext()) {
                 final ObjectVector<? extends Table> constituentVector = constituentIterator.next();
-                final String relativePath = relativePathBuilders.get(row).toString();
+                final List<String> partitionStrings = partitionStringsList.get(row);
+                final File relativePath = new File(destinationDir, String.join("/", partitionStrings));
                 int count = 0;
                 for (final Table constituent : constituentVector) {
-                    final File destination;
-                    if (partitionedTable.uniqueKeys()) {
-                        destination = new File(destinationDir, relativePath + baseName + ".parquet");
-                    } else {
-                        destination = new File(destinationDir, relativePath + baseName + "-part-" + count + ".parquet");
+                    String filename = baseName;
+                    if (hasPartitionInName) {
+                        filename = baseName.replace("{partitions}", String.join("_", partitionStrings));
+                    }
+                    if (hasIndexInName) {
+                        filename = filename.replace("{i}", Integer.toString(count));
                     }
-                    destinations.add(destination);
+                    filename += PARQUET_FILE_EXTENSION;
+                    destinations.add(new File(relativePath, filename));
                     partitionedData.add(constituent);
                     count++;
                 }
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index b4a91f0a07e..5ac491689d2 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -578,14 +578,14 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir, "{partitions}-data-{i}", writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
             for (int PC2 = 0; PC2 <= 1; PC2++) {
                 final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
                 assertTrue(dir.exists() && dir.isDirectory());
-                final File dataFile = new File(dir, "data.parquet");
+                final File dataFile = new File(dir, "PC1=" + PC1 + "_PC2=" + PC2 + "-data-0.parquet");
                 assertTrue(dataFile.exists() && dataFile.isFile());
             }
         }
@@ -637,7 +637,14 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setMetadataRootDir(parentDir.getAbsolutePath())
                 .build();
-        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data", writeInstructions);
+        try {
+            writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data", writeInstructions);
+            fail("Expected exception when writing the partitioned table with non-unique keys without {i} in base name");
+        } catch (final RuntimeException expected) {
+        }
+
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data-part-{i}",
+                writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {

From c17445297e18dc36283685aa5be50498a792d946 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 5 Mar 2024 13:51:38 -0600
Subject: [PATCH 24/45] Review comments

---
 .../io/deephaven/util/channel/Channels.java   |   2 -
 .../parquet/table/ParquetTableWriter.java     |   3 +
 .../deephaven/parquet/table/ParquetTools.java | 157 +++++++++++-------
 .../table/ParquetTableReadWriteTest.java      | 132 ++++++++++++++-
 4 files changed, 220 insertions(+), 74 deletions(-)

diff --git a/Util/channel/src/main/java/io/deephaven/util/channel/Channels.java b/Util/channel/src/main/java/io/deephaven/util/channel/Channels.java
index 22fb225f708..55bdaa95c02 100644
--- a/Util/channel/src/main/java/io/deephaven/util/channel/Channels.java
+++ b/Util/channel/src/main/java/io/deephaven/util/channel/Channels.java
@@ -5,8 +5,6 @@
 
 import java.io.InputStream;
 import java.nio.channels.ReadableByteChannel;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
 
 public final class Channels {
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index e8fa8b8107e..190e0fb0e67 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -240,6 +240,9 @@ private static void write(
     static MessageType getSchemaForTable(@NotNull final Table table,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions instructions) {
+        if (definition.numColumns() == 0) {
+            throw new IllegalArgumentException("Table definition must have at least one column");
+        }
         final Table pretransformTable = pretransformTable(table, definition);
         return MappedSchema.create(new HashMap<>(), definition, pretransformTable.getRowSet(),
                 pretransformTable.getColumnSourceMap(), instructions).getParquetSchema();
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index b69e5113622..7bef6f76874 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -476,16 +476,16 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
      *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column.
-     *        <p>
-     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
-     *        a directory. For example, a base name of "table-{i}" will result in files named
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
-     *        <p>
-     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
-     *        example, a base name of "{partitions}-table" will result in files
+     *        partitioning column. Users can also provide the following tokens to be replaced in the base name:
+     *        <ul>
+     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
+     *        partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.
+     *        partitioning columns.</li>
+     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
@@ -508,16 +508,16 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
      *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column.
-     *        <p>
-     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
-     *        a directory. For example, a base name of "table-{i}" will result in files named
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
-     *        <p>
-     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
-     *        example, a base name of "{partitions}-table" will result in files
+     *        partitioning column. Users can also provide the following tokens to be replaced in the base name:
+     *        <ul>
+     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
+     *        partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.
+     *        partitioning columns.</li>
+     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
@@ -546,34 +546,31 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
      *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column.
-     *        <p>
-     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
-     *        a directory. For example, a base name of "table-{i}" will result in files named
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
-     *        <p>
-     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
-     *        example, a base name of "{partitions}-table" will result in files
+     *        partitioning column. Users can also provide the following tokens to be replaced in the base name:
+     *        <ul>
+     *        <li>The token {@code {i}} will be replaced with an automatically incremented integer for files in a
+     *        directory. For example, a base name of "table-{i}" will result in files named like
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
+     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
+     *        partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.
+     *        partitioning columns.</li>
+     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final File destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
-        // Get key column definitions from the partitioned table definition and non-key column definitions from the
-        // constituent table definition, and combine them to build the overall table definition to be written
-        final Set<String> keyColumnNames = partitionedTable.keyColumnNames();
-        final Collection<ColumnDefinition<?>> columnDefinitions = new ArrayList<>(keyColumnNames.size() +
-                partitionedTable.constituentDefinition().numColumns());
-        keyColumnNames.stream().map(keyColName -> partitionedTable.table().getDefinition().getColumn(keyColName))
-                .forEach(columnDefinitions::add);
-        partitionedTable.constituentDefinition().getColumns().stream()
-                .filter(columnDefinition -> !keyColumnNames.contains(columnDefinition.getName()))
-                .forEach(columnDefinitions::add);
-        final TableDefinition definition = TableDefinition.of(columnDefinitions);
-        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, baseName, writeInstructions);
+        final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(),
+                partitionedTable.table().getDefinition());
+        final TableDefinition leafDefinition = getNonKeyTableDefinition(partitionedTable.keyColumnNames(),
+                partitionedTable.constituentDefinition());
+        writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
+                baseName, writeInstructions);
     }
 
     /**
@@ -587,16 +584,19 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      *        directories are created.
      * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
      *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column.
-     *        <p>
-     *        The token {@code {i}} in base name will be replaced with an automatically incremented integer for files in
-     *        a directory. For example, a base name of "table-{i}" will result in files named
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.
-     *        <p>
-     *        Also, token {@code {partitions}} will be replaced with a concatenated string of partition values. For
-     *        example, a base name of "{partitions}-table" will result in files
+     *        partitioning column.Users can also provide the following tokens to be replaced in the base name:
+     *        <ul>
+     *        <li>The token {@code {i}} will be replaced with an automatically incremented integer for files in a
+     *        directory. For example, a base name of "table-{i}" will result in files named like
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
+     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
+     *        partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.
+     *        partitioning columns.</li>
+     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
@@ -604,18 +604,29 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             @NotNull final File destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
-        final TableDefinition partitionedTableDefinition = getNonKeyTableDefiniton(partitionedTable.keyColumnNames(),
-                definition);
-        if (partitionedTableDefinition.numColumns() == 0) {
+        final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
+        final TableDefinition leafDefinition = getNonKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
+        writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
+                baseName, writeInstructions);
+    }
+
+    private static void writeKeyValuePartitionedTableImpl(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition keyTableDefinition,
+            @NotNull final TableDefinition leafDefinition,
+            @NotNull final File destinationDir,
+            @NotNull final String baseName,
+            @NotNull final ParquetInstructions writeInstructions) {
+        if (leafDefinition.numColumns() == 0) {
             throw new IllegalArgumentException("Cannot write a partitioned parquet table without any non-partitioning "
                     + "columns");
         }
         final boolean hasPartitionInName = baseName.contains("{partitions}");
         final boolean hasIndexInName = baseName.contains("{i}");
-        if (!partitionedTable.uniqueKeys() && !hasIndexInName) {
+        final boolean hasUUIDInName = baseName.contains("{uuid}");
+        if (!partitionedTable.uniqueKeys() && !hasIndexInName && !hasUUIDInName) {
             throw new IllegalArgumentException(
                     "Cannot write a partitioned parquet table with non-unique keys without " +
-                            "{i} in the base name because there can be multiple partitions with the same key values");
+                            "{i} or {uuid} in the base name because there can be multiple partitions with the same key values");
         }
         // Note that there can be multiple constituents with the same key values, so cannot directly use the
         // partitionedTable.constituentFor(keyValues) method, and we need to group them together
@@ -656,6 +667,9 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
                     if (hasIndexInName) {
                         filename = filename.replace("{i}", Integer.toString(count));
                     }
+                    if (hasUUIDInName) {
+                        filename = filename.replace("{uuid}", UUID.randomUUID().toString());
+                    }
                     filename += PARQUET_FILE_EXTENSION;
                     destinations.add(new File(relativePath, filename));
                     partitionedData.add(constituent);
@@ -664,33 +678,48 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
                 row++;
             }
         }
-        // If needed, generate schema for _common_metadata file from key table
         final ParquetInstructions updatedWriteInstructions;
         if (!ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(writeInstructions.getMetadataRootDir())) {
-            final MessageType commonSchema = getSchemaForTable(partitionedTable.table(), definition, writeInstructions);
+            // Generate initial schema for _common_metadata file from key columns. The schema for remaining columns will
+            // be inferred at the time of writing the parquet files and merged with the common schema.
+            final MessageType commonSchema =
+                    getSchemaForTable(partitionedTable.table(), keyTableDefinition, writeInstructions);
             updatedWriteInstructions = ParquetInstructions.createWithCommonSchema(writeInstructions, commonSchema);
         } else {
             updatedWriteInstructions = writeInstructions;
         }
         ParquetTools.writeParquetTables(
                 partitionedData.toArray(Table[]::new),
-                partitionedTableDefinition,
+                leafDefinition,
                 updatedWriteInstructions,
                 destinations.toArray(File[]::new),
-                partitionedTableDefinition.getGroupingColumnNamesArray());
+                leafDefinition.getGroupingColumnNamesArray());
+    }
+
+    /**
+     * Using the provided definition and key column names, create a sub table definition for the key columns that are
+     * present in the definition.
+     */
+    private static TableDefinition getKeyTableDefinition(@NotNull final Collection<String> keyColumnNames,
+            @NotNull final TableDefinition definition) {
+        final Collection<ColumnDefinition<?>> keyColumnDefinitions = new ArrayList<>(keyColumnNames.size());
+        for (final String keyColumnName : keyColumnNames) {
+            final ColumnDefinition<?> keyColumnDef = definition.getColumn(keyColumnName);
+            if (keyColumnDef != null) {
+                keyColumnDefinitions.add(keyColumnDef);
+            }
+        }
+        return TableDefinition.of(keyColumnDefinitions);
     }
 
     /**
      * Using the provided definition and key column names, create a sub table definition for the non-key columns.
      */
-    private static TableDefinition getNonKeyTableDefiniton(@NotNull final Collection<String> keyColumnNames,
+    private static TableDefinition getNonKeyTableDefinition(@NotNull final Collection<String> keyColumnNames,
             @NotNull final TableDefinition definition) {
-        final List<ColumnDefinition<?>> nonKeyColumnDefinition =
-                definition.getColumns().stream()
-                        .filter(columnDefinition -> {
-                            final String columnName = columnDefinition.getName();
-                            return !keyColumnNames.contains(columnName);
-                        }).collect(Collectors.toList());
+        final Collection<ColumnDefinition<?>> nonKeyColumnDefinition = definition.getColumns().stream()
+                .filter(columnDefinition -> !keyColumnNames.contains(columnDefinition.getName()))
+                .collect(Collectors.toList());
         return TableDefinition.of(nonKeyColumnDefinition);
     }
 
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 5ac491689d2..0dfc47e601b 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -87,6 +87,7 @@
 import java.util.function.Function;
 import java.util.function.IntConsumer;
 import java.util.function.LongConsumer;
+import java.util.stream.Collectors;
 
 import org.junit.experimental.categories.Category;
 
@@ -639,10 +640,12 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
                 .build();
         try {
             writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data", writeInstructions);
-            fail("Expected exception when writing the partitioned table with non-unique keys without {i} in base name");
+            fail("Expected exception when writing the partitioned table with non-unique keys without {i} or {uuid} in "
+                    + "base name");
         } catch (final RuntimeException expected) {
         }
 
+        // Write the partitioned table with non-unique keys with {i} in the base name
         writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data-part-{i}",
                 writeInstructions);
 
@@ -650,10 +653,11 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         for (int PC1 = 0; PC1 <= 2; PC1++) {
             final File dir = new File(parentDir, "PC1=" + PC1 + File.separator);
             assertTrue(dir.exists() && dir.isDirectory());
-            final File dataFile1 = new File(dir, "data-part-0.parquet");
-            assertTrue(dataFile1.exists() && dataFile1.isFile());
-            final File dataFile2 = new File(dir, "data-part-1.parquet");
-            assertTrue(dataFile2.exists() && dataFile2.isFile());
+            final String[] dataFileList = dir.list();
+            assertEquals(2, dataFileList.length);
+            for (final String dataFile : dataFileList) {
+                assertTrue(dataFile.equals("data-part-0.parquet") || dataFile.equals("data-part-1.parquet"));
+            }
         }
 
         final Table expected = merge(inputData, inputData);
@@ -663,6 +667,23 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         final File commonMetadata = new File(parentDir, "_common_metadata");
         final Table fromDiskWithMetadata = readTable(commonMetadata);
         assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+
+        FileUtils.deleteRecursively(parentDir);
+
+        // Write the partitioned table with non-unique keys with {uuid} in the base name
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data-{uuid}",
+                writeInstructions);
+
+        // Verify that the partitioned data exists with uuid in names
+        for (int PC1 = 0; PC1 <= 2; PC1++) {
+            final File dir = new File(parentDir, "PC1=" + PC1 + File.separator);
+            assertTrue(dir.exists() && dir.isDirectory());
+            final String[] dataFileList = dir.list();
+            assertEquals(2, dataFileList.length);
+            for (final String dataFile : dataFileList) {
+                assertTrue(dataFile.startsWith("data-") && dataFile.endsWith(".parquet"));
+            }
+        }
     }
 
     @Test
@@ -692,17 +713,20 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
                 ColumnDefinition.ofInt("PC1").withPartitioning(),
                 ColumnDefinition.ofChar("PC2"),
                 ColumnDefinition.ofString("PC3"),
+                ColumnDefinition.ofInt("PC4").withPartitioning(),
                 ColumnDefinition.ofLong("II"),
                 ColumnDefinition.ofInt("I"));
         final Table inputData = ((QueryTable) TableTools.emptyTable(10)
                 .updateView("PC1 = (int)(ii%3)",
                         "PC2 = (char)(65 + (ii % 2))",
                         "PC3 = java.time.LocalDate.ofEpochDay(i%2).toString()",
+                        "PC4 = (int)(ii%4)",
                         "II = ii",
                         "I = i"))
                 .withDefinitionUnsafe(definition);
 
-        // We skip one column in the definition, and add some more partitioning and non-partitioning columns
+        // We skip one partitioning and one non partitioing column in the definition, and add some more partitioning
+        // and non-partitioning columns
         final TableDefinition tableDefinitionToWrite = TableDefinition.of(
                 ColumnDefinition.ofInt("PC1").withPartitioning(),
                 ColumnDefinition.ofChar("PC2").withPartitioning(),
@@ -730,7 +754,7 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         }
 
         // Give then updated table definition used to write the data, we drop the column "II" and add a new column "J"
-        final Table expected = inputData.dropColumns("II").updateView("J = (int)null");
+        final Table expected = inputData.dropColumns("II", "PC4").updateView("J = (int)null");
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         assertTableEquals(expected.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
@@ -801,7 +825,8 @@ public void testAllPartitioningColumnTypes() {
                 ColumnDefinition.of("PC14", Type.find(LocalTime.class)).withPartitioning(),
                 ColumnDefinition.ofInt("data"));
 
-        final Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView("PC1 =  (ii%2 == 0) ? `AA` : `BB`",
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView(
+                "PC1 =  (ii%2 == 0) ? `AA` : `BB`",
                 "PC2 = (ii % 2 == 0)",
                 "PC3 = (char)(65 + (ii % 2))",
                 "PC4 = (byte)(ii % 2)",
@@ -836,6 +861,97 @@ public void testAllPartitioningColumnTypes() {
         assertTableEquals(inputData.sort(partitioningColumns), fromDiskWithMetadata.sort(partitioningColumns));
     }
 
+    @Test
+    public void testAllNonPartitioningColumnTypes() {
+        final TableDefinition definition = TableDefinition.of(
+                ColumnDefinition.ofString("PC1").withPartitioning(),
+                ColumnDefinition.ofInt("PC2").withPartitioning(),
+                ColumnDefinition.ofString("NPC1"),
+                ColumnDefinition.ofBoolean("NPC2"),
+                ColumnDefinition.ofChar("NPC3"),
+                ColumnDefinition.ofByte("NPC4"),
+                ColumnDefinition.ofShort("NPC5"),
+                ColumnDefinition.ofInt("NPC6"),
+                ColumnDefinition.ofLong("NPC7"),
+                ColumnDefinition.ofFloat("NPC8"),
+                ColumnDefinition.ofDouble("NPC9"),
+                ColumnDefinition.of("NPC10", Type.find(BigInteger.class)),
+                ColumnDefinition.of("NPC11", Type.find(BigDecimal.class)),
+                ColumnDefinition.of("NPC12", Type.find(Instant.class)),
+                ColumnDefinition.of("NPC13", Type.find(LocalDate.class)),
+                ColumnDefinition.of("NPC14", Type.find(LocalTime.class)));
+
+        final Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView(
+                "PC1 =  (ii%2 == 0) ? `AA` : `BB`",
+                "PC2 = (int)(ii%3)",
+                "NPC1 =  (ii%2 == 0) ? `AA` : `BB`",
+                "NPC2 = (ii % 2 == 0)",
+                "NPC3 = (char)(65 + (ii % 2))",
+                "NPC4 = (byte)(ii % 2)",
+                "NPC5 = (short)(ii % 2)",
+                "NPC6 = (int)(ii%3)",
+                "NPC7 = (long)(ii%2)",
+                "NPC8 = (float)(ii % 2)",
+                "NPC9 = (double)(ii % 2)",
+                "NPC10 = java.math.BigInteger.valueOf(ii)",
+                "NPC11 = java.math.BigDecimal.valueOf((double)ii)",
+                "NPC12 = java.time.Instant.ofEpochSecond(ii)",
+                "NPC13 = java.time.LocalDate.ofEpochDay(ii)",
+                "NPC14 = java.time.LocalTime.of(i%24, i%60, (i+10)%60)"))
+                .withDefinitionUnsafe(definition);
+
+        final File parentDir = new File(rootFile, "testAllNonPartitioningColumnTypes");
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .build();
+
+        // First API we test is passing the table directly without any table definition
+        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        final String[] partitioningColumns = definition.getPartitioningColumns().stream()
+                .map(ColumnDefinition::getName).toArray(String[]::new);
+        Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+        assertTableEquals(inputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
+        Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+        assertTableEquals(inputData.sort(partitioningColumns), fromDiskWithMetadata.sort(partitioningColumns));
+
+        FileUtils.deleteRecursively(parentDir);
+
+        // Next API we test is passing the partitioned table without any table definition
+        final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
+        writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
+        fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+        assertTableEquals(inputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
+        fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+        assertTableEquals(inputData.sort(partitioningColumns), fromDiskWithMetadata.sort(partitioningColumns));
+
+        FileUtils.deleteRecursively(parentDir);
+
+        // Next API we test is passing the regular table with an updated table definition where we drop
+        // some partitioning columns and non-partitioning columns and add some new non-partitioning columns
+        final List<ColumnDefinition<?>> oldColumns = definition.getColumns();
+        final List<ColumnDefinition<?>> newColumns = oldColumns.stream()
+                .filter(cd -> (cd.getName() != "PC2") && (cd.getName() != "NPC6"))
+                .collect(Collectors.toList());
+        newColumns.add(ColumnDefinition.ofInt("NPC15"));
+        final TableDefinition newDefinition = TableDefinition.of(newColumns);
+        writeKeyValuePartitionedTable(inputData, newDefinition, parentDir, "data", writeInstructions);
+        final Table expected = inputData.dropColumns("PC2", "NPC6").updateView("NPC15 = (int)null");
+        fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+        assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+        fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+        assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+
+        FileUtils.deleteRecursively(parentDir);
+
+        // Next API we test is passing the partitioned table with an updated table definition
+        writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir, "data", writeInstructions);
+        fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+        assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+        fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+        assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+    }
+
+
     @Test
     public void testVectorColumns() {
         final Table table = getTableFlat(20000, true, false);

From ab73df0c0fbd8aa0390fb39beb35ac1a7274e74c Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 5 Mar 2024 16:42:11 -0600
Subject: [PATCH 25/45] Minor touchups

---
 .../io/deephaven/parquet/table/ParquetTools.java | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 7bef6f76874..4656cd11dda 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -533,7 +533,11 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
                 .map(ColumnDefinition::getName)
                 .toArray(String[]::new);
         final PartitionedTable partitionedTable = sourceTable.partitionBy(partitioningColNames);
-        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, baseName, writeInstructions);
+        final TableDefinition keyTableDefinition = TableDefinition.of(partitioningColumns);
+        final TableDefinition leafDefinition =
+                getNonKeyTableDefinition(Arrays.asList(partitioningColNames), definition);
+        writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
+                baseName, writeInstructions);
     }
 
     /**
@@ -610,6 +614,16 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
                 baseName, writeInstructions);
     }
 
+    /**
+     * Write a partitioned table to disk in a key=value partitioning format with the already computed definition for the
+     * key table and leaf table.
+     * <p>
+     * Check {@link #writeKeyValuePartitionedTable(PartitionedTable, File, String, ParquetInstructions)},
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, File, String, ParquetInstructions)},
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, File, String, ParquetInstructions)}}, or
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, File, String, ParquetInstructions)} for
+     * more details.
+     */
     private static void writeKeyValuePartitionedTableImpl(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition keyTableDefinition,
             @NotNull final TableDefinition leafDefinition,

From 1cb8b81174e3b9bf2770f588579c5516e8184c35 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 5 Mar 2024 18:51:43 -0600
Subject: [PATCH 26/45] Added fix and tests for big decimals

Also fixed the instruction for writing metadata file
---
 .../parquet/table/ParquetInstructions.java    |  99 +++++------------
 .../table/ParquetMetadataFileWriterImpl.java  |  13 ++-
 .../parquet/table/ParquetTableWriter.java     |  30 +++---
 .../deephaven/parquet/table/ParquetTools.java | 101 ++++++++++++++----
 .../io/deephaven/parquet/table/TypeInfos.java |   4 +
 .../table/ParquetTableReadWriteTest.java      |  44 +++++---
 6 files changed, 161 insertions(+), 130 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
index ffe2e45b888..8b5fa3a0937 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
@@ -10,7 +10,6 @@
 import io.deephaven.hash.KeyedObjectKey;
 import io.deephaven.util.annotations.VisibleForTesting;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.apache.parquet.schema.MessageType;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -116,48 +115,10 @@ public static int getDefaultTargetPageSize() {
         return defaultTargetPageSize;
     }
 
-    static final String DEFAULT_METADATA_ROOT_DIR = ""; // Empty = No metadata files written
-
-    private static final MessageType DEFAULT_COMMON_SCHEMA = null; // No common schema
+    private static final boolean DEFAULT_GENERATE_METADATA_FILES = false;
 
     public ParquetInstructions() {}
 
-    /**
-     * Create a new ParquetInstructions object by copying all fields from the base instructions, and then adding the
-     * provided common schema. This method is intended for use internally and therefore is package-private.
-     *
-     * @param baseInstructions The base instructions to copy from
-     * @param commonSchema The common schema to add
-     * @return A new ParquetInstructions object with the common schema added
-     */
-    static ParquetInstructions createWithCommonSchema(@NotNull final ParquetInstructions baseInstructions,
-            @NotNull final MessageType commonSchema) {
-        if (!(baseInstructions instanceof ReadOnly)) {
-            throw new UnsupportedOperationException("Cannot add common schema to non-ReadOnly ParquetInstructions");
-        }
-        if (baseInstructions.getCommonSchema() != DEFAULT_COMMON_SCHEMA) {
-            throw new UnsupportedOperationException("Cannot add common schema to ParquetInstructions with existing " +
-                    "common schema");
-        }
-        if (DEFAULT_METADATA_ROOT_DIR.equals(baseInstructions.getMetadataRootDir())) {
-            throw new UnsupportedOperationException("Cannot add common schema to ParquetInstructions without any" +
-                    " metadata root directory");
-        }
-        final ReadOnly readOnly = (ReadOnly) baseInstructions;
-        return new ReadOnly(
-                readOnly.copyColumnNameToInstructions(),
-                readOnly.copyParquetColumnNameToInstructions(),
-                readOnly.getCompressionCodecName(),
-                readOnly.getMaximumDictionaryKeys(),
-                readOnly.getMaximumDictionarySize(),
-                readOnly.isLegacyParquet(),
-                readOnly.getTargetPageSize(),
-                readOnly.isRefreshing(),
-                readOnly.getSpecialInstructions(),
-                readOnly.getMetadataRootDir(),
-                commonSchema);
-    }
-
     public final String getColumnNameFromParquetColumnNameOrDefault(final String parquetColumnName) {
         final String mapped = getColumnNameFromParquetColumnName(parquetColumnName);
         return (mapped != null) ? mapped : parquetColumnName;
@@ -206,14 +167,9 @@ public final String getColumnNameFromParquetColumnNameOrDefault(final String par
     public abstract boolean isRefreshing();
 
     /**
-     * @return the directory in which metadata files should be stored.
-     */
-    public abstract String getMetadataRootDir();
-
-    /**
-     * @return the common schema used for writing _common_metadata file.
+     * @return should we generate _meteadata and _common_metadata files while writing parquet files?
      */
-    abstract MessageType getCommonSchema();
+    public abstract boolean generateMetadataFiles();
 
     @VisibleForTesting
     public static boolean sameColumnNamesAndCodecMappings(final ParquetInstructions i1, final ParquetInstructions i2) {
@@ -291,13 +247,8 @@ public boolean isRefreshing() {
         }
 
         @Override
-        public String getMetadataRootDir() {
-            return DEFAULT_METADATA_ROOT_DIR;
-        }
-
-        @Override
-        public MessageType getCommonSchema() {
-            return DEFAULT_COMMON_SCHEMA;
+        public boolean generateMetadataFiles() {
+            return DEFAULT_GENERATE_METADATA_FILES;
         }
     };
 
@@ -367,8 +318,7 @@ private static final class ReadOnly extends ParquetInstructions {
         private final int targetPageSize;
         private final boolean isRefreshing;
         private final Object specialInstructions;
-        private final String metadataRootDir;
-        private final MessageType commonSchema;
+        private final boolean generateMetadataFiles;
 
         private ReadOnly(
                 final KeyedObjectHashMap<String, ColumnInstructions> columnNameToInstructions,
@@ -380,8 +330,7 @@ private ReadOnly(
                 final int targetPageSize,
                 final boolean isRefreshing,
                 final Object specialInstructions,
-                final String metadataRootDir,
-                final MessageType commonSchema) {
+                final boolean generateMetadataFiles) {
             this.columnNameToInstructions = columnNameToInstructions;
             this.parquetColumnNameToInstructions = parquetColumnNameToColumnName;
             this.compressionCodecName = compressionCodecName;
@@ -391,8 +340,7 @@ private ReadOnly(
             this.targetPageSize = targetPageSize;
             this.isRefreshing = isRefreshing;
             this.specialInstructions = specialInstructions;
-            this.metadataRootDir = metadataRootDir;
-            this.commonSchema = commonSchema;
+            this.generateMetadataFiles = generateMetadataFiles;
         }
 
         private String getOrDefault(final String columnName, final String defaultValue,
@@ -487,13 +435,8 @@ public boolean isRefreshing() {
         }
 
         @Override
-        public String getMetadataRootDir() {
-            return metadataRootDir;
-        }
-
-        @Override
-        public MessageType getCommonSchema() {
-            return commonSchema;
+        public boolean generateMetadataFiles() {
+            return generateMetadataFiles;
         }
 
         KeyedObjectHashMap<String, ColumnInstructions> copyColumnNameToInstructions() {
@@ -547,7 +490,7 @@ public static class Builder {
         private int targetPageSize = defaultTargetPageSize;
         private boolean isRefreshing = DEFAULT_IS_REFRESHING;
         private Object specialInstructions;
-        private String metadataRootDir = DEFAULT_METADATA_ROOT_DIR;
+        private boolean generateMetadataFiles = DEFAULT_GENERATE_METADATA_FILES;
 
         public Builder() {}
 
@@ -725,13 +668,19 @@ public Builder setSpecialInstructions(final Object specialInstructions) {
         }
 
         /**
-         * Set the default metadata root directory.
-         *
-         * @param metadataRootDir the root directory to store metadata files in. All the parquet destinations should be
-         *        inside this directory.
+         * Set whether to generate _metadata and _common_metadata files while writing parquet files.
+         * <ul>
+         * <li>When writing a single parquet file, metadata files are generated in the same parent directory as the
+         * parquet file.
+         * <li>When writing multiple parquet files in a single write call, metadata files are only generated if all
+         * parquet files are written to the same parent directory. The metadata files will also be written to the same
+         * parent directory.
+         * <li>When writing key-value partitioned parquet files, metadata files are generated in the root directory of
+         * the partitioned parquet files.
+         * </ul>
          */
-        public Builder setMetadataRootDir(final String metadataRootDir) {
-            this.metadataRootDir = metadataRootDir;
+        public Builder setGenerateMetadataFiles(final boolean generateMetadataFiles) {
+            this.generateMetadataFiles = generateMetadataFiles;
             return this;
         }
 
@@ -743,7 +692,7 @@ public ParquetInstructions build() {
             parquetColumnNameToInstructions = null;
             return new ReadOnly(columnNameToInstructionsOut, parquetColumnNameToColumnNameOut, compressionCodecName,
                     maximumDictionaryKeys, maximumDictionarySize, isLegacyParquet, targetPageSize, isRefreshing,
-                    specialInstructions, metadataRootDir, DEFAULT_COMMON_SCHEMA);
+                    specialInstructions, generateMetadataFiles);
         }
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index 7a39be5308a..f77b010863c 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -67,20 +67,19 @@ private static class ParquetFileMetadata {
      * @param destinations The indivdual parquet file destinations, all of which must be contained in the metadata root
      * @param commonSchema The common schema to be included for writing the _common_metadata file.
      */
-    ParquetMetadataFileWriterImpl(@NotNull final String metadataRootDir, @NotNull final File[] destinations,
+    ParquetMetadataFileWriterImpl(@NotNull final File metadataRootDir, @NotNull final File[] destinations,
             @Nullable final MessageType commonSchema) {
+        this.metadataRootDirAbsPath = metadataRootDir.getAbsolutePath();
         for (final File destination : destinations) {
-            if (!destination.getAbsolutePath().startsWith(metadataRootDir)) {
+            if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPath)) {
                 throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
                         + " directory, provided destination " + destination.getAbsolutePath() + " is not in " +
-                        metadataRootDir);
+                        metadataRootDirAbsPath);
             }
         }
-        final File rootDir = new File(metadataRootDir);
-        this.metadataRootDirAbsPath = rootDir.getAbsolutePath();
         this.parquetFileMetadataList = new ArrayList<>(destinations.length);
-        this.channelsProvider =
-                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(rootDir.getPath()), null);
+        this.channelsProvider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader(
+                convertToURI(metadataRootDirAbsPath), null);
         this.columnTypes = null;
         this.commonSchema = commonSchema;
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 190e0fb0e67..f9be6badce7 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -40,6 +40,7 @@
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Types;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
 import java.io.IOException;
@@ -99,9 +100,11 @@ static class GroupingColumnWritingInfo {
      * @param groupingColumnsWritingInfoMap A map of grouping column names to their respective info used for writing
      *        grouping files
      * @param metadataFileWriter The writer for the _metadata and _common_metadata files
+     * @param computedCache When we need to perform some computation depending on column data to make a decision
+     *        impacting both schema and written data, we store results in computedCache to avoid having to calculate
+     *        twice. An example is the necessary precision and scale for a BigDecimal column written as a decimal
+     *        logical type.
      *
-     * @throws SchemaMappingException Error creating a parquet table schema for the given table (likely due to
-     *         unsupported types)
      * @throws IOException For file writing related errors
      */
     static void write(
@@ -111,9 +114,10 @@ static void write(
             @NotNull final File destFile,
             @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> incomingMeta,
-            final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap,
-            @NotNull final ParquetMetadataFileWriter metadataFileWriter)
-            throws SchemaMappingException, IOException {
+            @Nullable final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap,
+            @NotNull final ParquetMetadataFileWriter metadataFileWriter,
+            @NotNull final Map<String, Map<ParquetCacheTags, Object>> computedCache)
+            throws IOException {
         final TableInfo.Builder tableInfoBuilder = TableInfo.builder();
         List<File> cleanupFiles = null;
         try {
@@ -133,11 +137,11 @@ static void write(
                     // We don't accumulate metadata from grouping files into the main metadata file
                     write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions, groupingDestFile,
                             metadataGroupingFilePath, Collections.emptyMap(), TableInfo.builder(),
-                            NullParquetMetadataFileWriter.INSTANCE);
+                            NullParquetMetadataFileWriter.INSTANCE, computedCache);
                 }
             }
             write(t, definition, writeInstructions, destFile, metadataFilePath, incomingMeta, tableInfoBuilder,
-                    metadataFileWriter);
+                    metadataFileWriter, computedCache);
         } catch (Exception e) {
             if (cleanupFiles != null) {
                 for (final File cleanupFile : cleanupFiles) {
@@ -165,11 +169,10 @@ static void write(
      * @param tableMeta A map of metadata values to be stores in the file footer
      * @param tableInfoBuilder A partially constructed builder for the metadata object
      * @param metadataFileWriter The writer for the _metadata and _common_metadata files
-     * @throws SchemaMappingException Error creating a parquet table schema for the given table (likely due to
-     *         unsupported types)
+     * @param computedCache Per column cache tags
      * @throws IOException For file writing related errors
      */
-    public static void write(
+    static void write(
             @NotNull final Table table,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
@@ -177,15 +180,12 @@ public static void write(
             @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> tableMeta,
             @NotNull final TableInfo.Builder tableInfoBuilder,
-            @NotNull final ParquetMetadataFileWriter metadataFileWriter) throws SchemaMappingException, IOException {
+            @NotNull final ParquetMetadataFileWriter metadataFileWriter,
+            @NotNull final Map<String, Map<ParquetCacheTags, Object>> computedCache) throws IOException {
         try (final SafeCloseable ignored = LivenessScopeStack.open()) {
             final Table t = pretransformTable(table, definition);
             final TrackingRowSet tableRowSet = t.getRowSet();
             final Map<String, ? extends ColumnSource<?>> columnSourceMap = t.getColumnSourceMap();
-            // When we need to perform some computation depending on column data to make a decision impacting both
-            // schema and written data, we store results in computedCache to avoid having to calculate twice.
-            // An example is the necessary precision and scale for a BigDecimal column written as decimal logical type.
-            final Map<String, Map<ParquetCacheTags, Object>> computedCache = new HashMap<>();
             final ParquetFileWriter parquetFileWriter = getParquetFileWriter(computedCache, definition, tableRowSet,
                     columnSourceMap, destFile, metadataFilePath, writeInstructions, tableMeta, tableInfoBuilder,
                     metadataFileWriter);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 4656cd11dda..0770b922107 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -11,6 +11,7 @@
 import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.primitive.iterator.CloseableIterator;
 import io.deephaven.engine.table.ColumnDefinition;
+import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.PartitionedTable;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
@@ -52,6 +53,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.net.URI;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -692,22 +694,25 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
                 row++;
             }
         }
-        final ParquetInstructions updatedWriteInstructions;
-        if (!ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(writeInstructions.getMetadataRootDir())) {
+        final MessageType commonSchema;
+        if (writeInstructions.generateMetadataFiles()) {
             // Generate initial schema for _common_metadata file from key columns. The schema for remaining columns will
             // be inferred at the time of writing the parquet files and merged with the common schema.
-            final MessageType commonSchema =
-                    getSchemaForTable(partitionedTable.table(), keyTableDefinition, writeInstructions);
-            updatedWriteInstructions = ParquetInstructions.createWithCommonSchema(writeInstructions, commonSchema);
+            commonSchema = getSchemaForTable(partitionedTable.table(), keyTableDefinition, writeInstructions);
         } else {
-            updatedWriteInstructions = writeInstructions;
+            commonSchema = null;
         }
-        ParquetTools.writeParquetTables(
+        final Map<String, Map<ParquetCacheTags, Object>> computedCache =
+                buildComputedCache(partitionedTable, leafDefinition);
+        ParquetTools.writeParquetTablesImpl(
                 partitionedData.toArray(Table[]::new),
                 leafDefinition,
-                updatedWriteInstructions,
+                writeInstructions,
                 destinations.toArray(File[]::new),
-                leafDefinition.getGroupingColumnNamesArray());
+                leafDefinition.getGroupingColumnNamesArray(),
+                commonSchema,
+                destinationDir,
+                computedCache);
     }
 
     /**
@@ -737,6 +742,30 @@ private static TableDefinition getNonKeyTableDefinition(@NotNull final Collectio
         return TableDefinition.of(nonKeyColumnDefinition);
     }
 
+    /**
+     * Precompute the precision and scale values for big decimal columns and pass it to the write method so that all the
+     * constituent parquet files are written with the same precision and scale values, and thus have the same schema.
+     */
+    private static Map<String, Map<ParquetCacheTags, Object>> buildComputedCache(
+            @NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition leafDefinition) {
+        final Map<String, Map<ParquetCacheTags, Object>> computedCache = new HashMap<>();
+        Table mergedConstituents = null;
+        final List<ColumnDefinition<?>> leafColumnDefinitions = leafDefinition.getColumns();
+        for (final ColumnDefinition<?> columnDefinition : leafColumnDefinitions) {
+            if (columnDefinition.getDataType() == BigDecimal.class) {
+                if (mergedConstituents == null) {
+                    mergedConstituents = partitionedTable.merge();
+                }
+                final String columnName = columnDefinition.getName();
+                final ColumnSource<BigDecimal> bigDecimalColumnSource = mergedConstituents.getColumnSource(columnName);
+                TypeInfos.getPrecisionAndScale(computedCache, columnName, mergedConstituents.getRowSet(),
+                        () -> bigDecimalColumnSource);
+            }
+        }
+        return computedCache;
+    }
+
     /**
      * Writes tables to disk in parquet format to a supplied set of destinations. If you specify grouping columns, there
      * must already be grouping information for those columns in the sources. This can be accomplished with
@@ -756,6 +785,40 @@ public static void writeParquetTables(@NotNull final Table[] sources,
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final File[] destinations,
             @Nullable final String[] groupingColumns) {
+        final File metadataRootDir;
+        if (writeInstructions.generateMetadataFiles()) {
+            // We will write the metadata file in the same directory as the destination files, so we need to ensure that
+            // all destination files are in the same directory.
+            final String firstDestinationDir = destinations[0].getParentFile().getAbsolutePath();
+            for (int i = 1; i < destinations.length; i++) {
+                if (!firstDestinationDir.equals(destinations[i].getParentFile().getAbsolutePath())) {
+                    throw new IllegalArgumentException("All destination files must be in the same directory for " +
+                            " generating metadata files");
+                }
+            }
+            metadataRootDir = new File(firstDestinationDir);
+        } else {
+            metadataRootDir = null;
+        }
+        // We do not have any common schema, therefore, is always set to null. Schema will be generated by combining the
+        // metadata at the time of writing the parquet file
+        writeParquetTablesImpl(sources, definition, writeInstructions, destinations, groupingColumns,
+                null, metadataRootDir, new HashMap<>());
+    }
+
+
+    /**
+     * Refer to {@link #writeParquetTables(Table[], TableDefinition, ParquetInstructions, File[], String[])} for more
+     * details.
+     */
+    private static void writeParquetTablesImpl(@NotNull final Table[] sources,
+            @NotNull final TableDefinition definition,
+            @NotNull final ParquetInstructions writeInstructions,
+            @NotNull final File[] destinations,
+            @Nullable final String[] groupingColumns,
+            @Nullable final MessageType commonSchema,
+            @Nullable final File metadataRootDir,
+            @NotNull final Map<String, Map<ParquetCacheTags, Object>> computedCache) {
         Require.eq(sources.length, "sources.length", destinations.length, "destinations.length");
         if (definition.numColumns() == 0) {
             throw new TableDataException("Cannot write a parquet table with zero columns");
@@ -770,11 +833,11 @@ public static void writeParquetTables(@NotNull final Table[] sources,
                 Arrays.stream(shadowDestFiles).map(ParquetTools::prepareDestinationFileLocation).toArray(File[]::new);
 
         final ParquetMetadataFileWriter metadataFileWriter;
-        final String metadataRootDir = writeInstructions.getMetadataRootDir();
-        final boolean writeMetadataFiles = !ParquetInstructions.DEFAULT_METADATA_ROOT_DIR.equals(metadataRootDir);
-        if (writeMetadataFiles) {
-            metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations,
-                    writeInstructions.getCommonSchema());
+        if (writeInstructions.generateMetadataFiles()) {
+            if (metadataRootDir == null) {
+                throw new IllegalArgumentException("Metadata root directory must be set when writing metadata files");
+            }
+            metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations, commonSchema);
         } else {
             metadataFileWriter = NullParquetMetadataFileWriter.INSTANCE;
         }
@@ -793,7 +856,8 @@ public static void writeParquetTables(@NotNull final Table[] sources,
                     final Table source = sources[tableIdx];
                     ParquetTableWriter.write(source, definition, writeInstructions, shadowDestFiles[tableIdx],
                             destinations[tableIdx], Collections.emptyMap(),
-                            (Map<String, ParquetTableWriter.GroupingColumnWritingInfo>) null, metadataFileWriter);
+                            (Map<String, ParquetTableWriter.GroupingColumnWritingInfo>) null, metadataFileWriter,
+                            computedCache);
                 }
             } else {
                 // Create grouping info for each table and write the table and grouping files to shadow path
@@ -815,13 +879,14 @@ public static void writeParquetTables(@NotNull final Table[] sources,
 
                     final Table sourceTable = sources[tableIdx];
                     ParquetTableWriter.write(sourceTable, definition, writeInstructions, shadowDestFiles[tableIdx],
-                            tableDestination, Collections.emptyMap(), groupingColumnWritingInfoMap, metadataFileWriter);
+                            tableDestination, Collections.emptyMap(), groupingColumnWritingInfoMap, metadataFileWriter,
+                            computedCache);
                 }
             }
 
             // Write the combined metadata files to shadow destinations
             final File metadataDestFile, shadowMetadataFile, commonMetadataDestFile, shadowCommonMetadataFile;
-            if (writeMetadataFiles) {
+            if (writeInstructions.generateMetadataFiles()) {
                 metadataDestFile = new File(metadataRootDir, METADATA_FILE_NAME);
                 shadowMetadataFile = ParquetTools.getShadowFile(metadataDestFile);
                 shadowFiles.add(shadowMetadataFile);
@@ -848,7 +913,7 @@ public static void writeParquetTables(@NotNull final Table[] sources,
                     }
                 }
             }
-            if (writeMetadataFiles) {
+            if (writeInstructions.generateMetadataFiles()) {
                 destFiles.add(metadataDestFile);
                 installShadowFile(metadataDestFile, shadowMetadataFile);
                 destFiles.add(commonMetadataDestFile);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java
index 48d43f44dc4..2224df50102 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/TypeInfos.java
@@ -111,6 +111,10 @@ static Pair<String, String> getCodecAndArgs(
         return new ImmutablePair<>(SerializableCodec.class.getName(), null);
     }
 
+    /**
+     * Get the precision and scale for a given big decimal column. If already cached, fetch it directly, else compute it
+     * by scanning the entire column and store the values in the cache.
+     */
     public static PrecisionAndScale getPrecisionAndScale(
             @NotNull final Map<String, Map<ParquetCacheTags, Object>> computedCache,
             @NotNull final String columnName,
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 0dfc47e601b..5682a4f74a2 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -481,7 +481,7 @@ public void basicParquetWithMetadataTest() {
         final String filename = "basicParquetWithMetadataTest.parquet";
         final File destFile = new File(rootFile, filename);
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(rootFile.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         writeTable(table, destFile, writeInstructions);
 
@@ -508,7 +508,7 @@ public void parquetWithGroupingDataAndMetadataTest() {
 
         final File destFile = new File(parentDir, "parquetWithGroupingDataAndMetadataTest.parquet");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         writeTable(table, destFile, writeInstructions);
 
@@ -541,7 +541,7 @@ public void flatPartitionedParquetWithMetadataTest() throws IOException {
         parentDir.delete();
         parentDir.mkdir();
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         writeParquetTables(new Table[] {someTable, someTable}, someTable.getDefinition(), writeInstructions,
                 new File[] {firstDataFile, secondDataFile}, null);
@@ -561,6 +561,16 @@ public void flatPartitionedParquetWithMetadataTest() throws IOException {
         secondDataFile.createNewFile();
         final Table fromDiskWithMetadataWithoutData = readTable(metadataFile);
         assertEquals(source.size(), fromDiskWithMetadataWithoutData.size());
+
+        // Now write with flat partitioned parquet files to different directories with metadata file
+        parentDir.delete();
+        final File updatedSecondDataFile = new File(rootFile, "testDir/data2.parquet");
+        try {
+            writeParquetTables(new Table[] {someTable, someTable}, someTable.getDefinition(), writeInstructions,
+                    new File[] {firstDataFile, updatedSecondDataFile}, null);
+            fail("Expected exception when writing the metadata files for tables with different parent directories");
+        } catch (final RuntimeException expected) {
+        }
     }
 
     @Test
@@ -577,7 +587,7 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
 
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         writeKeyValuePartitionedTable(inputData, parentDir, "{partitions}-data-{i}", writeInstructions);
 
@@ -611,7 +621,7 @@ public void writeKeyValuePartitionedDataWithNoNonPartitioningColumnsTest() {
 
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         try {
             writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
@@ -636,7 +646,7 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         assertFalse(partitionedTableWithDuplicatedKeys.uniqueKeys());
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataWithNonUniqueKeys");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         try {
             writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data", writeInstructions);
@@ -698,7 +708,7 @@ public void writeKeyValuePartitionedDataWithNullKeys() {
         final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataWithNullKeys");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         try {
             writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
@@ -736,7 +746,7 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
 
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         writeKeyValuePartitionedTable(inputData, tableDefinitionToWrite, parentDir, "data", writeInstructions);
 
@@ -786,7 +796,7 @@ public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
 
         final File parentDir = new File(rootFile, "someTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         final PartitionedTable partitionedTable =
                 inputData.partitionBy("symbol", "epic_collection_id", "epic_request_id");
@@ -836,7 +846,7 @@ public void testAllPartitioningColumnTypes() {
                 "PC8 = (float)(ii % 2)",
                 "PC9 = (double)(ii % 2)",
                 "PC10 = java.math.BigInteger.valueOf(ii)",
-                "PC11 = java.math.BigDecimal.valueOf((double)ii)",
+                "PC11 = (ii%2 == 0) ? java.math.BigDecimal.valueOf((double)ii) : java.math.BigDecimal.valueOf((double)(ii*0.001))",
                 "PC12 = java.time.Instant.ofEpochSecond(ii)",
                 "PC13 = java.time.LocalDate.ofEpochDay(ii)",
                 "PC14 = java.time.LocalTime.of(i%24, i%60, (i+10)%60)",
@@ -845,7 +855,7 @@ public void testAllPartitioningColumnTypes() {
 
         final File parentDir = new File(rootFile, "testAllPartitioningColumnTypes");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
         writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
 
@@ -876,12 +886,12 @@ public void testAllNonPartitioningColumnTypes() {
                 ColumnDefinition.ofFloat("NPC8"),
                 ColumnDefinition.ofDouble("NPC9"),
                 ColumnDefinition.of("NPC10", Type.find(BigInteger.class)),
-                ColumnDefinition.of("NPC11", Type.find(BigDecimal.class)),
+                ColumnDefinition.of("bdColumn", Type.find(BigDecimal.class)),
                 ColumnDefinition.of("NPC12", Type.find(Instant.class)),
                 ColumnDefinition.of("NPC13", Type.find(LocalDate.class)),
                 ColumnDefinition.of("NPC14", Type.find(LocalTime.class)));
 
-        final Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView(
+        Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView(
                 "PC1 =  (ii%2 == 0) ? `AA` : `BB`",
                 "PC2 = (int)(ii%3)",
                 "NPC1 =  (ii%2 == 0) ? `AA` : `BB`",
@@ -894,7 +904,7 @@ public void testAllNonPartitioningColumnTypes() {
                 "NPC8 = (float)(ii % 2)",
                 "NPC9 = (double)(ii % 2)",
                 "NPC10 = java.math.BigInteger.valueOf(ii)",
-                "NPC11 = java.math.BigDecimal.valueOf((double)ii)",
+                "bdColumn = (ii%2 == 0) ? java.math.BigDecimal.valueOf((double)ii) : java.math.BigDecimal.valueOf((double)(ii*0.001))",
                 "NPC12 = java.time.Instant.ofEpochSecond(ii)",
                 "NPC13 = java.time.LocalDate.ofEpochDay(ii)",
                 "NPC14 = java.time.LocalTime.of(i%24, i%60, (i+10)%60)"))
@@ -902,11 +912,15 @@ public void testAllNonPartitioningColumnTypes() {
 
         final File parentDir = new File(rootFile, "testAllNonPartitioningColumnTypes");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
-                .setMetadataRootDir(parentDir.getAbsolutePath())
+                .setGenerateMetadataFiles(true)
                 .build();
 
         // First API we test is passing the table directly without any table definition
         writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+
+        // Store the big decimal with the precision and scale consistent with what we write to parquet
+        inputData = maybeFixBigDecimal(inputData);
+
         final String[] partitioningColumns = definition.getPartitioningColumns().stream()
                 .map(ColumnDefinition::getName).toArray(String[]::new);
         Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();

From 20e8204708157049071360dde5d492f7a3f03f62 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 5 Mar 2024 21:59:51 -0600
Subject: [PATCH 27/45] Updated a comment

---
 .../main/java/io/deephaven/parquet/table/ParquetTools.java   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 0770b922107..7104a4ecf70 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -800,13 +800,12 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         } else {
             metadataRootDir = null;
         }
-        // We do not have any common schema, therefore, is always set to null. Schema will be generated by combining the
-        // metadata at the time of writing the parquet file
+        // We do not have any additional common schema here. Schema for all columns will be generated at the time of
+        // writing the parquet files and merged to generate the metadata files.
         writeParquetTablesImpl(sources, definition, writeInstructions, destinations, groupingColumns,
                 null, metadataRootDir, new HashMap<>());
     }
 
-
     /**
      * Refer to {@link #writeParquetTables(Table[], TableDefinition, ParquetInstructions, File[], String[])} for more
      * details.

From 9892d14c37b27d43d5f03c3429c5544dd9f1fd3b Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Thu, 7 Mar 2024 11:10:55 -0600
Subject: [PATCH 28/45] Review with Ryan part 1

---
 BenchmarkSupport/BenchmarkSupport.gradle      |   1 +
 .../runner/TableBenchmarkState.java           |   2 +-
 engine/table/build.gradle                     |   1 +
 .../locations/util/PartitionFormatter.java    | 154 ++++++++++++++++++
 .../table/impl/QueryTableAggregationTest.java |   2 +-
 .../parquet/base/ColumnChunkReader.java       |   5 +-
 .../parquet/base/ColumnChunkReaderImpl.java   |  14 +-
 .../base/NullParquetMetadataFileWriter.java   |   7 +-
 .../parquet/base/OffsetIndexReader.java       |  10 +-
 .../parquet/base/OffsetIndexReaderImpl.java   |  16 +-
 .../parquet/base/ParquetFileReader.java       |   4 +-
 .../parquet/base/ParquetFileWriter.java       |  13 +-
 .../base/ParquetMetadataFileWriter.java       |   7 +-
 .../deephaven/parquet/base}/ParquetUtils.java |  10 +-
 .../base/PositionedBufferedOutputStream.java  |   9 +-
 .../parquet/base/RowGroupReader.java          |   5 +-
 .../parquet/base/RowGroupReaderImpl.java      |  12 +-
 .../parquet/table/ParquetInstructions.java    |  17 +-
 .../table/ParquetMetadataFileWriterImpl.java  | 142 +++++++++-------
 .../parquet/table/ParquetTableWriter.java     |  38 +++--
 .../deephaven/parquet/table/ParquetTools.java |  82 ++++++----
 .../parquet/table/PartitionFormatter.java     | 132 ---------------
 .../layout/ParquetFlatPartitionedLayout.java  |   2 +-
 .../ParquetKeyValuePartitionedLayout.java     |   2 +-
 .../layout/ParquetMetadataFileLayout.java     |  18 +-
 .../location/ParquetTableLocationKey.java     |   2 +-
 .../table/ParquetTableReadWriteTest.java      |  68 +++++---
 27 files changed, 439 insertions(+), 336 deletions(-)
 create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/locations/util/PartitionFormatter.java
 rename extensions/parquet/{table/src/main/java/io/deephaven/parquet/table => base/src/main/java/io/deephaven/parquet/base}/ParquetUtils.java (60%)
 delete mode 100644 extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java

diff --git a/BenchmarkSupport/BenchmarkSupport.gradle b/BenchmarkSupport/BenchmarkSupport.gradle
index 70118d168a0..8c8f9c0cf77 100644
--- a/BenchmarkSupport/BenchmarkSupport.gradle
+++ b/BenchmarkSupport/BenchmarkSupport.gradle
@@ -30,6 +30,7 @@ dependencies {
     implementation project(':Base')
     implementation project(':engine-table')
     implementation project(':extensions-parquet-table')
+    implementation project(':extensions-parquet-base')
     implementation project(':Configuration')
     implementation 'org.openjdk.jmh:jmh-core:1.20'
 
diff --git a/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java b/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java
index 93a982c275d..8d641fd2f97 100644
--- a/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java
+++ b/BenchmarkSupport/src/main/java/io/deephaven/benchmarking/runner/TableBenchmarkState.java
@@ -12,7 +12,7 @@
 import io.deephaven.benchmarking.BenchmarkTools;
 import org.openjdk.jmh.infra.BenchmarkParams;
 
-import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
+import static io.deephaven.parquet.base.ParquetUtils.PARQUET_FILE_EXTENSION;
 
 import java.io.File;
 import java.io.IOException;
diff --git a/engine/table/build.gradle b/engine/table/build.gradle
index 8809652e223..acac9fa9580 100644
--- a/engine/table/build.gradle
+++ b/engine/table/build.gradle
@@ -65,6 +65,7 @@ dependencies {
     testImplementation project(':BenchmarkSupport')
     testImplementation project(':extensions-csv')
     testImplementation project(':extensions-parquet-table')
+    testImplementation project(':extensions-parquet-base')
     testImplementation project(':extensions-source-support')
     testImplementation project(':Numerics')
     testImplementation project(':extensions-suanshu')
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/util/PartitionFormatter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/util/PartitionFormatter.java
new file mode 100644
index 00000000000..ec9be62d9c5
--- /dev/null
+++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/util/PartitionFormatter.java
@@ -0,0 +1,154 @@
+package io.deephaven.engine.table.impl.locations.util;
+
+import io.deephaven.time.DateTimeUtils;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This class takes a partition value object and formats it to a {@link String}. Useful when generating partitioning
+ * paths for table. Complementary to {@link PartitionParser}.
+ */
+public enum PartitionFormatter {
+    ForString {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return (String) value;
+        }
+    },
+    ForBoolean {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Boolean) value).toString();
+        }
+    },
+    ForChar {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Character) value).toString();
+        }
+    },
+    ForByte {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Byte) value).toString();
+        }
+    },
+    ForShort {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Short) value).toString();
+        }
+    },
+    ForInt {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Integer) value).toString();
+        }
+    },
+    ForLong {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Long) value).toString();
+        }
+    },
+    ForFloat {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Float) value).toString();
+        }
+    },
+    ForDouble {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Double) value).toString();
+        }
+    },
+    ForBigInteger {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((BigInteger) value).toString();
+        }
+    },
+    ForBigDecimal {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((BigDecimal) value).toString();
+        }
+    },
+    ForInstant {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((Instant) value).toString();
+        }
+    },
+    ForLocalDate {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return DateTimeUtils.formatDate((LocalDate) value);
+        }
+    },
+    ForLocalTime {
+        @Override
+        public String formatObject(@NotNull final Object value) {
+            return ((LocalTime) value).toString();
+        }
+    };
+
+    private static final Map<Class<?>, PartitionFormatter> typeMap = new HashMap<>();
+    static {
+        typeMap.put(String.class, ForString);
+        typeMap.put(Boolean.class, ForBoolean);
+        typeMap.put(boolean.class, ForBoolean);
+        typeMap.put(Character.class, ForChar);
+        typeMap.put(char.class, ForChar);
+        typeMap.put(Byte.class, ForByte);
+        typeMap.put(byte.class, ForByte);
+        typeMap.put(Short.class, ForShort);
+        typeMap.put(short.class, ForShort);
+        typeMap.put(Integer.class, ForInt);
+        typeMap.put(int.class, ForInt);
+        typeMap.put(Long.class, ForLong);
+        typeMap.put(long.class, ForLong);
+        typeMap.put(Float.class, ForFloat);
+        typeMap.put(float.class, ForFloat);
+        typeMap.put(Double.class, ForDouble);
+        typeMap.put(double.class, ForDouble);
+        typeMap.put(BigInteger.class, ForBigInteger);
+        typeMap.put(BigDecimal.class, ForBigDecimal);
+        typeMap.put(Instant.class, ForInstant);
+        typeMap.put(LocalDate.class, ForLocalDate);
+        typeMap.put(LocalTime.class, ForLocalTime);
+    }
+
+    abstract String formatObject(@NotNull final Object value);
+
+    /**
+     * Takes a partition value object and returns a formatted string. Returns an empty string if the object is null.
+     */
+    public String format(@Nullable final Object value) {
+        if (value == null) {
+            return "";
+        }
+        return formatObject(value);
+    }
+
+    /**
+     * Takes a partitioning column type and returns the corresponding formatter.
+     */
+    public static PartitionFormatter getFormatterForType(@NotNull final Class<?> clazz) {
+        final PartitionFormatter formatter = typeMap.get(clazz);
+        if (formatter != null) {
+            return formatter;
+        } else {
+            throw new UnsupportedOperationException("Unsupported type: " + clazz.getSimpleName());
+        }
+    }
+}
diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
index 00636c04aad..7c7604ba542 100644
--- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
+++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
@@ -74,7 +74,7 @@
 import static io.deephaven.util.QueryConstants.*;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.*;
-import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
+import static io.deephaven.parquet.base.ParquetUtils.PARQUET_FILE_EXTENSION;
 
 @Category(OutOfBandTest.class)
 public class QueryTableAggregationTest {
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java
index 4302f2b36bb..3bade6e6848 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java
@@ -36,10 +36,9 @@ public interface ColumnChunkReader {
     boolean hasOffsetIndex();
 
     /**
-     * Get the offset index for a column chunk.
-     *
      * @param context The channel context to use for reading the offset index.
-     *
+     * @return Get the offset index for a column chunk.
+     * @throws UnsupportedOperationException If the column chunk does not have an offset index.
      */
     OffsetIndex getOffsetIndex(final SeekableChannelContext context);
 
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
index d87b9360666..b99fba71431 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -27,10 +27,12 @@
 import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.function.Function;
 
+import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 import static org.apache.parquet.format.Encoding.PLAIN_DICTIONARY;
 import static org.apache.parquet.format.Encoding.RLE_DICTIONARY;
 
@@ -54,11 +56,10 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
      */
     private final String version;
 
-    ColumnChunkReaderImpl(ColumnChunk columnChunk, SeekableChannelsProvider channelsProvider, URI columnChunkURI,
+    ColumnChunkReaderImpl(ColumnChunk columnChunk, SeekableChannelsProvider channelsProvider, URI rootURI,
             MessageType type, List<Type> fieldTypes, final long numRows, final String version) {
         this.channelsProvider = channelsProvider;
         this.columnChunk = columnChunk;
-        this.columnChunkURI = columnChunkURI;
         this.path = type
                 .getColumnDescription(columnChunk.meta_data.getPath_in_schema().toArray(new String[0]));
         if (columnChunk.getMeta_data().isSetCodec()) {
@@ -72,6 +73,12 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
         this.nullMaterializerFactory = PageMaterializer.factoryForType(path.getPrimitiveType().getPrimitiveTypeName());
         this.numRows = numRows;
         this.version = version;
+        if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
+            this.columnChunkURI = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
+        } else {
+            // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
+            this.columnChunkURI = rootURI;
+        }
         // Construct the reader object but don't read the offset index yet
         this.offsetIndexReader = (columnChunk.isSetOffset_index_offset())
                 ? new OffsetIndexReaderImpl(channelsProvider, columnChunk, columnChunkURI)
@@ -100,7 +107,8 @@ public boolean hasOffsetIndex() {
 
     @Override
     public OffsetIndex getOffsetIndex(final SeekableChannelContext context) {
-        // Read the offset index if it hasn't been read yet
+        // Reads and caches the offset index if it hasn't been read yet. Throws an exception if the offset index cannot
+        // be read from this source
         return offsetIndexReader.getOffsetIndex(context);
     }
 
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
index 95c91ca8f6e..50b8f4fa341 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
@@ -7,11 +7,9 @@
 /**
  * A no-op implementation of MetadataFileWriterBase when we don't want to write metadata files for Parquet files.
  */
-public final class NullParquetMetadataFileWriter implements ParquetMetadataFileWriter {
+public enum NullParquetMetadataFileWriter implements ParquetMetadataFileWriter {
 
-    public static final NullParquetMetadataFileWriter INSTANCE = new NullParquetMetadataFileWriter();
-
-    private NullParquetMetadataFileWriter() {}
+    INSTANCE;
 
     @Override
     public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata metadata) {}
@@ -22,4 +20,3 @@ public void writeMetadataFiles(final File metadataFile, final File commonMetadat
     @Override
     public void clear() {}
 }
-
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java
index dbbe7264869..c3218624f31 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReader.java
@@ -9,14 +9,16 @@
 public interface OffsetIndexReader {
 
     /**
-     * Get the offset index for a column chunk.
-     *
      * @param context The channel context to use for reading the offset index.
+     * @return Reads, caches, and returns the offset index for a column chunk.
+     * @throws UnsupportedOperationException If the offset index cannot be read from this source.
      */
     OffsetIndex getOffsetIndex(SeekableChannelContext context);
 
     /**
-     * A null implementation of the offset index reader.
+     * A null implementation of the offset index reader which always throws an exception when called.
      */
-    OffsetIndexReader NULL = context -> null;
+    OffsetIndexReader NULL = context -> {
+        throw new UnsupportedOperationException("Cannot read offset index from this source.");
+    };
 }
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java
index ecaeba34664..12a719a0dc1 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/OffsetIndexReaderImpl.java
@@ -7,6 +7,7 @@
 import org.apache.parquet.internal.column.columnindex.OffsetIndex;
 import org.apache.parquet.format.Util;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -15,28 +16,33 @@
 import java.nio.channels.SeekableByteChannel;
 
 /**
- * Implementation of {@link OffsetIndexReader}, which reads the offset index for a column chunk on demand.
+ * Implementation of {@link OffsetIndexReader}, which reads the offset index for a column chunk on demand, and caches it
+ * for future requests.
  */
 final class OffsetIndexReaderImpl implements OffsetIndexReader {
 
     private final SeekableChannelsProvider channelsProvider;
-    private final ColumnChunk chunk;
+    private final ColumnChunk columnChunk;
     private final URI columnChunkURI;
     private OffsetIndex offsetIndex;
 
-    OffsetIndexReaderImpl(final SeekableChannelsProvider channelsProvider, final ColumnChunk chunk,
+    OffsetIndexReaderImpl(final SeekableChannelsProvider channelsProvider, final ColumnChunk columnChunk,
             final URI columnChunkURI) {
         this.channelsProvider = channelsProvider;
-        this.chunk = chunk;
+        this.columnChunk = columnChunk;
         this.columnChunkURI = columnChunkURI;
         this.offsetIndex = null;
     }
 
     @Override
+    @Nullable
     public OffsetIndex getOffsetIndex(@NotNull final SeekableChannelContext context) {
         if (offsetIndex != null) {
             return offsetIndex;
         }
+        if (!columnChunk.isSetOffset_index_offset()) {
+            throw new UnsupportedOperationException("Cannot read offset index from this source.");
+        }
         return readOffsetIndex(context);
     }
 
@@ -46,7 +52,7 @@ private OffsetIndex readOffsetIndex(@NotNull final SeekableChannelContext channe
                         SeekableChannelContext.ensureContext(channelsProvider, channelContext);
                 final SeekableByteChannel readChannel = channelsProvider.getReadChannel(holder.get(), columnChunkURI);
                 final InputStream in =
-                        channelsProvider.getInputStream(readChannel.position(chunk.getOffset_index_offset()))) {
+                        channelsProvider.getInputStream(readChannel.position(columnChunk.getOffset_index_offset()))) {
             return (offsetIndex = ParquetMetadataConverter.fromParquetOffsetIndex(Util.readOffsetIndex(in)));
         } catch (final IOException e) {
             throw new UncheckedIOException(e);
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java
index d8ffbb4307f..4502a401747 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java
@@ -14,10 +14,10 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.nio.channels.SeekableByteChannel;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.*;
 
+import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
 /**
@@ -26,8 +26,6 @@
  */
 public class ParquetFileReader {
     private static final int FOOTER_LENGTH_SIZE = 4;
-    private static final String MAGIC_STR = "PAR1";
-    public static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII);
     public static final String FILE_URI_SCHEME = "file";
 
     public final FileMetaData fileMetaData;
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 32e390dff14..071ad0e0355 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -25,12 +25,13 @@
 import java.util.List;
 import java.util.Map;
 
+import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
+import static io.deephaven.parquet.base.ParquetUtils.PARQUET_OUTPUT_BUFFER_SIZE;
 import static org.apache.parquet.format.Util.writeFileMetaData;
 
 public final class ParquetFileWriter {
     private static final ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter();
     private static final int VERSION = 1;
-    private static final int OUTPUT_BUFFER_SIZE = 1 << 18;
 
     private final PositionedBufferedOutputStream bufferedOutput;
     private final MessageType type;
@@ -44,7 +45,7 @@ public final class ParquetFileWriter {
     private final ParquetMetadataFileWriter metadataFileWriter;
 
     public ParquetFileWriter(
-            final File destFile,
+            final String filePath,
             final File metadataFilePath,
             final SeekableChannelsProvider channelsProvider,
             final int targetPageSize,
@@ -56,9 +57,9 @@ public ParquetFileWriter(
         this.targetPageSize = targetPageSize;
         this.allocator = allocator;
         this.extraMetaData = new HashMap<>(extraMetaData);
-        bufferedOutput = new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(destFile.getPath(), false),
-                OUTPUT_BUFFER_SIZE);
-        bufferedOutput.write(ParquetFileReader.MAGIC);
+        bufferedOutput = new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(filePath, false),
+                PARQUET_OUTPUT_BUFFER_SIZE);
+        bufferedOutput.write(MAGIC);
         this.type = type;
         this.compressorAdapter = DeephavenCompressorAdapterFactory.getInstance().getByName(codecName);
         this.metadataFilePath = metadataFilePath;
@@ -93,7 +94,7 @@ public static void serializeFooter(final ParquetMetadata footer,
                 metadataConverter.toParquetMetadata(VERSION, footer);
         writeFileMetaData(parquetMetadata, bufferedOutput);
         BytesUtils.writeIntLittleEndian(bufferedOutput, (int) (bufferedOutput.position() - footerIndex));
-        bufferedOutput.write(ParquetFileReader.MAGIC);
+        bufferedOutput.write(MAGIC);
     }
 
     private void serializeOffsetIndexes() throws IOException {
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
index a8f3fb9a0b3..2872c2c3cb6 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
@@ -6,7 +6,8 @@
 import java.io.IOException;
 
 /**
- * Used to write _metadata and _common_metadata files for Parquet.
+ * Used to write {@value ParquetUtils#METADATA_FILE_NAME} and {@value ParquetUtils#COMMON_METADATA_FILE_NAME} files for
+ * Parquet.
  */
 public interface ParquetMetadataFileWriter {
 
@@ -22,8 +23,8 @@ public interface ParquetMetadataFileWriter {
     /**
      * Write the combined metadata files for all metadata accumulated so far and clear the list.
      *
-     * @param metadataFile The destination file for the _metadata file
-     * @param commonMetadataFile The destination file for the _common_metadata file
+     * @param metadataFile The destination for the {@value ParquetUtils#METADATA_FILE_NAME} file
+     * @param commonMetadataFile The destination for the {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file
      */
     void writeMetadataFiles(File metadataFile, File commonMetadataFile) throws IOException;
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
similarity index 60%
rename from extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java
rename to extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
index 87932f18b54..8b1e193108e 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetUtils.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
@@ -1,5 +1,6 @@
-package io.deephaven.parquet.table;
+package io.deephaven.parquet.base;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 
 public final class ParquetUtils {
@@ -7,6 +8,13 @@ public final class ParquetUtils {
     public static final String METADATA_FILE_NAME = "_metadata";
     public static final String COMMON_METADATA_FILE_NAME = "_common_metadata";
     public static final String PARQUET_FILE_EXTENSION = ".parquet";
+    private static final String MAGIC_STR = "PAR1";
+    public static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII);
+
+    /**
+     * The number of bytes to buffer before flushing while writing parquet files and metadata files.
+     */
+    public static final int PARQUET_OUTPUT_BUFFER_SIZE = 1 << 18;
 
     /**
      * Used as a filter to select relevant parquet files while reading all files in a directory.
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java
index 9c857e5f3a3..112d1e38008 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/PositionedBufferedOutputStream.java
@@ -1,5 +1,7 @@
 package io.deephaven.parquet.base;
 
+import org.jetbrains.annotations.NotNull;
+
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.nio.channels.Channels;
@@ -9,12 +11,7 @@ public final class PositionedBufferedOutputStream extends BufferedOutputStream {
 
     private final SeekableByteChannel writeChannel;
 
-    public PositionedBufferedOutputStream(final SeekableByteChannel writeChannel) {
-        super(Channels.newOutputStream(writeChannel));
-        this.writeChannel = writeChannel;
-    }
-
-    PositionedBufferedOutputStream(final SeekableByteChannel writeChannel, final int size) {
+    public PositionedBufferedOutputStream(@NotNull final SeekableByteChannel writeChannel, final int size) {
         super(Channels.newOutputStream(writeChannel), size);
         this.writeChannel = writeChannel;
     }
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java
index cf340d148dd..dc147a2701c 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReader.java
@@ -3,9 +3,9 @@
  */
 package io.deephaven.parquet.base;
 
-import io.deephaven.util.channel.SeekableChannelContext;
 import org.apache.parquet.format.RowGroup;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.List;
 
@@ -17,8 +17,9 @@ public interface RowGroupReader {
      * Returns the accessor to a given Column Chunk
      * 
      * @param path the full column path
-     * @return the accessor to a given Column Chunk
+     * @return the accessor to a given Column Chunk, or null if the column is not present in this Row Group
      */
+    @Nullable
     ColumnChunkReader getColumnChunk(@NotNull List<String> path);
 
     long numRows();
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index 8212fc2458b..dfd80fa9cf8 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -62,6 +62,7 @@ final class RowGroupReaderImpl implements RowGroupReader {
     }
 
     @Override
+    @Nullable
     public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path) {
         final String key = path.toString();
         final ColumnChunk columnChunk = chunkMap.get(key);
@@ -69,16 +70,7 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List<String> path) {
         if (columnChunk == null) {
             return null;
         }
-
-        final URI columnChunkURI;
-        if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
-            columnChunkURI = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
-        } else {
-            // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
-            columnChunkURI = rootURI;
-        }
-        return new ColumnChunkReaderImpl(columnChunk, channelsProvider, columnChunkURI, type, fieldTypes, numRows(),
-                version);
+        return new ColumnChunkReaderImpl(columnChunk, channelsProvider, rootURI, type, fieldTypes, numRows(), version);
     }
 
     @Override
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
index 8b5fa3a0937..24caf138279 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
@@ -8,6 +8,7 @@
 import io.deephaven.engine.table.impl.ColumnToCodecMappings;
 import io.deephaven.hash.KeyedObjectHashMap;
 import io.deephaven.hash.KeyedObjectKey;
+import io.deephaven.parquet.base.ParquetUtils;
 import io.deephaven.util.annotations.VisibleForTesting;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 import org.jetbrains.annotations.NotNull;
@@ -167,7 +168,8 @@ public final String getColumnNameFromParquetColumnNameOrDefault(final String par
     public abstract boolean isRefreshing();
 
     /**
-     * @return should we generate _meteadata and _common_metadata files while writing parquet files?
+     * @return should we generate {@value ParquetUtils#METADATA_FILE_NAME} and
+     *         {@value ParquetUtils#COMMON_METADATA_FILE_NAME} files while writing parquet files?
      */
     public abstract boolean generateMetadataFiles();
 
@@ -668,14 +670,15 @@ public Builder setSpecialInstructions(final Object specialInstructions) {
         }
 
         /**
-         * Set whether to generate _metadata and _common_metadata files while writing parquet files.
+         * Set whether to generate {@value ParquetUtils#METADATA_FILE_NAME} and
+         * {@value ParquetUtils#COMMON_METADATA_FILE_NAME} files while writing parquet files. On setting this parameter,
          * <ul>
-         * <li>When writing a single parquet file, metadata files are generated in the same parent directory as the
+         * <li>When writing a single parquet file, metadata files will be generated in the same parent directory as the
          * parquet file.
-         * <li>When writing multiple parquet files in a single write call, metadata files are only generated if all
-         * parquet files are written to the same parent directory. The metadata files will also be written to the same
-         * parent directory.
-         * <li>When writing key-value partitioned parquet files, metadata files are generated in the root directory of
+         * <li>When writing multiple parquet files in a single write call, the writing code insists that all parquet
+         * files should be written to the same parent directory, and only then metadata files will be generated in the
+         * same parent directory.
+         * <li>When writing key-value partitioned parquet data, metadata files are generated in the root directory of
          * the partitioned parquet files.
          * </ul>
          */
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index f77b010863c..aaf9b1bf45c 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -1,10 +1,9 @@
 package io.deephaven.parquet.table;
 
 import io.deephaven.UncheckedDeephavenException;
-import io.deephaven.base.verify.Assert;
-import io.deephaven.parquet.base.ParquetFileReader;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
+import io.deephaven.parquet.base.ParquetUtils;
 import io.deephaven.parquet.base.PositionedBufferedOutputStream;
 import io.deephaven.parquet.table.metadata.ColumnTypeInfo;
 import io.deephaven.parquet.table.metadata.TableInfo;
@@ -19,6 +18,8 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -26,10 +27,12 @@
 import java.util.List;
 import java.util.Map;
 
+import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
 /**
- * Used to generate a combined _metadata and _common_metadata file for provided Parquet files.
+ * Used to generate a combined {@value ParquetUtils#METADATA_FILE_NAME} and
+ * {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file for provided Parquet files.
  */
 final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
 
@@ -46,47 +49,50 @@ private static class ParquetFileMetadata {
         }
     }
 
-    private final String metadataRootDirAbsPath;
+    private final Path metadataRootDirAbsPath;
     private final List<ParquetFileMetadata> parquetFileMetadataList;
     private final SeekableChannelsProvider channelsProvider;
-    private final MessageType commonSchema;
+    private final MessageType partitioningColumnsSchema;
 
     // The following fields are used to accumulate metadata for all parquet files
     private MessageType mergedSchema;
     private String mergedCreatedByString;
     private final Map<String, String> mergedKeyValueMetaData;
     private final List<BlockMetaData> mergedBlocks;
-
     /**
      * Per-column type information stored in key-value metadata
      */
-    private List<ColumnTypeInfo> columnTypes;
+    private List<ColumnTypeInfo> mergedColumnTypes;
+    private String mergedVersion;
 
     /**
      * @param metadataRootDir The root directory for the metadata files
-     * @param destinations The indivdual parquet file destinations, all of which must be contained in the metadata root
-     * @param commonSchema The common schema to be included for writing the _common_metadata file.
+     * @param destinations The individual parquet file destinations, all of which must be contained in the metadata root
+     * @param partitioningColumnsSchema The common schema for partitioning columns to be included in the
+     *        {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file, can be null if there are no partitioning columns.
      */
     ParquetMetadataFileWriterImpl(@NotNull final File metadataRootDir, @NotNull final File[] destinations,
-            @Nullable final MessageType commonSchema) {
-        this.metadataRootDirAbsPath = metadataRootDir.getAbsolutePath();
+            @Nullable final MessageType partitioningColumnsSchema) {
+        this.metadataRootDirAbsPath = metadataRootDir.getAbsoluteFile().toPath();
+        final String metadataRootDirAbsPathString = metadataRootDirAbsPath.toString();
         for (final File destination : destinations) {
-            if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPath)) {
-                throw new UncheckedDeephavenException("All destinations must be contained in the provided metadata root"
-                        + " directory, provided destination " + destination.getAbsolutePath() + " is not in " +
-                        metadataRootDirAbsPath);
+            if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPathString)) {
+                throw new UncheckedDeephavenException("All destinations must be nested under the provided metadata root"
+                        + " directory, provided destination " + destination.getAbsolutePath() + " is not under " +
+                        metadataRootDirAbsPathString);
             }
         }
         this.parquetFileMetadataList = new ArrayList<>(destinations.length);
         this.channelsProvider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader(
-                convertToURI(metadataRootDirAbsPath), null);
-        this.columnTypes = null;
-        this.commonSchema = commonSchema;
+                convertToURI(metadataRootDirAbsPathString), null);
+        this.partitioningColumnsSchema = partitioningColumnsSchema;
 
         this.mergedSchema = null;
         this.mergedCreatedByString = null;
         this.mergedKeyValueMetaData = new HashMap<>();
         this.mergedBlocks = new ArrayList<>();
+        this.mergedColumnTypes = null;
+        this.mergedVersion = null;
     }
 
     /**
@@ -102,23 +108,22 @@ public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata
     /**
      * Write the accumulated metadata to the provided files and clear the metadata accumulated so far.
      *
-     * @param metadataFile The destination file for the _metadata file
-     * @param commonMetadataFile The destination file for the _common_metadata file
+     * @param metadataFile The destination for the {@value ParquetUtils#METADATA_FILE_NAME} file
+     * @param commonMetadataFile The destination for the {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file
      */
     public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) throws IOException {
         if (parquetFileMetadataList.isEmpty()) {
             throw new UncheckedDeephavenException("No parquet files to write metadata for");
         }
         mergeMetadata();
-        final ParquetMetadata metadataFooter =
-                new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, mergedCreatedByString),
-                        mergedBlocks);
+        final ParquetMetadata metadataFooter = new ParquetMetadata(new FileMetaData(mergedSchema,
+                mergedKeyValueMetaData, mergedCreatedByString), mergedBlocks);
         writeMetadataFile(metadataFooter, metadataFile.getAbsolutePath());
 
-        // Skip the blocks data and merge schema with the common schema to write the common metadata file
-        // The ordering of arguments in method call is important because we want common schema to determine the overall
-        // ordering of the schema fields.
-        mergedSchema = mergeSchemaInto(mergedSchema, commonSchema);
+        // Skip the blocks data and merge schema with partitioning columns' schema to write the common metadata file.
+        // The ordering of arguments in method call is important because we want to keep partitioning columns in the
+        // beginning.
+        mergedSchema = mergeSchemaInto(mergedSchema, partitioningColumnsSchema);
         final ParquetMetadata commonMetadataFooter =
                 new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, mergedCreatedByString),
                         new ArrayList<>());
@@ -136,10 +141,16 @@ private void mergeMetadata() throws IOException {
         for (final ParquetFileMetadata parquetFileMetadata : parquetFileMetadataList) {
             final FileMetaData fileMetaData = parquetFileMetadata.metadata.getFileMetaData();
             mergedSchema = mergeSchemaInto(fileMetaData.getSchema(), mergedSchema);
-            mergeKeyValueMetaDataInto(fileMetaData.getKeyValueMetaData(), mergedKeyValueMetaData);
+            mergeKeyValueMetaData(fileMetaData.getKeyValueMetaData());
             mergeBlocksInto(parquetFileMetadata, metadataRootDirAbsPath, mergedBlocks);
             mergedCreatedBy.add(fileMetaData.getCreatedBy());
         }
+        // Generate the merged deephaven-specific metadata
+        final TableInfo.Builder tableInfoBuilder = TableInfo.builder().addAllColumnTypes(mergedColumnTypes);
+        if (mergedVersion != null) {
+            tableInfoBuilder.version(mergedVersion);
+        }
+        mergedKeyValueMetaData.put(ParquetTableWriter.METADATA_KEY, tableInfoBuilder.build().serializeToJSON());
         mergedCreatedByString =
                 mergedCreatedBy.size() == 1 ? mergedCreatedBy.iterator().next() : mergedCreatedBy.toString();
     }
@@ -158,64 +169,71 @@ private static MessageType mergeSchemaInto(final MessageType schema, final Messa
         return mergedSchema.union(schema, true);
     }
 
-    private void mergeKeyValueMetaDataInto(final Map<String, String> keyValueMetaData,
-            final Map<String, String> mergedKeyValueMetaData) throws IOException {
+    /**
+     * Merge the non-deephaven-specific key-value metadata for a file into the merged metadata. For deephaven specific
+     * fields, we filter the relevant fields here and merge them later once all the files have been processed.
+     */
+    private void mergeKeyValueMetaData(final Map<String, String> keyValueMetaData) throws IOException {
         for (final Map.Entry<String, String> entry : keyValueMetaData.entrySet()) {
             if (!entry.getKey().equals(ParquetTableWriter.METADATA_KEY)) {
                 // We should only have one value for each key
-                if (!mergedKeyValueMetaData.containsKey(entry.getKey())) {
-                    mergedKeyValueMetaData.put(entry.getKey(), entry.getValue());
-                } else if (!mergedKeyValueMetaData.get(entry.getKey()).equals(entry.getValue())) {
-                    throw new UncheckedDeephavenException("Could not merge metadata for key " + entry.getKey() +
-                            ", has conflicting values: " + entry.getValue() + " and "
-                            + mergedKeyValueMetaData.get(entry.getKey()));
-                }
+                mergedKeyValueMetaData.compute(entry.getKey(), (k, v) -> {
+                    if (v == null) {
+                        // No existing value for this key, so put the new value
+                        return entry.getValue();
+                    } else if (!v.equals(entry.getValue())) {
+                        // Existing value does not match the new value
+                        throw new UncheckedDeephavenException("Could not merge metadata for key " + entry.getKey() +
+                                ", has conflicting values: " + entry.getValue() + " and " + v);
+                    }
+                    // Existing value matches the new value, no action needed
+                    return v;
+                });
             } else {
                 // For merging deephaven-specific metadata,
                 // - groupingColumns, dataIndexes are skipped
-                // - version is optional, so we read it from the first file's metadata
                 // - columnTypes must be the same for all partitions
+                // - version is set as non-null if all the files have the same version
                 final TableInfo tableInfo = TableInfo.deserializeFromJSON(entry.getValue());
-                if (!mergedKeyValueMetaData.containsKey(ParquetTableWriter.METADATA_KEY)) {
-                    // First time we've seen deephaven specific metadata
-                    Assert.eqNull(columnTypes, "columnTypes");
-                    columnTypes = tableInfo.columnTypes();
-                    mergedKeyValueMetaData.put(ParquetTableWriter.METADATA_KEY,
-                            TableInfo.builder()
-                                    .addAllColumnTypes(columnTypes)
-                                    .version(tableInfo.version())
-                                    .build()
-                                    .serializeToJSON());
-                } else if (!columnTypes.equals(tableInfo.columnTypes())) {
-                    throw new UncheckedDeephavenException("Could not merge metadata for key " +
-                            ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
-                            entry.getValue() + " and " + mergedKeyValueMetaData.get(entry.getKey()));
+                if (mergedColumnTypes == null) {
+                    // First time we've seen deephaven specific metadata, so just copy the relevant fields
+                    mergedColumnTypes = tableInfo.columnTypes();
+                    mergedVersion = tableInfo.version();
+                } else {
+                    if (!mergedColumnTypes.equals(tableInfo.columnTypes())) {
+                        throw new UncheckedDeephavenException("Could not merge metadata for key " +
+                                ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
+                                tableInfo.columnTypes() + " and " + mergedColumnTypes);
+                    }
+                    if (!tableInfo.version().equals(mergedVersion)) {
+                        mergedVersion = null;
+                    }
                 }
             }
         }
     }
 
     private static void mergeBlocksInto(final ParquetFileMetadata parquetFileMetadata,
-            final String metadataRootDirAbsPath,
-            final List<BlockMetaData> mergedBlocks) {
-        final String fileAbsolutePath = parquetFileMetadata.file.getAbsolutePath();
-        String fileRelativePath = fileAbsolutePath.substring(metadataRootDirAbsPath.length());
+            final Path metadataRootDirAbsPath, final Collection<BlockMetaData> mergedBlocks) {
+        final Path parquetFileAbsPath = Paths.get(parquetFileMetadata.file.getAbsolutePath());
+        String fileRelativePathString = metadataRootDirAbsPath.relativize(parquetFileAbsPath).toString();
         // Remove leading slashes from the relative path
         int pos = 0;
-        while (pos < fileRelativePath.length() && fileRelativePath.charAt(pos) == '/') {
+        while (pos < fileRelativePathString.length() && fileRelativePathString.charAt(pos) == '/') {
             pos++;
         }
-        fileRelativePath = fileRelativePath.substring(pos);
+        fileRelativePathString = fileRelativePathString.substring(pos);
         for (final BlockMetaData block : parquetFileMetadata.metadata.getBlocks()) {
-            block.setPath(fileRelativePath);
+            block.setPath(fileRelativePathString);
             mergedBlocks.add(block);
         }
     }
 
     private void writeMetadataFile(final ParquetMetadata metadataFooter, final String outputPath) throws IOException {
         final PositionedBufferedOutputStream metadataOutputStream =
-                new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(outputPath, false));
-        metadataOutputStream.write(ParquetFileReader.MAGIC);
+                new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(outputPath, false),
+                        ParquetUtils.PARQUET_OUTPUT_BUFFER_SIZE);
+        metadataOutputStream.write(MAGIC);
         ParquetFileWriter.serializeFooter(metadataFooter, metadataOutputStream);
         metadataOutputStream.close();
     }
@@ -224,7 +242,7 @@ public void clear() {
         parquetFileMetadataList.clear();
         mergedKeyValueMetaData.clear();
         mergedBlocks.clear();
-        columnTypes = null;
+        mergedColumnTypes = null;
         mergedSchema = null;
         mergedCreatedByString = null;
     }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index f9be6badce7..488b31ab1ec 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -21,6 +21,7 @@
 import io.deephaven.parquet.base.NullParquetMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetFileWriter;
+import io.deephaven.parquet.base.ParquetUtils;
 import io.deephaven.parquet.base.RowGroupWriter;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
 import io.deephaven.parquet.table.metadata.CodecInfo;
@@ -46,6 +47,7 @@
 import java.io.IOException;
 import java.nio.*;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.*;
 
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
@@ -54,7 +56,7 @@
  * API for writing DH tables in parquet format
  */
 public class ParquetTableWriter {
-    public static final String METADATA_KEY = "deephaven";
+    static final String METADATA_KEY = "deephaven";
     public static final String BEGIN_POS = "dh_begin_pos";
     public static final String END_POS = "dh_end_pos";
     public static final String GROUPING_KEY = "dh_key";
@@ -92,14 +94,15 @@ static class GroupingColumnWritingInfo {
      * @param t The table to write
      * @param definition Table definition
      * @param writeInstructions Write instructions for customizations while writing
-     * @param destFile The destination to write the parquet file
+     * @param destPathName The destination path
      * @param metadataFilePath The destination path to store in the metadata files. This can be different from
      *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
      *        always hold the accurate path.
      * @param incomingMeta A map of metadata values to be stores in the file footer
      * @param groupingColumnsWritingInfoMap A map of grouping column names to their respective info used for writing
      *        grouping files
-     * @param metadataFileWriter The writer for the _metadata and _common_metadata files
+     * @param metadataFileWriter The writer for the {@value ParquetUtils#METADATA_FILE_NAME} and
+     *        {@value ParquetUtils#COMMON_METADATA_FILE_NAME} files
      * @param computedCache When we need to perform some computation depending on column data to make a decision
      *        impacting both schema and written data, we store results in computedCache to avoid having to calculate
      *        twice. An example is the necessary precision and scale for a BigDecimal column written as a decimal
@@ -111,7 +114,7 @@ static void write(
             @NotNull final Table t,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
-            @NotNull final File destFile,
+            @NotNull final String destPathName,
             @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> incomingMeta,
             @Nullable final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap,
@@ -123,7 +126,7 @@ static void write(
         try {
             if (groupingColumnsWritingInfoMap != null) {
                 cleanupFiles = new ArrayList<>(groupingColumnsWritingInfoMap.size());
-                final Path destDirPath = destFile.toPath().getParent();
+                final Path destDirPath = Paths.get(destPathName).getParent();
                 for (final Map.Entry<String, GroupingColumnWritingInfo> entry : groupingColumnsWritingInfoMap
                         .entrySet()) {
                     final String groupingColumnName = entry.getKey();
@@ -135,12 +138,13 @@ static void write(
                     tableInfoBuilder.addGroupingColumns(GroupingColumnInfo.of(parquetColumnName,
                             destDirPath.relativize(metadataGroupingFilePath.toPath()).toString()));
                     // We don't accumulate metadata from grouping files into the main metadata file
-                    write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions, groupingDestFile,
+                    write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions,
+                            groupingDestFile.getAbsolutePath(),
                             metadataGroupingFilePath, Collections.emptyMap(), TableInfo.builder(),
                             NullParquetMetadataFileWriter.INSTANCE, computedCache);
                 }
             }
-            write(t, definition, writeInstructions, destFile, metadataFilePath, incomingMeta, tableInfoBuilder,
+            write(t, definition, writeInstructions, destPathName, metadataFilePath, incomingMeta, tableInfoBuilder,
                     metadataFileWriter, computedCache);
         } catch (Exception e) {
             if (cleanupFiles != null) {
@@ -162,13 +166,14 @@ static void write(
      * @param table The table to write
      * @param definition The table definition
      * @param writeInstructions Write instructions for customizations while writing
-     * @param destFile The destination path
+     * @param path The destination path
      * @param metadataFilePath The destination path to store in the metadata files. This can be different from
      *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
      *        always hold the accurate path.
      * @param tableMeta A map of metadata values to be stores in the file footer
      * @param tableInfoBuilder A partially constructed builder for the metadata object
-     * @param metadataFileWriter The writer for the _metadata and _common_metadata files
+     * @param metadataFileWriter The writer for the {@value ParquetUtils#METADATA_FILE_NAME} and
+     *        {@value ParquetUtils#COMMON_METADATA_FILE_NAME} files
      * @param computedCache Per column cache tags
      * @throws IOException For file writing related errors
      */
@@ -176,7 +181,7 @@ static void write(
             @NotNull final Table table,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
-            @NotNull final File destFile,
+            @NotNull final String path,
             @NotNull final File metadataFilePath,
             @NotNull final Map<String, String> tableMeta,
             @NotNull final TableInfo.Builder tableInfoBuilder,
@@ -187,7 +192,7 @@ static void write(
             final TrackingRowSet tableRowSet = t.getRowSet();
             final Map<String, ? extends ColumnSource<?>> columnSourceMap = t.getColumnSourceMap();
             final ParquetFileWriter parquetFileWriter = getParquetFileWriter(computedCache, definition, tableRowSet,
-                    columnSourceMap, destFile, metadataFilePath, writeInstructions, tableMeta, tableInfoBuilder,
+                    columnSourceMap, path, metadataFilePath, writeInstructions, tableMeta, tableInfoBuilder,
                     metadataFileWriter);
             // Given the transformation, do not use the original table's "definition" for writing
             write(t, writeInstructions, parquetFileWriter, computedCache);
@@ -295,14 +300,15 @@ private static Table pretransformTable(@NotNull final Table table, @NotNull fina
      * @param definition the writable definition
      * @param tableRowSet the row set being written
      * @param columnSourceMap the columns of the table
-     * @param destFile the destination to write to
+     * @param path the destination to write to
      * @param metadataFilePath The destination path to store in the metadata files. This can be different from
      *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
      *        always hold the accurate path.
      * @param writeInstructions write instructions for the file
      * @param tableMeta metadata to include in the parquet metadata
      * @param tableInfoBuilder a builder for accumulating per-column information to construct the deephaven metadata
-     * @param metadataFileWriter The writer for the _metadata and _common_metadata files
+     * @param metadataFileWriter The writer for the {@value ParquetUtils#METADATA_FILE_NAME} and
+     *        {@value ParquetUtils#COMMON_METADATA_FILE_NAME} files
      *
      * @return a new file writer
      */
@@ -312,7 +318,7 @@ private static ParquetFileWriter getParquetFileWriter(
             @NotNull final TableDefinition definition,
             @NotNull final RowSet tableRowSet,
             @NotNull final Map<String, ? extends ColumnSource<?>> columnSourceMap,
-            @NotNull final File destFile,
+            @NotNull final String path,
             @NotNull final File metadataFilePath,
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final Map<String, String> tableMeta,
@@ -359,8 +365,8 @@ private static ParquetFileWriter getParquetFileWriter(
 
         final Map<String, String> extraMetaData = new HashMap<>(tableMeta);
         extraMetaData.put(METADATA_KEY, tableInfoBuilder.build().serializeToJSON());
-        return new ParquetFileWriter(destFile, metadataFilePath,
-                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(destFile.getPath()), null),
+        return new ParquetFileWriter(path, metadataFilePath,
+                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(path), null),
                 writeInstructions.getTargetPageSize(),
                 new HeapByteBufferAllocator(), mappedSchema.getParquetSchema(),
                 writeInstructions.getCompressionCodecName(), extraMetaData, metadataFileWriter);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 7104a4ecf70..2ded0e1e719 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -13,8 +13,10 @@
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.PartitionedTable;
+import io.deephaven.engine.table.PartitionedTableFactory;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
+import io.deephaven.engine.table.impl.locations.util.PartitionFormatter;
 import io.deephaven.engine.table.impl.locations.util.TableDataRefreshService;
 import io.deephaven.engine.updategraph.UpdateSourceRegistrar;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
@@ -60,13 +62,14 @@
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
 import static io.deephaven.parquet.table.ParquetTableWriter.getSchemaForTable;
-import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
-import static io.deephaven.parquet.table.ParquetUtils.COMMON_METADATA_FILE_NAME;
-import static io.deephaven.parquet.table.ParquetUtils.METADATA_FILE_NAME;
+import static io.deephaven.parquet.base.ParquetUtils.PARQUET_FILE_EXTENSION;
+import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME;
+import static io.deephaven.parquet.base.ParquetUtils.METADATA_FILE_NAME;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 import static io.deephaven.util.type.TypeUtils.getUnboxedTypeIfBoxed;
 
@@ -537,7 +540,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
         final PartitionedTable partitionedTable = sourceTable.partitionBy(partitioningColNames);
         final TableDefinition keyTableDefinition = TableDefinition.of(partitioningColumns);
         final TableDefinition leafDefinition =
-                getNonKeyTableDefinition(Arrays.asList(partitioningColNames), definition);
+                getNonKeyTableDefinition(new HashSet<>(Arrays.asList(partitioningColNames)), definition);
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
                 baseName, writeInstructions);
     }
@@ -629,7 +632,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
     private static void writeKeyValuePartitionedTableImpl(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition keyTableDefinition,
             @NotNull final TableDefinition leafDefinition,
-            @NotNull final File destinationDir,
+            @NotNull final File destinationRoot,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         if (leafDefinition.numColumns() == 0) {
@@ -655,10 +658,12 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
             partitionStringsList.add(new ArrayList<>(partitioningColumnNames.length));
         }
         Arrays.stream(partitioningColumnNames).forEach(columnName -> {
+            final PartitionFormatter partitionFormatter = PartitionFormatter.getFormatterForType(
+                    withGroupConstituents.getColumnSource(columnName).getType());
             try (final CloseableIterator<?> valueIterator = withGroupConstituents.columnIterator(columnName)) {
                 int row = 0;
                 while (valueIterator.hasNext()) {
-                    final String partitioningValue = PartitionFormatter.formatToString(valueIterator.next());
+                    final String partitioningValue = partitionFormatter.format(valueIterator.next());
                     partitionStringsList.get(row).add(columnName + "=" + partitioningValue);
                     row++;
                 }
@@ -673,7 +678,7 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
             while (constituentIterator.hasNext()) {
                 final ObjectVector<? extends Table> constituentVector = constituentIterator.next();
                 final List<String> partitionStrings = partitionStringsList.get(row);
-                final File relativePath = new File(destinationDir, String.join("/", partitionStrings));
+                final File relativePath = new File(destinationRoot, String.join("/", partitionStrings));
                 int count = 0;
                 for (final Table constituent : constituentVector) {
                     String filename = baseName;
@@ -694,24 +699,25 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
                 row++;
             }
         }
-        final MessageType commonSchema;
+        final MessageType partitioningColumnsSchema;
         if (writeInstructions.generateMetadataFiles()) {
-            // Generate initial schema for _common_metadata file from key columns. The schema for remaining columns will
+            // Generate schema for partitioning columns for _common_metadata file. The schema for remaining columns will
             // be inferred at the time of writing the parquet files and merged with the common schema.
-            commonSchema = getSchemaForTable(partitionedTable.table(), keyTableDefinition, writeInstructions);
+            partitioningColumnsSchema =
+                    getSchemaForTable(partitionedTable.table(), keyTableDefinition, writeInstructions);
         } else {
-            commonSchema = null;
+            partitioningColumnsSchema = null;
         }
         final Map<String, Map<ParquetCacheTags, Object>> computedCache =
-                buildComputedCache(partitionedTable, leafDefinition);
+                buildComputedCache(partitionedTable::merge, leafDefinition);
         ParquetTools.writeParquetTablesImpl(
                 partitionedData.toArray(Table[]::new),
                 leafDefinition,
                 writeInstructions,
                 destinations.toArray(File[]::new),
                 leafDefinition.getGroupingColumnNamesArray(),
-                commonSchema,
-                destinationDir,
+                partitioningColumnsSchema,
+                destinationRoot,
                 computedCache);
     }
 
@@ -743,23 +749,25 @@ private static TableDefinition getNonKeyTableDefinition(@NotNull final Collectio
     }
 
     /**
-     * Precompute the precision and scale values for big decimal columns and pass it to the write method so that all the
-     * constituent parquet files are written with the same precision and scale values, and thus have the same schema.
+     * If the definition has any big decimal columns, precompute the precision and scale values for big decimal columns
+     * for the merged table so that all the constituent parquet files are written with the same schema, precision and
+     * scale values. We only need to perform the merge operation if there is a big decimal column in the definition.
+     * That is why this method accepts a supplier instead of the table itself.
      */
     private static Map<String, Map<ParquetCacheTags, Object>> buildComputedCache(
-            @NotNull final PartitionedTable partitionedTable,
-            @NotNull final TableDefinition leafDefinition) {
+            @NotNull final Supplier<Table> mergedTableSupplier,
+            @NotNull final TableDefinition definition) {
         final Map<String, Map<ParquetCacheTags, Object>> computedCache = new HashMap<>();
-        Table mergedConstituents = null;
-        final List<ColumnDefinition<?>> leafColumnDefinitions = leafDefinition.getColumns();
+        Table mergedTable = null;
+        final List<ColumnDefinition<?>> leafColumnDefinitions = definition.getColumns();
         for (final ColumnDefinition<?> columnDefinition : leafColumnDefinitions) {
             if (columnDefinition.getDataType() == BigDecimal.class) {
-                if (mergedConstituents == null) {
-                    mergedConstituents = partitionedTable.merge();
+                if (mergedTable == null) {
+                    mergedTable = mergedTableSupplier.get();
                 }
                 final String columnName = columnDefinition.getName();
-                final ColumnSource<BigDecimal> bigDecimalColumnSource = mergedConstituents.getColumnSource(columnName);
-                TypeInfos.getPrecisionAndScale(computedCache, columnName, mergedConstituents.getRowSet(),
+                final ColumnSource<BigDecimal> bigDecimalColumnSource = mergedTable.getColumnSource(columnName);
+                TypeInfos.getPrecisionAndScale(computedCache, columnName, mergedTable.getRowSet(),
                         () -> bigDecimalColumnSource);
             }
         }
@@ -775,8 +783,8 @@ private static Map<String, Map<ParquetCacheTags, Object>> buildComputedCache(
      * @param definition The common definition for all the tables to write
      * @param writeInstructions Write instructions for customizations while writing
      * @param destinations The destination paths. Any non-existing directories in the paths provided are created. If
-     *        there is an error any intermediate directories previously created are removed; note this makes this method
-     *        unsafe for concurrent use
+     *        there is an error, any intermediate directories previously created are removed; note this makes this
+     *        method unsafe for concurrent use
      * @param groupingColumns List of columns the tables are grouped by (the write operation will store the grouping
      *        info)
      */
@@ -787,8 +795,8 @@ public static void writeParquetTables(@NotNull final Table[] sources,
             @Nullable final String[] groupingColumns) {
         final File metadataRootDir;
         if (writeInstructions.generateMetadataFiles()) {
-            // We will write the metadata file in the same directory as the destination files, so we need to ensure that
-            // all destination files are in the same directory.
+            // We insist on writing the metadata file in the same directory as the destination files, thus all
+            // destination files are in the same directory.
             final String firstDestinationDir = destinations[0].getParentFile().getAbsolutePath();
             for (int i = 1; i < destinations.length; i++) {
                 if (!firstDestinationDir.equals(destinations[i].getParentFile().getAbsolutePath())) {
@@ -800,10 +808,13 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         } else {
             metadataRootDir = null;
         }
+
+        final Map<String, Map<ParquetCacheTags, Object>> computedCache =
+                buildComputedCache(() -> PartitionedTableFactory.ofTables(definition, sources).merge(), definition);
         // We do not have any additional common schema here. Schema for all columns will be generated at the time of
         // writing the parquet files and merged to generate the metadata files.
         writeParquetTablesImpl(sources, definition, writeInstructions, destinations, groupingColumns,
-                null, metadataRootDir, new HashMap<>());
+                null, metadataRootDir, computedCache);
     }
 
     /**
@@ -815,7 +826,7 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final File[] destinations,
             @Nullable final String[] groupingColumns,
-            @Nullable final MessageType commonSchema,
+            @Nullable final MessageType partitioningColumnsSchema,
             @Nullable final File metadataRootDir,
             @NotNull final Map<String, Map<ParquetCacheTags, Object>> computedCache) {
         Require.eq(sources.length, "sources.length", destinations.length, "destinations.length");
@@ -836,7 +847,8 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
             if (metadataRootDir == null) {
                 throw new IllegalArgumentException("Metadata root directory must be set when writing metadata files");
             }
-            metadataFileWriter = new ParquetMetadataFileWriterImpl(metadataRootDir, destinations, commonSchema);
+            metadataFileWriter =
+                    new ParquetMetadataFileWriterImpl(metadataRootDir, destinations, partitioningColumnsSchema);
         } else {
             metadataFileWriter = NullParquetMetadataFileWriter.INSTANCE;
         }
@@ -853,7 +865,7 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
                 for (int tableIdx = 0; tableIdx < sources.length; tableIdx++) {
                     shadowFiles.add(shadowDestFiles[tableIdx]);
                     final Table source = sources[tableIdx];
-                    ParquetTableWriter.write(source, definition, writeInstructions, shadowDestFiles[tableIdx],
+                    ParquetTableWriter.write(source, definition, writeInstructions, shadowDestFiles[tableIdx].getPath(),
                             destinations[tableIdx], Collections.emptyMap(),
                             (Map<String, ParquetTableWriter.GroupingColumnWritingInfo>) null, metadataFileWriter,
                             computedCache);
@@ -877,9 +889,9 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
                     groupingColumnWritingInfoMap.values().forEach(gcwi -> shadowFiles.add(gcwi.destFile));
 
                     final Table sourceTable = sources[tableIdx];
-                    ParquetTableWriter.write(sourceTable, definition, writeInstructions, shadowDestFiles[tableIdx],
-                            tableDestination, Collections.emptyMap(), groupingColumnWritingInfoMap, metadataFileWriter,
-                            computedCache);
+                    ParquetTableWriter.write(sourceTable, definition, writeInstructions,
+                            shadowDestFiles[tableIdx].getPath(), tableDestination, Collections.emptyMap(),
+                            groupingColumnWritingInfoMap, metadataFileWriter, computedCache);
                 }
             }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
deleted file mode 100644
index fef23a696a1..00000000000
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/PartitionFormatter.java
+++ /dev/null
@@ -1,132 +0,0 @@
-package io.deephaven.parquet.table;
-
-import io.deephaven.time.DateTimeUtils;
-import org.jetbrains.annotations.NotNull;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalTime;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * This class takes an object read from a parquet file and formats it to a string used for generate partitioning column
- * values.
- */
-enum PartitionFormatter {
-    ForString {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return (String) obj;
-        }
-    },
-    ForBoolean {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Boolean) obj).toString();
-        }
-    },
-    ForChar {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Character) obj).toString();
-        }
-    },
-    ForByte {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Byte) obj).toString();
-        }
-    },
-    ForShort {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Short) obj).toString();
-        }
-    },
-    ForInt {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Integer) obj).toString();
-        }
-    },
-    ForLong {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Long) obj).toString();
-        }
-    },
-    ForFloat {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Float) obj).toString();
-        }
-    },
-    ForDouble {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Double) obj).toString();
-        }
-    },
-    ForBigInteger {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((BigInteger) obj).toString();
-        }
-    },
-    ForBigDecimal {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((BigDecimal) obj).toString();
-        }
-    },
-    ForInstant {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((Instant) obj).toString();
-        }
-    },
-    ForLocalDate {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return DateTimeUtils.formatDate((LocalDate) obj);
-        }
-    },
-    ForLocalTime {
-        @Override
-        public String format(@NotNull final Object obj) {
-            return ((LocalTime) obj).toString();
-        }
-    };
-
-    private static final Map<Class<?>, PartitionFormatter> typeMap = new HashMap<>();
-    static {
-        typeMap.put(String.class, ForString);
-        typeMap.put(Boolean.class, ForBoolean);
-        typeMap.put(Character.class, ForChar);
-        typeMap.put(Byte.class, ForByte);
-        typeMap.put(Short.class, ForShort);
-        typeMap.put(Integer.class, ForInt);
-        typeMap.put(Long.class, ForLong);
-        typeMap.put(Float.class, ForFloat);
-        typeMap.put(Double.class, ForDouble);
-        typeMap.put(BigInteger.class, ForBigInteger);
-        typeMap.put(BigDecimal.class, ForBigDecimal);
-        typeMap.put(Instant.class, ForInstant);
-        typeMap.put(LocalDate.class, ForLocalDate);
-        typeMap.put(LocalTime.class, ForLocalTime);
-    }
-
-    abstract String format(@NotNull Object obj);
-
-    static String formatToString(@NotNull final Object obj) {
-        final PartitionFormatter formatter = typeMap.get(obj.getClass());
-        if (formatter != null) {
-            return formatter.format(obj);
-        } else {
-            throw new UnsupportedOperationException("Unsupported type: " + obj.getClass().getSimpleName());
-        }
-    }
-}
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java
index 6e2ef6c000c..b8d51a38409 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java
@@ -7,7 +7,7 @@
 import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder;
 import io.deephaven.parquet.table.ParquetInstructions;
 import io.deephaven.parquet.table.location.ParquetTableLocationKey;
-import io.deephaven.parquet.table.ParquetUtils;
+import io.deephaven.parquet.base.ParquetUtils;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java
index 2404a69fcd2..102cb73aff1 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java
@@ -10,7 +10,7 @@
 import io.deephaven.engine.table.impl.locations.local.LocationTableBuilderDefinition;
 import io.deephaven.parquet.table.ParquetInstructions;
 import io.deephaven.parquet.table.location.ParquetTableLocationKey;
-import io.deephaven.parquet.table.ParquetUtils;
+import io.deephaven.parquet.base.ParquetUtils;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
index 75ffb95913e..f23f56aede3 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
@@ -12,6 +12,7 @@
 import io.deephaven.parquet.table.ParquetTools;
 import io.deephaven.engine.table.impl.locations.TableDataException;
 import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder;
+import io.deephaven.parquet.base.ParquetUtils;
 import io.deephaven.parquet.table.location.ParquetTableLocationKey;
 import io.deephaven.parquet.table.ParquetInstructions;
 import io.deephaven.parquet.base.ParquetFileReader;
@@ -36,8 +37,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static io.deephaven.parquet.table.ParquetUtils.COMMON_METADATA_FILE_NAME;
-import static io.deephaven.parquet.table.ParquetUtils.METADATA_FILE_NAME;
+import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME;
+import static io.deephaven.parquet.base.ParquetUtils.METADATA_FILE_NAME;
 import static java.util.stream.Collectors.toMap;
 
 /**
@@ -47,12 +48,13 @@
  * <p>
  * Note that we expect to find the following files:
  * <ul>
- * <li>{@code _metadata} - A file containing Parquet metadata for all {@link RowGroup row groups} in all
- * {@code .parquet} files for the entire data set, including schema information non-partitioning columns and key-value
- * metadata</li>
- * <li>{@code _common_metadata} <i>(optional)</i> - A file containing Parquet metadata with schema information that
- * applies to the entire data set, including partitioning columns that are inferred from file paths rather than
- * explicitly written in {@link org.apache.parquet.format.ColumnChunk column chunks} within {@code .parquet} files</li>
+ * <li>{@value ParquetUtils#METADATA_FILE_NAME} - A file containing Parquet metadata for all {@link RowGroup row groups}
+ * in all {@code .parquet} files for the entire data set, including schema information non-partitioning columns and
+ * key-value metadata</li>
+ * <li>{@value ParquetUtils#COMMON_METADATA_FILE_NAME} <i>(optional)</i> - A file containing Parquet metadata with
+ * schema information that applies to the entire data set, including partitioning columns that are inferred from file
+ * paths rather than explicitly written in {@link org.apache.parquet.format.ColumnChunk column chunks} within
+ * {@code .parquet} files</li>
  * </ul>
  */
 public class ParquetMetadataFileLayout implements TableLocationKeyFinder<ParquetTableLocationKey> {
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
index cdc6d3b5a02..12c50f720fb 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
@@ -22,7 +22,7 @@
 import java.util.Map;
 import java.util.stream.IntStream;
 
-import static io.deephaven.parquet.table.ParquetUtils.PARQUET_FILE_EXTENSION;
+import static io.deephaven.parquet.base.ParquetUtils.PARQUET_FILE_EXTENSION;
 
 /**
  * {@link TableLocationKey} implementation for use with data stored in the parquet format.
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 5682a4f74a2..b6cf0baa55f 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -573,6 +573,34 @@ public void flatPartitionedParquetWithMetadataTest() throws IOException {
         }
     }
 
+    @Test
+    public void flatPartitionedParquetWithBigDecimalMetadataTest() throws IOException {
+        final File parentDir = new File(rootFile, "tempDir");
+
+        // Both tables have different precision for big decimal column
+        final Table firstTable = TableTools.emptyTable(5).update("bdColumn = java.math.BigDecimal.valueOf((double)ii)");
+        final Table secondTable =
+                TableTools.emptyTable(5).update("bdColumn = java.math.BigDecimal.valueOf((double)(ii*0.001))");
+        final File firstDataFile = new File(parentDir, "data1.parquet");
+        final File secondDataFile = new File(parentDir, "data2.parquet");
+
+        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+                .setGenerateMetadataFiles(true)
+                .build();
+        final Table[] sources = new Table[] {firstTable, secondTable};
+        writeParquetTables(sources, firstTable.getDefinition(), writeInstructions,
+                new File[] {firstDataFile, secondDataFile}, null);
+
+        // Merge the tables and compute the precision and scale as per the union of the two tables
+        final Table expected =
+                maybeFixBigDecimal(PartitionedTableFactory.ofTables(firstTable.getDefinition(), sources).merge());
+
+        final Table fromDisk = readTable(parentDir).select();
+        assertTableEquals(expected, fromDisk);
+        final Table fromDiskWithMetadata = readTable(new File(parentDir, "_metadata"));
+        assertTableEquals(expected, fromDiskWithMetadata);
+    }
+
     @Test
     public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         final TableDefinition definition = TableDefinition.of(
@@ -702,7 +730,7 @@ public void writeKeyValuePartitionedDataWithNullKeys() {
                 ColumnDefinition.ofInt("PC1").withPartitioning(),
                 ColumnDefinition.ofLong("I"));
         final Table inputData = ((QueryTable) TableTools.emptyTable(10)
-                .updateView("PC1 = (ii%2==1)? null : (int)(ii%3)",
+                .updateView("PC1 = (ii%2==0)? null : (int)(ii%2)",
                         "I = ii"))
                 .withDefinitionUnsafe(definition);
         final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
@@ -710,11 +738,11 @@ public void writeKeyValuePartitionedDataWithNullKeys() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        try {
-            writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
-            fail("Expected exception when writing the partitioned table with null keys");
-        } catch (final RuntimeException expected) {
-        }
+        writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
+        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+        assertTableEquals(inputData.sort("PC1"), fromDisk.sort("PC1"));
+        final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata")).select();
+        assertTableEquals(inputData.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
     }
 
     @Test
@@ -836,20 +864,20 @@ public void testAllPartitioningColumnTypes() {
                 ColumnDefinition.ofInt("data"));
 
         final Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView(
-                "PC1 =  (ii%2 == 0) ? `AA` : `BB`",
-                "PC2 = (ii % 2 == 0)",
-                "PC3 = (char)(65 + (ii % 2))",
-                "PC4 = (byte)(ii % 2)",
-                "PC5 = (short)(ii % 2)",
-                "PC6 = (int)(ii%3)",
-                "PC7 = (long)(ii%2)",
-                "PC8 = (float)(ii % 2)",
-                "PC9 = (double)(ii % 2)",
-                "PC10 = java.math.BigInteger.valueOf(ii)",
-                "PC11 = (ii%2 == 0) ? java.math.BigDecimal.valueOf((double)ii) : java.math.BigDecimal.valueOf((double)(ii*0.001))",
-                "PC12 = java.time.Instant.ofEpochSecond(ii)",
-                "PC13 = java.time.LocalDate.ofEpochDay(ii)",
-                "PC14 = java.time.LocalTime.of(i%24, i%60, (i+10)%60)",
+                "PC1 =  (ii%2 == 0) ? null: ((ii%3 == 0) ? `AA` : `BB`)",
+                "PC2 = (ii%2 == 0) ? null : (ii % 3 == 0)",
+                "PC3 = (ii%2 == 0) ? null : (char)(65 + (ii % 2))",
+                "PC4 = (ii%2 == 0) ? null : (byte)(ii % 2)",
+                "PC5 = (ii%2 == 0) ? null : (short)(ii % 2)",
+                "PC6 = (ii%2 == 0) ? null : (int)(ii%3)",
+                "PC7 = (ii%2 == 0) ? null : (long)(ii%2)",
+                "PC8 = (ii%2 == 0) ? null : (float)(ii % 2)",
+                "PC9 = (ii%2 == 0) ? null : (double)(ii % 2)",
+                "PC10 = (ii%2 == 0) ? null : java.math.BigInteger.valueOf(ii)",
+                "PC11 = (ii%2 == 0) ? null : ((ii%3 == 0) ? java.math.BigDecimal.valueOf((double)ii) : java.math.BigDecimal.valueOf((double)(ii*0.001)))",
+                "PC12 = (ii%2 == 0) ? null : java.time.Instant.ofEpochSecond(ii)",
+                "PC13 = (ii%2 == 0) ? null : java.time.LocalDate.ofEpochDay(ii)",
+                "PC14 = (ii%2 == 0) ? null : java.time.LocalTime.of(i%24, i%60, (i+10)%60)",
                 "data = (int)(ii)"))
                 .withDefinitionUnsafe(definition);
 

From 1d98927b30ddcb64f4ea2cc71b35c4170514f89f Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 11 Mar 2024 12:33:55 -0500
Subject: [PATCH 29/45] Review with Ryan part 2

Tests are failing
---
 .../base/NullParquetMetadataFileWriter.java   |  6 +-
 .../parquet/base/ParquetFileWriter.java       | 12 +--
 .../base/ParquetMetadataFileWriter.java       | 10 +--
 .../deephaven/parquet/base/ParquetUtils.java  | 13 +++
 .../table/ParquetMetadataFileWriterImpl.java  | 89 +++++++++++++------
 .../parquet/table/ParquetSchemaReader.java    |  6 +-
 .../parquet/table/ParquetTableWriter.java     | 69 +++++++-------
 .../deephaven/parquet/table/ParquetTools.java |  9 +-
 .../layout/ParquetMetadataFileLayout.java     | 35 +++++++-
 .../table/pagestore/ColumnChunkPageStore.java |  2 +-
 .../OffsetIndexBasedColumnChunkPageStore.java | 86 +++++++++++++-----
 .../table/ParquetTableReadWriteTest.java      |  6 +-
 12 files changed, 233 insertions(+), 110 deletions(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
index 50b8f4fa341..e066df8739e 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/NullParquetMetadataFileWriter.java
@@ -2,8 +2,6 @@
 
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 
-import java.io.File;
-
 /**
  * A no-op implementation of MetadataFileWriterBase when we don't want to write metadata files for Parquet files.
  */
@@ -12,10 +10,10 @@ public enum NullParquetMetadataFileWriter implements ParquetMetadataFileWriter {
     INSTANCE;
 
     @Override
-    public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata metadata) {}
+    public void addParquetFileMetadata(final String parquetFilePath, final ParquetMetadata metadata) {}
 
     @Override
-    public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) {}
+    public void writeMetadataFiles(final String metadataFilePath, final String commonMetadataFilePath) {}
 
     @Override
     public void clear() {}
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
index 071ad0e0355..2d736b43232 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileWriter.java
@@ -41,12 +41,12 @@ public final class ParquetFileWriter {
     private final Map<String, String> extraMetaData;
     private final List<BlockMetaData> blocks = new ArrayList<>();
     private final List<List<OffsetIndex>> offsetIndexes = new ArrayList<>();
-    private final File metadataFilePath;
+    private final String destFilePathForMetadata;
     private final ParquetMetadataFileWriter metadataFileWriter;
 
     public ParquetFileWriter(
-            final String filePath,
-            final File metadataFilePath,
+            final String destFilePath,
+            final String destFilePathForMetadata,
             final SeekableChannelsProvider channelsProvider,
             final int targetPageSize,
             final ByteBufferAllocator allocator,
@@ -57,12 +57,12 @@ public ParquetFileWriter(
         this.targetPageSize = targetPageSize;
         this.allocator = allocator;
         this.extraMetaData = new HashMap<>(extraMetaData);
-        bufferedOutput = new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(filePath, false),
+        bufferedOutput = new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(destFilePath, false),
                 PARQUET_OUTPUT_BUFFER_SIZE);
         bufferedOutput.write(MAGIC);
         this.type = type;
         this.compressorAdapter = DeephavenCompressorAdapterFactory.getInstance().getByName(codecName);
-        this.metadataFilePath = metadataFilePath;
+        this.destFilePathForMetadata = destFilePathForMetadata;
         this.metadataFileWriter = metadataFileWriter;
     }
 
@@ -80,7 +80,7 @@ public void close() throws IOException {
         final ParquetMetadata footer =
                 new ParquetMetadata(new FileMetaData(type, extraMetaData, Version.FULL_VERSION), blocks);
         serializeFooter(footer, bufferedOutput);
-        metadataFileWriter.addParquetFileMetadata(metadataFilePath, footer);
+        metadataFileWriter.addParquetFileMetadata(destFilePathForMetadata, footer);
         // Flush any buffered data and close the channel
         bufferedOutput.close();
         compressorAdapter.close();
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
index 2872c2c3cb6..2576c56d850 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
@@ -15,18 +15,18 @@ public interface ParquetMetadataFileWriter {
      * Add the parquet metadata for the provided parquet file to the list of metadata to be written to combined metadata
      * files.
      *
-     * @param parquetFile The parquet file destination path
+     * @param parquetFilePath The parquet file destination path
      * @param metadata The parquet metadata corresponding to the parquet file
      */
-    void addParquetFileMetadata(File parquetFile, ParquetMetadata metadata);
+    void addParquetFileMetadata(String parquetFilePath, ParquetMetadata metadata);
 
     /**
      * Write the combined metadata files for all metadata accumulated so far and clear the list.
      *
-     * @param metadataFile The destination for the {@value ParquetUtils#METADATA_FILE_NAME} file
-     * @param commonMetadataFile The destination for the {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file
+     * @param metadataFilePath The destination path for the {@value ParquetUtils#METADATA_FILE_NAME} file
+     * @param commonMetadataFilePath The destination path for the {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file
      */
-    void writeMetadataFiles(File metadataFile, File commonMetadataFile) throws IOException;
+    void writeMetadataFiles(String metadataFilePath, String commonMetadataFilePath) throws IOException;
 
     /**
      * Clear the list of metadata accumulated so far.
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
index 8b1e193108e..1c0135db7de 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
@@ -16,6 +16,11 @@ public final class ParquetUtils {
      */
     public static final int PARQUET_OUTPUT_BUFFER_SIZE = 1 << 18;
 
+    /**
+     * Used as a key for storing deephaven specific metadata in the key-value metadata of parquet files.
+     */
+    public static final String METADATA_KEY = "deephaven";
+
     /**
      * Used as a filter to select relevant parquet files while reading all files in a directory.
      */
@@ -23,4 +28,12 @@ public static boolean fileNameMatches(final Path path) {
         final String fileName = path.getFileName().toString();
         return fileName.endsWith(PARQUET_FILE_EXTENSION) && fileName.charAt(0) != '.';
     }
+
+    /**
+     * @return the key value for storing each file's metadata in the common metadata file.
+     */
+    public static String getKeyForFilePath(final Path path) {
+        final String fileName = path.getFileName().toString();
+        return "deephaven_per_file_" + fileName;
+    }
 }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index aaf9b1bf45c..486146e99c3 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -1,6 +1,7 @@
 package io.deephaven.parquet.table;
 
 import io.deephaven.UncheckedDeephavenException;
+import io.deephaven.base.verify.Assert;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetUtils;
@@ -19,7 +20,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -28,6 +28,8 @@
 import java.util.Map;
 
 import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
+import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
+import static io.deephaven.parquet.base.ParquetUtils.getKeyForFilePath;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
 /**
@@ -40,11 +42,11 @@ final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
      * A class to hold the parquet file and its metadata.
      */
     private static class ParquetFileMetadata {
-        final File file;
+        final String filePath;
         final ParquetMetadata metadata;
 
-        ParquetFileMetadata(final File file, final ParquetMetadata metadata) {
-            this.file = file;
+        ParquetFileMetadata(final String filePath, final ParquetMetadata metadata) {
+            this.filePath = filePath;
             this.metadata = metadata;
         }
     }
@@ -73,8 +75,29 @@ private static class ParquetFileMetadata {
      */
     ParquetMetadataFileWriterImpl(@NotNull final File metadataRootDir, @NotNull final File[] destinations,
             @Nullable final MessageType partitioningColumnsSchema) {
+        if (destinations.length == 0) {
+            throw new IllegalArgumentException("No destinations provided");
+        }
         this.metadataRootDirAbsPath = metadataRootDir.getAbsoluteFile().toPath();
         final String metadataRootDirAbsPathString = metadataRootDirAbsPath.toString();
+        final File firstDestination = destinations[0];
+        for (int i = 0; i < destinations.length; i++) {
+            final File destination = destinations[i];
+            if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPathString)) {
+                throw new UncheckedDeephavenException("All destinations must be nested under the provided metadata root"
+                        + " directory, provided destination " + destination.getAbsolutePath() + " is not under " +
+                        metadataRootDirAbsPathString);
+            }
+            // TODO How should I change the basename in the API for writeKeyValuePartitioned data for this check?
+            if (i > 0) {
+                // We use filename to generate the key for each file's metadata in the common metadata file, therefore
+                // all files must have unique names.
+                if (destination.getName().equals(firstDestination.getName())) {
+                    throw new UncheckedDeephavenException("When generating common metadata for multiple parquet files, "
+                            + "all files must have unique names, but " + destination.getName() + " is repeated.");
+                }
+            }
+        }
         for (final File destination : destinations) {
             if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPathString)) {
                 throw new UncheckedDeephavenException("All destinations must be nested under the provided metadata root"
@@ -98,27 +121,28 @@ private static class ParquetFileMetadata {
     /**
      * Added parquet metadata for provided parquet file.
      *
-     * @param parquetFile The parquet file destination path
+     * @param parquetFilePath The parquet file destination path
      * @param metadata The parquet metadata
      */
-    public void addParquetFileMetadata(final File parquetFile, final ParquetMetadata metadata) {
-        parquetFileMetadataList.add(new ParquetFileMetadata(parquetFile, metadata));
+    public void addParquetFileMetadata(final String parquetFilePath, final ParquetMetadata metadata) {
+        parquetFileMetadataList.add(new ParquetFileMetadata(parquetFilePath, metadata));
     }
 
     /**
      * Write the accumulated metadata to the provided files and clear the metadata accumulated so far.
      *
-     * @param metadataFile The destination for the {@value ParquetUtils#METADATA_FILE_NAME} file
-     * @param commonMetadataFile The destination for the {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file
+     * @param metadataFilePath The destination path for the {@value ParquetUtils#METADATA_FILE_NAME} file
+     * @param commonMetadataFilePath The destination path for the {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file
      */
-    public void writeMetadataFiles(final File metadataFile, final File commonMetadataFile) throws IOException {
+    public void writeMetadataFiles(final String metadataFilePath, final String commonMetadataFilePath)
+            throws IOException {
         if (parquetFileMetadataList.isEmpty()) {
             throw new UncheckedDeephavenException("No parquet files to write metadata for");
         }
         mergeMetadata();
         final ParquetMetadata metadataFooter = new ParquetMetadata(new FileMetaData(mergedSchema,
                 mergedKeyValueMetaData, mergedCreatedByString), mergedBlocks);
-        writeMetadataFile(metadataFooter, metadataFile.getAbsolutePath());
+        writeMetadataFile(metadataFooter, metadataFilePath);
 
         // Skip the blocks data and merge schema with partitioning columns' schema to write the common metadata file.
         // The ordering of arguments in method call is important because we want to keep partitioning columns in the
@@ -127,7 +151,7 @@ public void writeMetadataFiles(final File metadataFile, final File commonMetadat
         final ParquetMetadata commonMetadataFooter =
                 new ParquetMetadata(new FileMetaData(mergedSchema, mergedKeyValueMetaData, mergedCreatedByString),
                         new ArrayList<>());
-        writeMetadataFile(commonMetadataFooter, commonMetadataFile.toString());
+        writeMetadataFile(commonMetadataFooter, commonMetadataFilePath);
 
         // Clear the accumulated metadata
         clear();
@@ -141,16 +165,16 @@ private void mergeMetadata() throws IOException {
         for (final ParquetFileMetadata parquetFileMetadata : parquetFileMetadataList) {
             final FileMetaData fileMetaData = parquetFileMetadata.metadata.getFileMetaData();
             mergedSchema = mergeSchemaInto(fileMetaData.getSchema(), mergedSchema);
-            mergeKeyValueMetaData(fileMetaData.getKeyValueMetaData());
+            mergeKeyValueMetaData(parquetFileMetadata);
             mergeBlocksInto(parquetFileMetadata, metadataRootDirAbsPath, mergedBlocks);
             mergedCreatedBy.add(fileMetaData.getCreatedBy());
         }
-        // Generate the merged deephaven-specific metadata
+        // Add table info to the merged key-value metadata
         final TableInfo.Builder tableInfoBuilder = TableInfo.builder().addAllColumnTypes(mergedColumnTypes);
         if (mergedVersion != null) {
             tableInfoBuilder.version(mergedVersion);
         }
-        mergedKeyValueMetaData.put(ParquetTableWriter.METADATA_KEY, tableInfoBuilder.build().serializeToJSON());
+        mergedKeyValueMetaData.put(METADATA_KEY, tableInfoBuilder.build().serializeToJSON());
         mergedCreatedByString =
                 mergedCreatedBy.size() == 1 ? mergedCreatedBy.iterator().next() : mergedCreatedBy.toString();
     }
@@ -170,13 +194,20 @@ private static MessageType mergeSchemaInto(final MessageType schema, final Messa
     }
 
     /**
-     * Merge the non-deephaven-specific key-value metadata for a file into the merged metadata. For deephaven specific
-     * fields, we filter the relevant fields here and merge them later once all the files have been processed.
+     * This method processes both deephaven specific and non-deephaven key-value metadata for each file.
+     * <ul>
+     * <li>For non-deephaven specific key-value metadata, we accumulate it directly and enforce that there is only one
+     * value for each key</li>
+     * <li>For deephaven specific key-value metadata, we copy each file's metadata directly into the merged metadata as
+     * well as accumulate the required fields to generate a common table info later once all files are processed.</li>
+     * </ul>
      */
-    private void mergeKeyValueMetaData(final Map<String, String> keyValueMetaData) throws IOException {
+    private void mergeKeyValueMetaData(@NotNull final ParquetFileMetadata parquetFileMetadata) throws IOException {
+        final Map<String, String> keyValueMetaData =
+                parquetFileMetadata.metadata.getFileMetaData().getKeyValueMetaData();
         for (final Map.Entry<String, String> entry : keyValueMetaData.entrySet()) {
-            if (!entry.getKey().equals(ParquetTableWriter.METADATA_KEY)) {
-                // We should only have one value for each key
+            if (!entry.getKey().equals(METADATA_KEY)) {
+                // Make sure we have unique value for each key.
                 mergedKeyValueMetaData.compute(entry.getKey(), (k, v) -> {
                     if (v == null) {
                         // No existing value for this key, so put the new value
@@ -190,20 +221,26 @@ private void mergeKeyValueMetaData(final Map<String, String> keyValueMetaData) t
                     return v;
                 });
             } else {
-                // For merging deephaven-specific metadata,
+                // Add a separate entry for each file
+                final String fileKey = getKeyForFilePath(Path.of(parquetFileMetadata.filePath));
+                // Assuming the keys are unique for each file because file names are unique, verified in the constructor
+                Assert.eqNull(mergedKeyValueMetaData.get(fileKey), "mergedKeyValueMetaData.get(fileKey)");
+                mergedKeyValueMetaData.put(fileKey, entry.getValue());
+
+                // Also, process and accumulate the relevant fields:
                 // - groupingColumns, dataIndexes are skipped
                 // - columnTypes must be the same for all partitions
                 // - version is set as non-null if all the files have the same version
                 final TableInfo tableInfo = TableInfo.deserializeFromJSON(entry.getValue());
                 if (mergedColumnTypes == null) {
-                    // First time we've seen deephaven specific metadata, so just copy the relevant fields
+                    // The First file for which we've seen deephaven specific metadata, so just copy the relevant fields
                     mergedColumnTypes = tableInfo.columnTypes();
                     mergedVersion = tableInfo.version();
                 } else {
                     if (!mergedColumnTypes.equals(tableInfo.columnTypes())) {
-                        throw new UncheckedDeephavenException("Could not merge metadata for key " +
-                                ParquetTableWriter.METADATA_KEY + ", has conflicting values for columnTypes: " +
-                                tableInfo.columnTypes() + " and " + mergedColumnTypes);
+                        throw new UncheckedDeephavenException("Could not merge metadata for key " + METADATA_KEY +
+                                ", has conflicting values for columnTypes: " + tableInfo.columnTypes() + " and "
+                                + mergedColumnTypes);
                     }
                     if (!tableInfo.version().equals(mergedVersion)) {
                         mergedVersion = null;
@@ -215,7 +252,7 @@ private void mergeKeyValueMetaData(final Map<String, String> keyValueMetaData) t
 
     private static void mergeBlocksInto(final ParquetFileMetadata parquetFileMetadata,
             final Path metadataRootDirAbsPath, final Collection<BlockMetaData> mergedBlocks) {
-        final Path parquetFileAbsPath = Paths.get(parquetFileMetadata.file.getAbsolutePath());
+        final Path parquetFileAbsPath = new File(parquetFileMetadata.filePath).getAbsoluteFile().toPath();
         String fileRelativePathString = metadataRootDirAbsPath.relativize(parquetFileAbsPath).toString();
         // Remove leading slashes from the relative path
         int pos = 0;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java
index d6bb66d7c10..6e10562e673 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java
@@ -32,6 +32,8 @@
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
 
+import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
+
 public class ParquetSchemaReader {
     @FunctionalInterface
     public interface ColumnDefinitionConsumer {
@@ -107,14 +109,14 @@ public static ParquetInstructions readParquetSchema(
     }
 
     public static Optional<TableInfo> parseMetadata(@NotNull final Map<String, String> keyValueMetadata) {
-        final String tableInfoRaw = keyValueMetadata.get(ParquetTableWriter.METADATA_KEY);
+        final String tableInfoRaw = keyValueMetadata.get(METADATA_KEY);
         if (tableInfoRaw == null) {
             return Optional.empty();
         }
         try {
             return Optional.of(TableInfo.deserializeFromJSON(tableInfoRaw));
         } catch (IOException e) {
-            throw new TableDataException("Failed to parse " + ParquetTableWriter.METADATA_KEY + " metadata", e);
+            throw new TableDataException("Failed to parse " + METADATA_KEY + " metadata", e);
         }
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 488b31ab1ec..2d5ca456530 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -50,13 +50,13 @@
 import java.nio.file.Paths;
 import java.util.*;
 
+import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
 /**
  * API for writing DH tables in parquet format
  */
 public class ParquetTableWriter {
-    static final String METADATA_KEY = "deephaven";
     public static final String BEGIN_POS = "dh_begin_pos";
     public static final String END_POS = "dh_end_pos";
     public static final String GROUPING_KEY = "dh_key";
@@ -72,8 +72,7 @@ static class GroupingColumnWritingInfo {
         /**
          * File path to be added in the grouping metadata of main parquet file
          */
-        final File metadataFilePath;
-
+        final File destFileForMetadata;
         /**
          * Destination path for writing the grouping file. The two filenames can differ because we write grouping files
          * to shadow file paths first and then place them at the final path once the write is complete. But the metadata
@@ -81,9 +80,9 @@ static class GroupingColumnWritingInfo {
          */
         final File destFile;
 
-        GroupingColumnWritingInfo(final String parquetColumnName, final File metadataFilePath, final File destFile) {
+        GroupingColumnWritingInfo(final String parquetColumnName, final File destFileForMetadata, final File destFile) {
             this.parquetColumnName = parquetColumnName;
-            this.metadataFilePath = metadataFilePath;
+            this.destFileForMetadata = destFileForMetadata;
             this.destFile = destFile;
         }
     }
@@ -94,10 +93,10 @@ static class GroupingColumnWritingInfo {
      * @param t The table to write
      * @param definition Table definition
      * @param writeInstructions Write instructions for customizations while writing
-     * @param destPathName The destination path
-     * @param metadataFilePath The destination path to store in the metadata files. This can be different from
-     *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
-     *        always hold the accurate path.
+     * @param destFilePath The destination path
+     * @param destFilePathForMetadata The destination path to store in the metadata files. This can be different from
+     *        {@code destFilePath} if we are writing the parquet file to a shadow location first since the metadata
+     *        should always hold the accurate path.
      * @param incomingMeta A map of metadata values to be stores in the file footer
      * @param groupingColumnsWritingInfoMap A map of grouping column names to their respective info used for writing
      *        grouping files
@@ -114,8 +113,8 @@ static void write(
             @NotNull final Table t,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
-            @NotNull final String destPathName,
-            @NotNull final File metadataFilePath,
+            @NotNull final String destFilePath,
+            @NotNull final String destFilePathForMetadata,
             @NotNull final Map<String, String> incomingMeta,
             @Nullable final Map<String, GroupingColumnWritingInfo> groupingColumnsWritingInfoMap,
             @NotNull final ParquetMetadataFileWriter metadataFileWriter,
@@ -126,25 +125,26 @@ static void write(
         try {
             if (groupingColumnsWritingInfoMap != null) {
                 cleanupFiles = new ArrayList<>(groupingColumnsWritingInfoMap.size());
-                final Path destDirPath = Paths.get(destPathName).getParent();
+                final Path destDirPath = Paths.get(destFilePath).getParent();
                 for (final Map.Entry<String, GroupingColumnWritingInfo> entry : groupingColumnsWritingInfoMap
                         .entrySet()) {
                     final String groupingColumnName = entry.getKey();
                     final Table auxiliaryTable = groupingAsTable(t, groupingColumnName);
                     final String parquetColumnName = entry.getValue().parquetColumnName;
-                    final File metadataGroupingFilePath = entry.getValue().metadataFilePath;
-                    final File groupingDestFile = entry.getValue().destFile;
-                    cleanupFiles.add(groupingDestFile);
+                    final File groupingFileDestForMetadata = entry.getValue().destFileForMetadata;
+                    final File groupingFileDest = entry.getValue().destFile;
+                    cleanupFiles.add(groupingFileDest);
                     tableInfoBuilder.addGroupingColumns(GroupingColumnInfo.of(parquetColumnName,
-                            destDirPath.relativize(metadataGroupingFilePath.toPath()).toString()));
+                            destDirPath.relativize(groupingFileDestForMetadata.toPath()).toString()));
                     // We don't accumulate metadata from grouping files into the main metadata file
                     write(auxiliaryTable, auxiliaryTable.getDefinition(), writeInstructions,
-                            groupingDestFile.getAbsolutePath(),
-                            metadataGroupingFilePath, Collections.emptyMap(), TableInfo.builder(),
+                            groupingFileDest.getAbsolutePath(),
+                            groupingFileDestForMetadata.getAbsolutePath(), Collections.emptyMap(), TableInfo.builder(),
                             NullParquetMetadataFileWriter.INSTANCE, computedCache);
                 }
             }
-            write(t, definition, writeInstructions, destPathName, metadataFilePath, incomingMeta, tableInfoBuilder,
+            write(t, definition, writeInstructions, destFilePath, destFilePathForMetadata, incomingMeta,
+                    tableInfoBuilder,
                     metadataFileWriter, computedCache);
         } catch (Exception e) {
             if (cleanupFiles != null) {
@@ -166,10 +166,10 @@ static void write(
      * @param table The table to write
      * @param definition The table definition
      * @param writeInstructions Write instructions for customizations while writing
-     * @param path The destination path
-     * @param metadataFilePath The destination path to store in the metadata files. This can be different from
-     *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
-     *        always hold the accurate path.
+     * @param destFilePath The destination path
+     * @param destFilePathForMetadata The destination path to store in the metadata files. This can be different from
+     *        {@code destFilePath} if we are writing the parquet file to a shadow location first since the metadata
+     *        should always hold the accurate path.
      * @param tableMeta A map of metadata values to be stores in the file footer
      * @param tableInfoBuilder A partially constructed builder for the metadata object
      * @param metadataFileWriter The writer for the {@value ParquetUtils#METADATA_FILE_NAME} and
@@ -181,8 +181,8 @@ static void write(
             @NotNull final Table table,
             @NotNull final TableDefinition definition,
             @NotNull final ParquetInstructions writeInstructions,
-            @NotNull final String path,
-            @NotNull final File metadataFilePath,
+            @NotNull final String destFilePath,
+            @NotNull final String destFilePathForMetadata,
             @NotNull final Map<String, String> tableMeta,
             @NotNull final TableInfo.Builder tableInfoBuilder,
             @NotNull final ParquetMetadataFileWriter metadataFileWriter,
@@ -192,7 +192,8 @@ static void write(
             final TrackingRowSet tableRowSet = t.getRowSet();
             final Map<String, ? extends ColumnSource<?>> columnSourceMap = t.getColumnSourceMap();
             final ParquetFileWriter parquetFileWriter = getParquetFileWriter(computedCache, definition, tableRowSet,
-                    columnSourceMap, path, metadataFilePath, writeInstructions, tableMeta, tableInfoBuilder,
+                    columnSourceMap, destFilePath, destFilePathForMetadata, writeInstructions, tableMeta,
+                    tableInfoBuilder,
                     metadataFileWriter);
             // Given the transformation, do not use the original table's "definition" for writing
             write(t, writeInstructions, parquetFileWriter, computedCache);
@@ -300,10 +301,10 @@ private static Table pretransformTable(@NotNull final Table table, @NotNull fina
      * @param definition the writable definition
      * @param tableRowSet the row set being written
      * @param columnSourceMap the columns of the table
-     * @param path the destination to write to
-     * @param metadataFilePath The destination path to store in the metadata files. This can be different from
-     *        {@code destFile} if we are writing the parquet file to a shadow location first since the metadata should
-     *        always hold the accurate path.
+     * @param destFilePath the destination to write to
+     * @param destFilePathForMetadata The destination path to store in the metadata files. This can be different from
+     *        {@code destFilePath} if we are writing the parquet file to a shadow location first since the metadata
+     *        should always hold the accurate path.
      * @param writeInstructions write instructions for the file
      * @param tableMeta metadata to include in the parquet metadata
      * @param tableInfoBuilder a builder for accumulating per-column information to construct the deephaven metadata
@@ -318,8 +319,8 @@ private static ParquetFileWriter getParquetFileWriter(
             @NotNull final TableDefinition definition,
             @NotNull final RowSet tableRowSet,
             @NotNull final Map<String, ? extends ColumnSource<?>> columnSourceMap,
-            @NotNull final String path,
-            @NotNull final File metadataFilePath,
+            @NotNull final String destFilePath,
+            @NotNull final String destFilePathForMetadata,
             @NotNull final ParquetInstructions writeInstructions,
             @NotNull final Map<String, String> tableMeta,
             @NotNull final TableInfo.Builder tableInfoBuilder,
@@ -365,8 +366,8 @@ private static ParquetFileWriter getParquetFileWriter(
 
         final Map<String, String> extraMetaData = new HashMap<>(tableMeta);
         extraMetaData.put(METADATA_KEY, tableInfoBuilder.build().serializeToJSON());
-        return new ParquetFileWriter(path, metadataFilePath,
-                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(path), null),
+        return new ParquetFileWriter(destFilePath, destFilePathForMetadata,
+                SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(destFilePath), null),
                 writeInstructions.getTargetPageSize(),
                 new HeapByteBufferAllocator(), mappedSchema.getParquetSchema(),
                 writeInstructions.getCompressionCodecName(), extraMetaData, metadataFileWriter);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 2ded0e1e719..18dfe7acc76 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -866,7 +866,7 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
                     shadowFiles.add(shadowDestFiles[tableIdx]);
                     final Table source = sources[tableIdx];
                     ParquetTableWriter.write(source, definition, writeInstructions, shadowDestFiles[tableIdx].getPath(),
-                            destinations[tableIdx], Collections.emptyMap(),
+                            destinations[tableIdx].getPath(), Collections.emptyMap(),
                             (Map<String, ParquetTableWriter.GroupingColumnWritingInfo>) null, metadataFileWriter,
                             computedCache);
                 }
@@ -890,7 +890,7 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
 
                     final Table sourceTable = sources[tableIdx];
                     ParquetTableWriter.write(sourceTable, definition, writeInstructions,
-                            shadowDestFiles[tableIdx].getPath(), tableDestination, Collections.emptyMap(),
+                            shadowDestFiles[tableIdx].getPath(), tableDestination.getPath(), Collections.emptyMap(),
                             groupingColumnWritingInfoMap, metadataFileWriter, computedCache);
                 }
             }
@@ -904,7 +904,8 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
                 commonMetadataDestFile = new File(metadataRootDir, COMMON_METADATA_FILE_NAME);
                 shadowCommonMetadataFile = ParquetTools.getShadowFile(commonMetadataDestFile);
                 shadowFiles.add(shadowCommonMetadataFile);
-                metadataFileWriter.writeMetadataFiles(shadowMetadataFile, shadowCommonMetadataFile);
+                metadataFileWriter.writeMetadataFiles(shadowMetadataFile.getAbsolutePath(),
+                        shadowCommonMetadataFile.getAbsolutePath());
             } else {
                 metadataDestFile = shadowMetadataFile = commonMetadataDestFile = shadowCommonMetadataFile = null;
             }
@@ -917,7 +918,7 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
                     final Map<String, ParquetTableWriter.GroupingColumnWritingInfo> gcwim =
                             groupingColumnWritingInfoMaps.get(tableIdx);
                     for (final ParquetTableWriter.GroupingColumnWritingInfo gfwi : gcwim.values()) {
-                        final File indexDestFile = gfwi.metadataFilePath;
+                        final File indexDestFile = gfwi.destFileForMetadata;
                         final File shadowIndexFile = gfwi.destFile;
                         destFiles.add(indexDestFile);
                         installShadowFile(indexDestFile, shadowIndexFile);
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
index f23f56aede3..fad36721d6b 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
@@ -20,6 +20,7 @@
 import io.deephaven.util.type.TypeUtils;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.parquet.format.RowGroup;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -39,6 +40,8 @@
 
 import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_FILE_NAME;
+import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
+import static io.deephaven.parquet.base.ParquetUtils.getKeyForFilePath;
 import static java.util.stream.Collectors.toMap;
 
 /**
@@ -197,15 +200,43 @@ public ParquetMetadataFileLayout(
                     partitions.put(partitionKey, partitionValue);
                 }
             }
-            final ParquetTableLocationKey tlk = new ParquetTableLocationKey(new File(directory, filePathString),
+            final File partitionFile = new File(directory, filePathString);
+            final ParquetTableLocationKey tlk = new ParquetTableLocationKey(partitionFile,
                     partitionOrder.getAndIncrement(), partitions, inputInstructions);
             tlk.setFileReader(metadataFileReader);
-            tlk.setMetadata(metadataFileMetadata);
+            tlk.setMetadata(getParquetMetadataForFile(partitionFile, metadataFileMetadata));
             tlk.setRowGroupIndices(rowGroupIndices);
             return tlk;
         }).collect(Collectors.toList());
     }
 
+    /**
+     * This method takes the {@link ParquetMetadata} from the metadata file, extracts the key-value metadata specific to
+     * the provided file, and creates a new {@link ParquetMetadata} for this file.
+     *
+     * @param parquetFile The parquet file to read metadata for
+     * @param metadataFileMetadata The overall metadata in the metadata file
+     */
+    private static ParquetMetadata getParquetMetadataForFile(@NotNull final File parquetFile,
+            @NotNull final ParquetMetadata metadataFileMetadata) {
+        final String fileMetadataString = metadataFileMetadata.getFileMetaData().getKeyValueMetaData()
+                .get(getKeyForFilePath(parquetFile.toPath()));
+        final ParquetMetadata fileMetadata;
+        if (fileMetadataString != null) {
+            // Create a new file metadata object using the key-value metadata for that file
+            final Map<String, String> keyValueMetadata = Map.of(METADATA_KEY, fileMetadataString);
+            fileMetadata = new ParquetMetadata(
+                    new FileMetaData(metadataFileMetadata.getFileMetaData().getSchema(),
+                            keyValueMetadata,
+                            metadataFileMetadata.getFileMetaData().getCreatedBy()),
+                    metadataFileMetadata.getBlocks());
+        } else {
+            // File specific metadata not found, the the metadata file's metadata
+            fileMetadata = metadataFileMetadata;
+        }
+        return fileMetadata;
+    }
+
     public String toString() {
         return ParquetMetadataFileLayout.class.getSimpleName() + '[' + metadataFile + ',' + commonMetadataFile + ']';
     }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java
index 5301676e3b0..065d5407e52 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java
@@ -37,7 +37,7 @@ public abstract class ColumnChunkPageStore<ATTR extends Any>
         implements PageStore<ATTR, ATTR, ChunkPage<ATTR>>, Page<ATTR>, SafeCloseable, Releasable {
 
     final PageCache<ATTR> pageCache;
-    private final ColumnChunkReader columnChunkReader;
+    final ColumnChunkReader columnChunkReader;
     private final long mask;
     private final ToPage<ATTR, ?> toPage;
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
index d8881962d88..468da6c08a7 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
@@ -28,6 +28,7 @@
  */
 final class OffsetIndexBasedColumnChunkPageStore<ATTR extends Any> extends ColumnChunkPageStore<ATTR> {
     private static final long PAGE_SIZE_NOT_FIXED = -1;
+    private static final int NUM_PAGES_NOT_INITIALIZED = -1;
 
     private static final class PageState<ATTR extends Any> {
         private volatile WeakReference<PageCache.IntrusivePage<ATTR>> pageRef;
@@ -37,8 +38,9 @@ private static final class PageState<ATTR extends Any> {
         }
     }
 
-    private final OffsetIndex offsetIndex;
-    private final int numPages;
+    private volatile boolean isInitialized; // This class is initialized when reading the first page
+    private OffsetIndex offsetIndex;
+    private int numPages;
     /**
      * Fixed number of rows per page. Set as positive value if first ({@link #numPages}-1) pages have equal number of
      * rows, else equal to {@value #PAGE_SIZE_NOT_FIXED}. We cannot find the number of rows in the last page size from
@@ -47,9 +49,9 @@ private static final class PageState<ATTR extends Any> {
      * the same size and calculate the page number as {@code row_index / fixed_page_size -> page_number}. If it is
      * greater than {@link #numPages}, we will infer that the row is coming from last page.
      */
-    private final long fixedPageSize;
-    private final AtomicReferenceArray<PageState<ATTR>> pageStates;
-    private final ColumnChunkReader.ColumnPageDirectAccessor columnPageDirectAccessor;
+    private long fixedPageSize;
+    private AtomicReferenceArray<PageState<ATTR>> pageStates;
+    private ColumnChunkReader.ColumnPageDirectAccessor columnPageDirectAccessor;
 
     OffsetIndexBasedColumnChunkPageStore(
             @NotNull final PageCache<ATTR> pageCache,
@@ -57,29 +59,47 @@ private static final class PageState<ATTR extends Any> {
             final long mask,
             @NotNull final ToPage<ATTR, ?> toPage) throws IOException {
         super(pageCache, columnChunkReader, mask, toPage);
-        try (final SeekableChannelContext channelContext =
-                columnChunkReader.getChannelsProvider().makeSingleUseContext()) {
-            offsetIndex = columnChunkReader.getOffsetIndex(channelContext);
-        }
-        Assert.neqNull(offsetIndex, "offsetIndex");
-        numPages = offsetIndex.getPageCount();
-        Assert.gtZero(numPages, "numPages");
-        pageStates = new AtomicReferenceArray<>(numPages);
-        columnPageDirectAccessor = columnChunkReader.getPageAccessor(offsetIndex);
-
-        if (numPages == 1) {
-            fixedPageSize = numRows();
+        isInitialized = false;
+        numPages = NUM_PAGES_NOT_INITIALIZED;
+        fixedPageSize = PAGE_SIZE_NOT_FIXED;
+        offsetIndex = null;
+        pageStates = null;
+        columnPageDirectAccessor = null;
+    }
+
+    private void initialize(@Nullable final FillContext fillContext) {
+        if (isInitialized) {
             return;
         }
-        boolean isPageSizeFixed = true;
-        final long firstPageSize = offsetIndex.getFirstRowIndex(1) - offsetIndex.getFirstRowIndex(0);
-        for (int i = 2; i < numPages; ++i) {
-            if (offsetIndex.getFirstRowIndex(i) - offsetIndex.getFirstRowIndex(i - 1) != firstPageSize) {
-                isPageSizeFixed = false;
-                break;
+        synchronized (this) {
+            if (isInitialized) {
+                return;
             }
+            try (final ContextHolder holder = SeekableChannelContext.ensureContext(
+                    columnChunkReader.getChannelsProvider(), innerFillContext(fillContext))) {
+                offsetIndex = columnChunkReader.getOffsetIndex(holder.get());
+            }
+            Assert.neqNull(offsetIndex, "offsetIndex");
+            numPages = offsetIndex.getPageCount();
+            Assert.gtZero(numPages, "numPages");
+            pageStates = new AtomicReferenceArray<>(numPages);
+            columnPageDirectAccessor = columnChunkReader.getPageAccessor(offsetIndex);
+
+            if (numPages == 1) {
+                fixedPageSize = numRows();
+                return;
+            }
+            boolean isPageSizeFixed = true;
+            final long firstPageSize = offsetIndex.getFirstRowIndex(1) - offsetIndex.getFirstRowIndex(0);
+            for (int i = 2; i < numPages; ++i) {
+                if (offsetIndex.getFirstRowIndex(i) - offsetIndex.getFirstRowIndex(i - 1) != firstPageSize) {
+                    isPageSizeFixed = false;
+                    break;
+                }
+            }
+            fixedPageSize = isPageSizeFixed ? firstPageSize : PAGE_SIZE_NOT_FIXED;
+            isInitialized = true;
         }
-        fixedPageSize = isPageSizeFixed ? firstPageSize : PAGE_SIZE_NOT_FIXED;
     }
 
     /**
@@ -143,6 +163,24 @@ private ChunkPage<ATTR> getPageImpl(@Nullable FillContext fillContext, int pageN
     @Override
     @NotNull
     public ChunkPage<ATTR> getPageContaining(@Nullable final FillContext fillContext, long rowKey) {
+        if (isInitialized) {
+            return getPageContainingImpl(fillContext, rowKey);
+        }
+        // We need to initialize this class
+        if (fillContext != null) {
+            initialize(fillContext);
+            return getPageContainingImpl(fillContext, rowKey);
+        }
+        // Create a fill context and use it to initialize this class as well as read the page
+        // TODO What should be the chunk capacity here?
+        try (final FillContext fillContext1 = makeFillContext(1, null)) {
+            initialize(fillContext1);
+            return getPageContainingImpl(fillContext1, rowKey);
+        }
+    }
+
+    @NotNull
+    private ChunkPage<ATTR> getPageContainingImpl(@Nullable final FillContext fillContext, long rowKey) {
         rowKey &= mask();
         Require.inRange(rowKey, "rowKey", numRows(), "numRows");
 
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index b6cf0baa55f..08ea59e2e72 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -606,8 +606,8 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         final TableDefinition definition = TableDefinition.of(
                 ColumnDefinition.ofInt("PC1").withPartitioning(),
                 ColumnDefinition.ofInt("PC2").withPartitioning(),
-                ColumnDefinition.ofLong("I"));
-        final Table inputData = ((QueryTable) TableTools.emptyTable(20)
+                ColumnDefinition.ofLong("I").withGrouping());
+        final Table inputData = ((QueryTable) TableTools.emptyTable(1_000_000)
                 .updateView("PC1 = (int)(ii%3)",
                         "PC2 = (int)(ii%2)",
                         "I = ii"))
@@ -630,10 +630,12 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         }
 
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        fromDisk.where("I == 3").select();
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
         final File commonMetadata = new File(parentDir, "_common_metadata");
         final Table fromDiskWithMetadata = readTable(commonMetadata);
+        fromDiskWithMetadata.where("I == 3").select();
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
     }
 

From 5018feb8839562f51124cafccc54316016d57f20 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 11 Mar 2024 12:42:13 -0500
Subject: [PATCH 30/45] Minor touchups

---
 .../java/io/deephaven/parquet/table/ParquetTableWriter.java | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 2d5ca456530..c8bfbef46c2 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -144,8 +144,7 @@ static void write(
                 }
             }
             write(t, definition, writeInstructions, destFilePath, destFilePathForMetadata, incomingMeta,
-                    tableInfoBuilder,
-                    metadataFileWriter, computedCache);
+                    tableInfoBuilder, metadataFileWriter, computedCache);
         } catch (Exception e) {
             if (cleanupFiles != null) {
                 for (final File cleanupFile : cleanupFiles) {
@@ -193,8 +192,7 @@ static void write(
             final Map<String, ? extends ColumnSource<?>> columnSourceMap = t.getColumnSourceMap();
             final ParquetFileWriter parquetFileWriter = getParquetFileWriter(computedCache, definition, tableRowSet,
                     columnSourceMap, destFilePath, destFilePathForMetadata, writeInstructions, tableMeta,
-                    tableInfoBuilder,
-                    metadataFileWriter);
+                    tableInfoBuilder, metadataFileWriter);
             // Given the transformation, do not use the original table's "definition" for writing
             write(t, writeInstructions, parquetFileWriter, computedCache);
         }

From a529fc2bdd4d650495c32bd2e7b28d47bcd40650 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Fri, 15 Mar 2024 14:54:50 -0500
Subject: [PATCH 31/45] Fixed failing tests

---
 .../deephaven/parquet/base/ParquetUtils.java  |  5 +-
 .../table/ParquetMetadataFileWriterImpl.java  | 21 +++--
 .../deephaven/parquet/table/ParquetTools.java | 87 ++++++++++++++++++-
 .../layout/ParquetMetadataFileLayout.java     |  4 +-
 .../table/ParquetTableReadWriteTest.java      | 55 ++++++++----
 5 files changed, 142 insertions(+), 30 deletions(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
index 1c0135db7de..9abdf0850e3 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
@@ -30,10 +30,9 @@ public static boolean fileNameMatches(final Path path) {
     }
 
     /**
-     * @return the key value for storing each file's metadata in the common metadata file.
+     * @return the key value derived from the file name, used for storing each file's metadata in the metadata files.
      */
-    public static String getKeyForFilePath(final Path path) {
-        final String fileName = path.getFileName().toString();
+    public static String getKeyForFile(final String fileName) {
         return "deephaven_per_file_" + fileName;
     }
 }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index 486146e99c3..871d335baac 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -1,7 +1,6 @@
 package io.deephaven.parquet.table;
 
 import io.deephaven.UncheckedDeephavenException;
-import io.deephaven.base.verify.Assert;
 import io.deephaven.parquet.base.ParquetFileWriter;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.ParquetUtils;
@@ -29,7 +28,7 @@
 
 import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
-import static io.deephaven.parquet.base.ParquetUtils.getKeyForFilePath;
+import static io.deephaven.parquet.base.ParquetUtils.getKeyForFile;
 import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
 /**
@@ -119,7 +118,10 @@ private static class ParquetFileMetadata {
     }
 
     /**
-     * Added parquet metadata for provided parquet file.
+     * Add parquet metadata for the provided parquet file the combined metadata file. We store deephaven-specific
+     * metadata for each file individually inside the key-value metadata of the combined metadata file, with keys being
+     * derived from the file names and values being the metadata for the file. Therefore, the provided parquet files
+     * must have unique names.
      *
      * @param parquetFilePath The parquet file destination path
      * @param metadata The parquet metadata
@@ -169,6 +171,11 @@ private void mergeMetadata() throws IOException {
             mergeBlocksInto(parquetFileMetadata, metadataRootDirAbsPath, mergedBlocks);
             mergedCreatedBy.add(fileMetaData.getCreatedBy());
         }
+        if (mergedKeyValueMetaData.size() != parquetFileMetadataList.size()) {
+            throw new IllegalStateException("We should have one entry for each file in the merged key-value metadata, "
+                    + "but we have " + mergedKeyValueMetaData.size() + " entries for " + parquetFileMetadataList.size()
+                    + " files.");
+        }
         // Add table info to the merged key-value metadata
         final TableInfo.Builder tableInfoBuilder = TableInfo.builder().addAllColumnTypes(mergedColumnTypes);
         if (mergedVersion != null) {
@@ -222,9 +229,13 @@ private void mergeKeyValueMetaData(@NotNull final ParquetFileMetadata parquetFil
                 });
             } else {
                 // Add a separate entry for each file
-                final String fileKey = getKeyForFilePath(Path.of(parquetFileMetadata.filePath));
+                final String fileKey = getKeyForFile(new File(parquetFileMetadata.filePath).getName());
                 // Assuming the keys are unique for each file because file names are unique, verified in the constructor
-                Assert.eqNull(mergedKeyValueMetaData.get(fileKey), "mergedKeyValueMetaData.get(fileKey)");
+                if (mergedKeyValueMetaData.containsKey(fileKey)) {
+                    throw new UncheckedDeephavenException("Could not merge metadata for for file " +
+                            parquetFileMetadata.filePath + " because has conflicting file name with another file. For "
+                            + " generating metadata files, file names should be unique");
+                }
                 mergedKeyValueMetaData.put(fileKey, entry.getValue());
 
                 // Also, process and accumulate the relevant fields:
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 18dfe7acc76..a02857ff702 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -470,6 +470,25 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
         return gcwim;
     }
 
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
+     * table. The parquet files generated will have names of the format {@code "uuid.parquet"} where "uuid" is a random
+     * UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(Table, File, String, ParquetInstructions)} instead.
+     *
+     * @param sourceTable The table to partition and write
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final File destinationDir,
+            @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(sourceTable, destinationDir, "{uuid}", writeInstructions);
+    }
+
     /**
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
@@ -501,6 +520,27 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
                 writeInstructions);
     }
 
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
+     * table definition. The parquet files generated will have names of the format {@code "uuid.parquet"} where "uuid"
+     * is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(Table, TableDefinition, File, String, ParquetInstructions)} instead.
+     *
+     * @param sourceTable The table to partition and write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final File destinationDir,
+            @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, "{uuid}", writeInstructions);
+    }
+
     /**
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
@@ -545,6 +585,24 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
                 baseName, writeInstructions);
     }
 
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
+     * names of the format {@code "uuid.parquet"} where "uuid" is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, File, String, ParquetInstructions)} instead.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final File destinationDir,
+            @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(partitionedTable, destinationDir, "{uuid}", writeInstructions);
+    }
+
     /**
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
@@ -582,6 +640,27 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
                 baseName, writeInstructions);
     }
 
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
+     * names of the format {@code "uuid.parquet"} where "uuid" is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, File, String, ParquetInstructions)}
+     * instead.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
+     *        directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final File destinationDir,
+            @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, "{uuid}", writeInstructions);
+    }
+
     /**
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
@@ -647,6 +726,12 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
                     "Cannot write a partitioned parquet table with non-unique keys without " +
                             "{i} or {uuid} in the base name because there can be multiple partitions with the same key values");
         }
+        if (writeInstructions.generateMetadataFiles()) {
+            if (!hasUUIDInName && !(hasIndexInName && hasPartitionInName)) {
+                throw new IllegalArgumentException("For generating metadata files, file names for generated parquet " +
+                        "files should be unique. Try passing tokens like {uuid} in the file base name");
+            }
+        }
         // Note that there can be multiple constituents with the same key values, so cannot directly use the
         // partitionedTable.constituentFor(keyValues) method, and we need to group them together
         final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
@@ -796,7 +881,7 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         final File metadataRootDir;
         if (writeInstructions.generateMetadataFiles()) {
             // We insist on writing the metadata file in the same directory as the destination files, thus all
-            // destination files are in the same directory.
+            // destination files should be in the same directory. This also ensures that the file names would be unique.
             final String firstDestinationDir = destinations[0].getParentFile().getAbsolutePath();
             for (int i = 1; i < destinations.length; i++) {
                 if (!firstDestinationDir.equals(destinations[i].getParentFile().getAbsolutePath())) {
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
index fad36721d6b..d52810d4441 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
@@ -41,7 +41,7 @@
 import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_FILE_NAME;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
-import static io.deephaven.parquet.base.ParquetUtils.getKeyForFilePath;
+import static io.deephaven.parquet.base.ParquetUtils.getKeyForFile;
 import static java.util.stream.Collectors.toMap;
 
 /**
@@ -220,7 +220,7 @@ public ParquetMetadataFileLayout(
     private static ParquetMetadata getParquetMetadataForFile(@NotNull final File parquetFile,
             @NotNull final ParquetMetadata metadataFileMetadata) {
         final String fileMetadataString = metadataFileMetadata.getFileMetaData().getKeyValueMetaData()
-                .get(getKeyForFilePath(parquetFile.toPath()));
+                .get(getKeyForFile(parquetFile.getName()));
         final ParquetMetadata fileMetadata;
         if (fileMetadataString != null) {
             // Create a new file metadata object using the key-value metadata for that file
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 08ea59e2e72..2e3bcd84728 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -686,7 +686,7 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         }
 
         // Write the partitioned table with non-unique keys with {i} in the base name
-        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data-part-{i}",
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "{partitions}-data-{i}",
                 writeInstructions);
 
         // Verify that the partitioned data exists
@@ -696,7 +696,8 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
             final String[] dataFileList = dir.list();
             assertEquals(2, dataFileList.length);
             for (final String dataFile : dataFileList) {
-                assertTrue(dataFile.equals("data-part-0.parquet") || dataFile.equals("data-part-1.parquet"));
+                assertTrue(dataFile.equals("PC1=" + PC1 + "-data-0.parquet") ||
+                        dataFile.equals("PC1=" + PC1 + "-data-1.parquet"));
             }
         }
 
@@ -724,6 +725,22 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
                 assertTrue(dataFile.startsWith("data-") && dataFile.endsWith(".parquet"));
             }
         }
+
+        FileUtils.deleteRecursively(parentDir);
+
+        // Write the partitioned table with non-unique keys without a base name
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, writeInstructions);
+
+        // Verify that the partitioned data exists with uuid in names
+        for (int PC1 = 0; PC1 <= 2; PC1++) {
+            final File dir = new File(parentDir, "PC1=" + PC1 + File.separator);
+            assertTrue(dir.exists() && dir.isDirectory());
+            final String[] dataFileList = dir.list();
+            assertEquals(2, dataFileList.length);
+            for (final String dataFile : dataFileList) {
+                assertTrue(dataFile.endsWith(".parquet"));
+            }
+        }
     }
 
     @Test
@@ -740,7 +757,7 @@ public void writeKeyValuePartitionedDataWithNullKeys() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, parentDir, writeInstructions);
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(inputData.sort("PC1"), fromDisk.sort("PC1"));
         final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata")).select();
@@ -778,7 +795,7 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(inputData, tableDefinitionToWrite, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, tableDefinitionToWrite, parentDir, writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
@@ -788,8 +805,9 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
                 final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2 +
                         File.separator + "PC3=" + PC3);
                 assertTrue(dir.exists() && dir.isDirectory());
-                final File dataFile = new File(dir, "data.parquet");
-                assertTrue(dataFile.exists() && dataFile.isFile());
+                final String[] dataFileList = dir.list();
+                assertEquals(1, dataFileList.length);
+                assertTrue(dataFileList[0].endsWith(".parquet"));
             }
         }
 
@@ -828,15 +846,14 @@ public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        final PartitionedTable partitionedTable =
-                inputData.partitionBy("symbol", "epic_collection_id", "epic_request_id");
-        writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
-        assertTrue(new File(parentDir,
-                "symbol=AA/epic_collection_id=fss_tick%1234%4321/epic_request_id=223ea-asd43/data.parquet").exists());
-        assertTrue(new File(parentDir,
-                "symbol=BB/epic_collection_id=fss_tick%5678%8765/epic_request_id=98dce-oiu23/data.parquet").exists());
+        final String[] partitioningCols = new String[] {"symbol", "epic_collection_id", "epic_request_id"};
+        final PartitionedTable partitionedTable = inputData.partitionBy(partitioningCols);
+        writeKeyValuePartitionedTable(partitionedTable, parentDir, writeInstructions);
 
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        for (final String col : partitioningCols) {
+            assertTrue(fromDisk.getDefinition().getColumn(col).isPartitioning());
+        }
         assertTableEquals(inputData.sort("symbol", "epic_collection_id"),
                 fromDisk.sort("symbol", "epic_collection_id"));
 
@@ -887,11 +904,11 @@ public void testAllPartitioningColumnTypes() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir, writeInstructions);
 
         // Verify that we can read the partition values, but types like LocalDate or LocalTime will be read as strings
         // Therefore, we cannot compare the tables directly
-        final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+        readKeyValuePartitionedTable(parentDir, EMPTY).select();
 
         // Reading with metadata file should deduce the correct type, so we can compare the tables
         final File commonMetadata = new File(parentDir, "_common_metadata");
@@ -946,7 +963,7 @@ public void testAllNonPartitioningColumnTypes() {
                 .build();
 
         // First API we test is passing the table directly without any table definition
-        writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir, writeInstructions);
 
         // Store the big decimal with the precision and scale consistent with what we write to parquet
         inputData = maybeFixBigDecimal(inputData);
@@ -962,7 +979,7 @@ public void testAllNonPartitioningColumnTypes() {
 
         // Next API we test is passing the partitioned table without any table definition
         final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
-        writeKeyValuePartitionedTable(partitionedTable, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, parentDir, writeInstructions);
         fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(inputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
         fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
@@ -978,7 +995,7 @@ public void testAllNonPartitioningColumnTypes() {
                 .collect(Collectors.toList());
         newColumns.add(ColumnDefinition.ofInt("NPC15"));
         final TableDefinition newDefinition = TableDefinition.of(newColumns);
-        writeKeyValuePartitionedTable(inputData, newDefinition, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, newDefinition, parentDir, writeInstructions);
         final Table expected = inputData.dropColumns("PC2", "NPC6").updateView("NPC15 = (int)null");
         fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
@@ -988,7 +1005,7 @@ public void testAllNonPartitioningColumnTypes() {
         FileUtils.deleteRecursively(parentDir);
 
         // Next API we test is passing the partitioned table with an updated table definition
-        writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir, "data", writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir, writeInstructions);
         fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
         fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));

From 819a1b9997872e42d1e92f6e581177dc27c121c6 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Sat, 16 Mar 2024 09:36:23 -0500
Subject: [PATCH 32/45] Added python APIs and improved comments

Also, moved from File to String
---
 .../table/ParquetMetadataFileWriterImpl.java  |   4 +-
 .../deephaven/parquet/table/ParquetTools.java | 348 ++++++++++++++----
 .../table/ParquetTableReadWriteTest.java      |  34 +-
 py/server/deephaven/parquet.py                |  86 ++++-
 py/server/tests/test_parquet.py               |  80 +++-
 5 files changed, 450 insertions(+), 102 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index cf440564ef3..477dfc433c2 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -29,10 +29,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static io.deephaven.base.FileUtils.convertToURI;
 import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
 import static io.deephaven.parquet.base.ParquetUtils.getKeyForFile;
-import static io.deephaven.util.channel.SeekableChannelsProvider.convertToURI;
 
 /**
  * Used to generate a combined {@value ParquetUtils#METADATA_FILE_NAME} and
@@ -109,7 +109,7 @@ private static class ParquetFileMetadata {
         }
         this.parquetFileMetadataList = new ArrayList<>(destinations.length);
         this.channelsProvider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader(
-                convertToURI(metadataRootDirAbsPathString), null);
+                convertToURI(metadataRootDirAbsPathString, true), null);
         this.partitioningColumnsSchema = partitioningColumnsSchema;
 
         this.mergedSchema = null;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 8effc0267cc..6a8bf2cfc04 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -85,6 +85,10 @@ private ParquetTools() {}
 
     private static final Logger log = LoggerFactory.getLogger(ParquetTools.class);
 
+    private static final String UUID_TOKEN = "{uuid}";
+    private static final String PARTITIONS_TOKEN = "{partitions}";
+    private static final String FILE_INDEX_TOKEN = "{i}";
+
     /**
      * Reads in a table from a single parquet file, metadata file, or directory with recognized layout. The source
      * provided can be a local file path or a URI to be resolved via the provided
@@ -487,19 +491,36 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
-     * table. The parquet files generated will have names of the format {@code "uuid.parquet"} where "uuid" is a random
-     * UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(Table, File, String, ParquetInstructions)} instead.
+     * table. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid" is a
+     * random UUID. To provide custom file names, use {@link #writeKeyValuePartitionedTable(Table, String, String)}
+     * instead.
      *
      * @param sourceTable The table to partition and write
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final String destinationDir) {
+        writeKeyValuePartitionedTable(sourceTable, destinationDir, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
+     * table. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid" is a
+     * random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(Table, String, String, ParquetInstructions)} instead.
+     *
+     * @param sourceTable The table to partition and write
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(sourceTable, destinationDir, "{uuid}", writeInstructions);
+        writeKeyValuePartitionedTable(sourceTable, destinationDir, UUID_TOKEN, writeInstructions);
     }
 
     /**
@@ -509,11 +530,10 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * table.
      *
      * @param sourceTable The table to partition and write
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
-     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column. Users can also provide the following tokens to be replaced in the base name:
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
      *        <ul>
      *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
      *        "table-{uuid}" will result in files named like
@@ -523,10 +543,37 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
      *        partitioning columns.</li>
      *        </ul>
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final String destinationDir,
+            @NotNull final String baseName) {
+        writeKeyValuePartitionedTable(sourceTable, destinationDir, baseName, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
+     * table.
+     *
+     * @param sourceTable The table to partition and write
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
+     *        <ul>
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.</li>
+     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         writeKeyValuePartitionedTable(sourceTable, sourceTable.getDefinition(), destinationDir, baseName,
@@ -537,21 +584,40 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
-     * table definition. The parquet files generated will have names of the format {@code "uuid.parquet"} where "uuid"
+     * table definition. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid"
      * is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(Table, TableDefinition, File, String, ParquetInstructions)} instead.
+     * {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, String)} instead.
      *
      * @param sourceTable The table to partition and write
      * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final String destinationDir) {
+        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
+     * table definition. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid"
+     * is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, String, ParquetInstructions)} instead.
+     *
+     * @param sourceTable The table to partition and write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
             @NotNull final TableDefinition definition,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, "{uuid}", writeInstructions);
+        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, UUID_TOKEN, writeInstructions);
     }
 
     /**
@@ -562,17 +628,45 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      *
      * @param sourceTable The table to partition and write
      * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
-     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column. Users can also provide the following tokens to be replaced in the base name:
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
      *        <ul>
-     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
      *        "table-{uuid}" will result in files named like
      *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
-     *        partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.</li>
+     *        </ul>
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final String destinationDir,
+            @NotNull final String baseName) {
+        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, baseName, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
+     * table definition.
+     *
+     * @param sourceTable The table to partition and write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
+     *        <ul>
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
      *        partitioning columns.</li>
      *        </ul>
@@ -580,7 +674,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
             @NotNull final TableDefinition definition,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final List<ColumnDefinition<?>> partitioningColumns = definition.getPartitioningColumns();
@@ -602,18 +696,34 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
      * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
-     * names of the format {@code "uuid.parquet"} where "uuid" is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, File, String, ParquetInstructions)} instead.
+     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String)} instead.
      *
      * @param partitionedTable The partitioned table to write
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final String destinationDir) {
+        writeKeyValuePartitionedTable(partitionedTable, destinationDir, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
+     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String, ParquetInstructions)} instead.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(partitionedTable, destinationDir, "{uuid}", writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, destinationDir, UUID_TOKEN, writeInstructions);
     }
 
     /**
@@ -622,27 +732,55 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      * {@link Table#partitionBy(String...) partitionBy} on the required columns.
      *
      * @param partitionedTable The partitioned table to write
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
-     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column. Users can also provide the following tokens to be replaced in the base name:
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
      *        <ul>
-     *        <li>The token {@code {i}} will be replaced with an automatically incremented integer for files in a
-     *        directory. For example, a base name of "table-{i}" will result in files named like
+     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
+     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
      *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
-     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
      *        "table-{uuid}" will result in files named like
      *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
-     *        partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.</li>
+     *        </ul>
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final String destinationDir,
+            @NotNull final String baseName) {
+        writeKeyValuePartitionedTable(partitionedTable, destinationDir, baseName, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
+     *        <ul>
+     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
+     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
      *        partitioning columns.</li>
      *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(),
@@ -657,21 +795,40 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
      * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
-     * names of the format {@code "uuid.parquet"} where "uuid" is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, File, String, ParquetInstructions)}
+     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)}
+     * instead.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final String destinationDir) {
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
+     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)}
      * instead.
      *
      * @param partitionedTable The partitioned table to write
      * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition definition,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, "{uuid}", writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, UUID_TOKEN, writeInstructions);
     }
 
     /**
@@ -681,20 +838,51 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      *
      * @param partitionedTable The partitioned table to write
      * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The destination root directory to store partitioned data in nested format. Non-existing
-     *        directories are created.
-     * @param baseName The base name for the individual partitioned tables. For example, a base name of "table" will
-     *        result in files named "PC=partition1/table.parquet", "PC=partition2/table.parquet", etc., where "PC" is a
-     *        partitioning column.Users can also provide the following tokens to be replaced in the base name:
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
      *        <ul>
-     *        <li>The token {@code {i}} will be replaced with an automatically incremented integer for files in a
-     *        directory. For example, a base name of "table-{i}" will result in files named like
+     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
+     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
      *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
-     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
      *        "table-{uuid}" will result in files named like
      *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
-     *        partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
+     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+     *        partitioning columns.</li>
+     *        </ul>
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final String destinationDir,
+            @NotNull final String baseName) {
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, baseName,
+                ParquetInstructions.EMPTY);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
+     *        replaced in the base name:
+     *        <ul>
+     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
+     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
+     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
+     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
+     *        "table-{uuid}" will result in files named like
+     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
+     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
      *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
      *        partitioning columns.</li>
      *        </ul>
@@ -702,7 +890,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition definition,
-            @NotNull final File destinationDir,
+            @NotNull final String destinationDir,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
@@ -715,33 +903,33 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      * Write a partitioned table to disk in a key=value partitioning format with the already computed definition for the
      * key table and leaf table.
      * <p>
-     * Check {@link #writeKeyValuePartitionedTable(PartitionedTable, File, String, ParquetInstructions)},
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, File, String, ParquetInstructions)},
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, File, String, ParquetInstructions)}}, or
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, File, String, ParquetInstructions)} for
-     * more details.
+     * Check {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String, ParquetInstructions)},
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)},
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String, ParquetInstructions)}}, or
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)}
+     * for more details.
      */
     private static void writeKeyValuePartitionedTableImpl(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition keyTableDefinition,
             @NotNull final TableDefinition leafDefinition,
-            @NotNull final File destinationRoot,
+            @NotNull final String destinationRoot,
             @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         if (leafDefinition.numColumns() == 0) {
             throw new IllegalArgumentException("Cannot write a partitioned parquet table without any non-partitioning "
                     + "columns");
         }
-        final boolean hasPartitionInName = baseName.contains("{partitions}");
-        final boolean hasIndexInName = baseName.contains("{i}");
-        final boolean hasUUIDInName = baseName.contains("{uuid}");
+        final boolean hasPartitionInName = baseName.contains(PARTITIONS_TOKEN);
+        final boolean hasIndexInName = baseName.contains(FILE_INDEX_TOKEN);
+        final boolean hasUUIDInName = baseName.contains(UUID_TOKEN);
         if (!partitionedTable.uniqueKeys() && !hasIndexInName && !hasUUIDInName) {
             throw new IllegalArgumentException(
-                    "Cannot write a partitioned parquet table with non-unique keys without " +
-                            "{i} or {uuid} in the base name because there can be multiple partitions with the same key values");
+                    "Cannot write a partitioned parquet table with non-unique keys without {i} or {uuid} in the base " +
+                            "name because there can be multiple partitions with the same key values");
         }
         if (writeInstructions.generateMetadataFiles()) {
-            if (!hasUUIDInName && !(hasIndexInName && hasPartitionInName)) {
-                throw new IllegalArgumentException("For generating metadata files, file names for generated parquet " +
+            if (!hasUUIDInName && !hasPartitionInName) {
+                throw new IllegalArgumentException("For generating metadata files, file names for the parquet " +
                         "files should be unique. Try passing tokens like {uuid} in the file base name");
             }
         }
@@ -781,13 +969,13 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
                 for (final Table constituent : constituentVector) {
                     String filename = baseName;
                     if (hasPartitionInName) {
-                        filename = baseName.replace("{partitions}", String.join("_", partitionStrings));
+                        filename = baseName.replace(PARTITIONS_TOKEN, String.join("_", partitionStrings));
                     }
                     if (hasIndexInName) {
-                        filename = filename.replace("{i}", Integer.toString(count));
+                        filename = filename.replace(FILE_INDEX_TOKEN, Integer.toString(count));
                     }
                     if (hasUUIDInName) {
-                        filename = filename.replace("{uuid}", UUID.randomUUID().toString());
+                        filename = filename.replace(UUID_TOKEN, UUID.randomUUID().toString());
                     }
                     filename += PARQUET_FILE_EXTENSION;
                     destinations.add(new File(relativePath, filename));
@@ -808,14 +996,14 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
         }
         final Map<String, Map<ParquetCacheTags, Object>> computedCache =
                 buildComputedCache(partitionedTable::merge, leafDefinition);
-        ParquetTools.writeParquetTablesImpl(
+        writeParquetTablesImpl(
                 partitionedData.toArray(Table[]::new),
                 leafDefinition,
                 writeInstructions,
                 destinations.toArray(File[]::new),
                 leafDefinition.getGroupingColumnNamesArray(),
                 partitioningColumnsSchema,
-                destinationRoot,
+                new File(destinationRoot),
                 computedCache);
     }
 
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index a0497300d40..2602bee676b 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -618,14 +618,14 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir, "{partitions}-data-{i}", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), "{partitions}-data", writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
             for (int PC2 = 0; PC2 <= 1; PC2++) {
                 final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
                 assertTrue(dir.exists() && dir.isDirectory());
-                final File dataFile = new File(dir, "PC1=" + PC1 + "_PC2=" + PC2 + "-data-0.parquet");
+                final File dataFile = new File(dir, "PC1=" + PC1 + "_PC2=" + PC2 + "-data.parquet");
                 assertTrue(dataFile.exists() && dataFile.isFile());
             }
         }
@@ -655,7 +655,7 @@ public void writeKeyValuePartitionedDataWithNoNonPartitioningColumnsTest() {
                 .setGenerateMetadataFiles(true)
                 .build();
         try {
-            writeKeyValuePartitionedTable(inputData, parentDir, "data", writeInstructions);
+            writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), "data", writeInstructions);
             fail("Expected exception when writing the partitioned table with no non-partitioning columns");
         } catch (final RuntimeException expected) {
         }
@@ -680,15 +680,16 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
                 .setGenerateMetadataFiles(true)
                 .build();
         try {
-            writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data", writeInstructions);
+            writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(),
+                    "{partitions}-data", writeInstructions);
             fail("Expected exception when writing the partitioned table with non-unique keys without {i} or {uuid} in "
                     + "base name");
         } catch (final RuntimeException expected) {
         }
 
         // Write the partitioned table with non-unique keys with {i} in the base name
-        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "{partitions}-data-{i}",
-                writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(),
+                "{partitions}-data-{i}", writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
@@ -713,7 +714,7 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         FileUtils.deleteRecursively(parentDir);
 
         // Write the partitioned table with non-unique keys with {uuid} in the base name
-        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, "data-{uuid}",
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(), "data-{uuid}",
                 writeInstructions);
 
         // Verify that the partitioned data exists with uuid in names
@@ -730,7 +731,8 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         FileUtils.deleteRecursively(parentDir);
 
         // Write the partitioned table with non-unique keys without a base name
-        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(),
+                writeInstructions);
 
         // Verify that the partitioned data exists with uuid in names
         for (int PC1 = 0; PC1 <= 2; PC1++) {
@@ -758,7 +760,7 @@ public void writeKeyValuePartitionedDataWithNullKeys() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(partitionedTable, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, parentDir.getAbsolutePath(), writeInstructions);
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(inputData.sort("PC1"), fromDisk.sort("PC1"));
         final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata")).select();
@@ -796,7 +798,7 @@ public void writeKeyValuePartitionedDataWithMixedPartitionsTest() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(inputData, tableDefinitionToWrite, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(inputData, tableDefinitionToWrite, parentDir.getPath(), writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
@@ -849,7 +851,7 @@ public void someMoreKeyValuePartitionedTestsWithComplexKeys() {
                 .build();
         final String[] partitioningCols = new String[] {"symbol", "epic_collection_id", "epic_request_id"};
         final PartitionedTable partitionedTable = inputData.partitionBy(partitioningCols);
-        writeKeyValuePartitionedTable(partitionedTable, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, parentDir.getPath(), writeInstructions);
 
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         for (final String col : partitioningCols) {
@@ -905,7 +907,7 @@ public void testAllPartitioningColumnTypes() {
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir.getPath(), writeInstructions);
 
         // Verify that we can read the partition values, but types like LocalDate or LocalTime will be read as strings
         // Therefore, we cannot compare the tables directly
@@ -964,7 +966,7 @@ public void testAllNonPartitioningColumnTypes() {
                 .build();
 
         // First API we test is passing the table directly without any table definition
-        writeKeyValuePartitionedTable(inputData, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir.getPath(), writeInstructions);
 
         // Store the big decimal with the precision and scale consistent with what we write to parquet
         inputData = maybeFixBigDecimal(inputData);
@@ -980,7 +982,7 @@ public void testAllNonPartitioningColumnTypes() {
 
         // Next API we test is passing the partitioned table without any table definition
         final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
-        writeKeyValuePartitionedTable(partitionedTable, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, parentDir.getPath(), writeInstructions);
         fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(inputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
         fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
@@ -996,7 +998,7 @@ public void testAllNonPartitioningColumnTypes() {
                 .collect(Collectors.toList());
         newColumns.add(ColumnDefinition.ofInt("NPC15"));
         final TableDefinition newDefinition = TableDefinition.of(newColumns);
-        writeKeyValuePartitionedTable(inputData, newDefinition, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(inputData, newDefinition, parentDir.getAbsolutePath(), writeInstructions);
         final Table expected = inputData.dropColumns("PC2", "NPC6").updateView("NPC15 = (int)null");
         fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
@@ -1006,7 +1008,7 @@ public void testAllNonPartitioningColumnTypes() {
         FileUtils.deleteRecursively(parentDir);
 
         // Next API we test is passing the partitioned table with an updated table definition
-        writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir, writeInstructions);
+        writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir.getPath(), writeInstructions);
         fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
         assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
         fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index 4da7874ff7a..8f8608c4bae 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -13,7 +13,7 @@
 from deephaven import DHError
 from deephaven.column import Column
 from deephaven.dtypes import DType
-from deephaven.table import Table
+from deephaven.table import Table, PartitionedTable
 from deephaven.experimental import s3
 
 _JParquetTools = jpy.get_type("io.deephaven.parquet.table.ParquetTools")
@@ -280,6 +280,90 @@ def write(
         raise DHError(e, "failed to write to parquet data.") from e
 
 
+def write_key_value_partitioned_table(
+        table: Union[Table, PartitionedTable],
+        destination_dir: str,
+        col_definitions: Optional[List[Column]] = None,
+        col_instructions: Optional[List[ColumnInstruction]] = None,
+        compression_codec_name: Optional[str] = None,
+        max_dictionary_keys: Optional[int] = None,
+        max_dictionary_size: Optional[int] = None,
+        target_page_size: Optional[int] = None,
+        base_name: Optional[str] = None
+) -> None:
+    """ Write a table to a Parquet file in a key-value partitoned format.
+
+    Args:
+        table (Table): the source table or partitioned table
+        destination_dir (str): the destination root directory to store partitioned data in nested format. Non-existing
+            directories are created.
+        col_definitions (Optional[List[Column]]): the column definitions to use, default is None
+        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing, default is None
+        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY
+        max_dictionary_keys (Optional[int]): the maximum dictionary keys allowed, if not specified, defaults to 2^20 (1,048,576)
+        max_dictionary_size (Optional[int]): the maximum dictionary size (in bytes) allowed, defaults to 2^20 (1,048,576)
+        target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
+        base_name (Optional[str]): The base name for the individual partitioned tables, if not specified, defaults to
+            `{uuid}`, so files will have names of the format `<uuid>.parquet` where `uuid` is a randomly generated UUID.
+            Users can provide the following tokens in the base_name:
+            - The token `{uuid}` will be replaced with a random UUID. For example, a base name of
+            "table-{uuid}" will result in files named like "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".
+            - The token `{partitions}` will be replaced with an underscore-delimited, concatenated string of
+            partition values. For example, a base name of "{partitions}-table" will result in files like
+            "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+            partitioning columns.
+            - The token `{i}` will be replaced with an automatically incremented integer for files in a directory. For
+            example, a base name of "table-{i}" will result in files named like "PC=partition1/table-0.parquet",
+            "PC=partition1/table-1.parquet", etc.
+
+    Raises:
+        DHError
+    """
+    try:
+        write_instructions = _build_parquet_instructions(
+            col_instructions=col_instructions,
+            compression_codec_name=compression_codec_name,
+            max_dictionary_keys=max_dictionary_keys,
+            max_dictionary_size=max_dictionary_size,
+            target_page_size=target_page_size,
+            for_read=False,
+        )
+
+        table_definition = None
+        if col_definitions is not None:
+            table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
+
+        if table_definition:
+            if write_instructions:
+                if base_name:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
+                                                                 base_name, write_instructions)
+                else:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
+                                                                 write_instructions)
+            else:
+                if base_name:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
+                                                                 base_name)
+                else:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir)
+        else:
+            if write_instructions:
+                if base_name:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, base_name,
+                                                                 write_instructions)
+                else:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir,
+                                                                 write_instructions)
+            else:
+                if base_name:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, base_name)
+                else:
+                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir)
+    except Exception as e:
+        raise DHError(e, "failed to write to parquet data.") from e
+
+
 def batch_write(
     tables: List[Table],
     paths: List[str],
diff --git a/py/server/tests/test_parquet.py b/py/server/tests/test_parquet.py
index 782048e45f3..c99d651141d 100644
--- a/py/server/tests/test_parquet.py
+++ b/py/server/tests/test_parquet.py
@@ -6,15 +6,17 @@
 import shutil
 import tempfile
 import unittest
+import fnmatch
 
 import pandas
 import pyarrow.parquet
 
 from deephaven import DHError, empty_table, dtypes, new_table
 from deephaven import arrow as dharrow
-from deephaven.column import InputColumn, Column, ColumnType
+from deephaven.column import InputColumn, Column, ColumnType, string_col, int_col
 from deephaven.pandas import to_pandas, to_table
-from deephaven.parquet import write, batch_write, read, delete, ColumnInstruction, ParquetFileLayout
+from deephaven.parquet import (write, batch_write, read, delete, ColumnInstruction, ParquetFileLayout,
+                               write_key_value_partitioned_table)
 from tests.testbase import BaseTestCase
 from deephaven.experimental import s3
 
@@ -113,7 +115,8 @@ def test_crd_with_instructions(self):
 
         # Reading
         with self.subTest(msg="read_table(str)"):
-            table2 = read(path=file_location, col_instructions=[col_inst, col_inst1], file_layout=ParquetFileLayout.SINGLE_FILE)
+            table2 = read(path=file_location, col_instructions=[col_inst, col_inst1],
+                          file_layout=ParquetFileLayout.SINGLE_FILE)
             self.assert_table_equals(table, table2)
 
         # Delete
@@ -576,6 +579,77 @@ def test_read_parquet_from_s3(self):
             read("s3://dh-s3-parquet-test1/multiColFile.parquet", special_instructions=s3_instructions).select()
         # TODO(deephaven-core#5064): Add support for local S3 testing
 
+    def test_writing_partitioned_data(self):
+        source = new_table([
+            string_col("X", ["Aa", "Bb", "Aa", "Cc", "Bb", "Aa", "Bb", "Bb", "Cc"]),
+            string_col("Y", ["M", "N", "O", "N", "P", "M", "O", "P", "M"]),
+            int_col("Number", [55, 76, 20, 130, 230, 50, 73, 137, 214]),
+        ])
+        partitioned_table = source.partition_by(by="X")
+        base_name = "test-{uuid}"
+        max_dictionary_keys = 1024
+
+        root_dir = os.path.join(self.temp_dir.name, "test_partitioned_writing")
+        if os.path.exists(root_dir):
+            shutil.rmtree(root_dir)
+
+        def verify_table_from_disk(table):
+            self.assertTrue(len(table.columns))
+            self.assertTrue(table.columns[0].name == "X")
+            self.assertTrue(table.columns[0].column_type == ColumnType.PARTITIONING)
+            self.assert_table_equals(table.select().sort("X", "Y"), source.sort("X", "Y"))
+
+        def verify_file_names():
+            partition_dir_path = os.path.join(root_dir, 'X=Aa')
+            self.assertTrue(os.path.exists(partition_dir_path))
+            self.assertTrue(len(os.listdir(partition_dir_path)) == 1)
+            parquet_file = os.listdir(partition_dir_path)[0]
+            self.assertTrue(fnmatch.fnmatch(parquet_file, 'test-*.parquet'))
+
+        # Test all different APIs
+        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir, base_name=base_name,
+                                          max_dictionary_keys=max_dictionary_keys)
+        from_disk = read(root_dir)
+        definition = from_disk.columns
+        verify_table_from_disk(from_disk)
+        verify_file_names()
+        from_disk = None
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir,
+                                          max_dictionary_keys=max_dictionary_keys)
+        verify_table_from_disk(read(root_dir))
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir, base_name=base_name)
+        verify_table_from_disk(read(root_dir))
+        verify_file_names()
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir)
+        verify_table_from_disk(read(root_dir))
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
+                                          base_name=base_name, max_dictionary_keys=max_dictionary_keys)
+        verify_table_from_disk(read(root_dir))
+        verify_file_names()
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
+                                          max_dictionary_keys=max_dictionary_keys)
+        verify_table_from_disk(read(root_dir))
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
+                                          base_name=base_name)
+        verify_table_from_disk(read(root_dir))
+        verify_file_names()
+
+        shutil.rmtree(root_dir)
+        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir)
+        verify_table_from_disk(read(root_dir))
+
 
 if __name__ == '__main__':
     unittest.main()

From 10b7e0d073dde71454635cd2240a590a381fd7c3 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 19 Mar 2024 11:03:03 -0500
Subject: [PATCH 33/45] Added more fixes for python

---
 .../io/deephaven/parquet/table/ParquetTools.java |  2 +-
 py/server/deephaven/parquet.py                   | 16 +++++++++++++++-
 py/server/tests/test_parquet.py                  | 16 +++++++++++++---
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 6a8bf2cfc04..e9d6affa735 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -1083,7 +1083,7 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         if (writeInstructions.generateMetadataFiles()) {
             // We insist on writing the metadata file in the same directory as the destination files, thus all
             // destination files should be in the same directory. This also ensures that the file names would be unique.
-            final String firstDestinationDir = destinations[0].getParentFile().getAbsolutePath();
+            final String firstDestinationDir = destinations[0].getAbsoluteFile().getParentFile().getAbsolutePath();
             for (int i = 1; i < destinations.length; i++) {
                 if (!firstDestinationDir.equals(destinations[i].getParentFile().getAbsolutePath())) {
                     throw new IllegalArgumentException("All destination files must be in the same directory for " +
diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index 8f8608c4bae..a40840e50c0 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -43,6 +43,7 @@ def _build_parquet_instructions(
     is_refreshing: bool = False,
     for_read: bool = True,
     force_build: bool = False,
+    generate_metadata_files: Optional[bool] = False,
     special_instructions: Optional[s3.S3Instructions] = None,
 ):
     if not any(
@@ -55,6 +56,7 @@ def _build_parquet_instructions(
             is_legacy_parquet,
             target_page_size is not None,
             is_refreshing,
+            generate_metadata_files is not None,
             special_instructions is not None
         ]
     ):
@@ -92,6 +94,9 @@ def _build_parquet_instructions(
     if is_refreshing:
         builder.setIsRefreshing(is_refreshing)
 
+    if generate_metadata_files:
+        builder.setGenerateMetadataFiles(generate_metadata_files)
+
     if special_instructions is not None:
         builder.setSpecialInstructions(special_instructions.j_object)
 
@@ -234,6 +239,7 @@ def write(
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
+    generate_metadata_files: Optional[bool] = False,
 ) -> None:
     """ Write a table to a Parquet file.
 
@@ -248,6 +254,7 @@ def write(
         max_dictionary_keys (Optional[int]): the maximum dictionary keys allowed, if not specified, defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum dictionary size (in bytes) allowed, defaults to 2^20 (1,048,576)
         target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
+        generate_metadata_files (Optional[bool]): whether to generate _metadata and _common_metadata files, default is False
 
     Raises:
         DHError
@@ -260,6 +267,7 @@ def write(
             max_dictionary_size=max_dictionary_size,
             target_page_size=target_page_size,
             for_read=False,
+            generate_metadata_files=generate_metadata_files,
         )
 
         table_definition = None
@@ -289,7 +297,8 @@ def write_key_value_partitioned_table(
         max_dictionary_keys: Optional[int] = None,
         max_dictionary_size: Optional[int] = None,
         target_page_size: Optional[int] = None,
-        base_name: Optional[str] = None
+        base_name: Optional[str] = None,
+        generate_metadata_files: Optional[bool] = False,
 ) -> None:
     """ Write a table to a Parquet file in a key-value partitoned format.
 
@@ -315,6 +324,7 @@ def write_key_value_partitioned_table(
             - The token `{i}` will be replaced with an automatically incremented integer for files in a directory. For
             example, a base name of "table-{i}" will result in files named like "PC=partition1/table-0.parquet",
             "PC=partition1/table-1.parquet", etc.
+        generate_metadata_files (Optional[bool]): whether to generate _metadata and _common_metadata files, default is False
 
     Raises:
         DHError
@@ -327,6 +337,7 @@ def write_key_value_partitioned_table(
             max_dictionary_size=max_dictionary_size,
             target_page_size=target_page_size,
             for_read=False,
+            generate_metadata_files=generate_metadata_files,
         )
 
         table_definition = None
@@ -374,6 +385,7 @@ def batch_write(
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
     grouping_cols: Optional[List[str]] = None,
+    generate_metadata_files: Optional[bool] = False,
 ):
     """ Writes tables to disk in parquet format to a supplied set of paths.
 
@@ -394,6 +406,7 @@ def batch_write(
         max_dictionary_size (Optional[int]): the maximum dictionary size (in bytes) allowed, defaults to 2^20 (1,048,576)
         target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
         grouping_cols (Optional[List[str]]): the group column names
+        generate_metadata_files (Optional[bool]): whether to generate _metadata and _common_metadata files, default is False
 
     Raises:
         DHError
@@ -406,6 +419,7 @@ def batch_write(
             max_dictionary_size=max_dictionary_size,
             target_page_size=target_page_size,
             for_read=False,
+            generate_metadata_files=generate_metadata_files,
         )
 
         table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
diff --git a/py/server/tests/test_parquet.py b/py/server/tests/test_parquet.py
index c99d651141d..c3e253759e5 100644
--- a/py/server/tests/test_parquet.py
+++ b/py/server/tests/test_parquet.py
@@ -325,7 +325,9 @@ def test_dictionary_encoding(self):
             "someIntColumn = i"
         ])
         # Force "longStringColumn" to use non-dictionary encoding
-        write(dh_table, "data_from_dh.parquet", max_dictionary_size=100)
+        write(dh_table, "data_from_dh.parquet", max_dictionary_size=100, generate_metadata_files=True)
+        self.verify_metadata_files(".")
+
         from_disk = read('data_from_dh.parquet', file_layout=ParquetFileLayout.SINGLE_FILE)
         self.assert_table_equals(dh_table, from_disk)
 
@@ -579,6 +581,12 @@ def test_read_parquet_from_s3(self):
             read("s3://dh-s3-parquet-test1/multiColFile.parquet", special_instructions=s3_instructions).select()
         # TODO(deephaven-core#5064): Add support for local S3 testing
 
+    def verify_metadata_files(self, root_dir):
+        metadata_file_path = os.path.join(root_dir, '_metadata')
+        self.assertTrue(os.path.exists(metadata_file_path))
+        common_metadata_file_path = os.path.join(root_dir, '_common_metadata')
+        self.assertTrue(os.path.exists(common_metadata_file_path))
+
     def test_writing_partitioned_data(self):
         source = new_table([
             string_col("X", ["Aa", "Bb", "Aa", "Cc", "Bb", "Aa", "Bb", "Bb", "Cc"]),
@@ -617,8 +625,9 @@ def verify_file_names():
 
         shutil.rmtree(root_dir)
         write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir,
-                                          max_dictionary_keys=max_dictionary_keys)
+                                          max_dictionary_keys=max_dictionary_keys, generate_metadata_files=True)
         verify_table_from_disk(read(root_dir))
+        self.verify_metadata_files(root_dir)
 
         shutil.rmtree(root_dir)
         write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir, base_name=base_name)
@@ -637,8 +646,9 @@ def verify_file_names():
 
         shutil.rmtree(root_dir)
         write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
-                                          max_dictionary_keys=max_dictionary_keys)
+                                          max_dictionary_keys=max_dictionary_keys, generate_metadata_files=True)
         verify_table_from_disk(read(root_dir))
+        self.verify_metadata_files(root_dir)
 
         shutil.rmtree(root_dir)
         write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,

From 9f7c55e805e6a84c575eee4abd2e7aca6d4b9060 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Thu, 21 Mar 2024 17:32:38 -0500
Subject: [PATCH 34/45] Review with Ryan and Chip

---
 .../parquet/base/ColumnChunkReaderImpl.java   |   2 +-
 .../base/ParquetMetadataFileWriter.java       |   1 -
 .../deephaven/parquet/base/ParquetUtils.java  |   8 +-
 .../parquet/table/ParquetInstructions.java    |  55 ++-
 .../table/ParquetMetadataFileWriterImpl.java  |  64 ++-
 .../deephaven/parquet/table/ParquetTools.java | 396 ++----------------
 .../layout/ParquetMetadataFileLayout.java     |  14 +-
 .../OffsetIndexBasedColumnChunkPageStore.java |  27 +-
 .../table/ParquetTableReadWriteTest.java      |  54 ++-
 py/server/deephaven/parquet.py                | 135 +++---
 py/server/tests/test_parquet.py               |  42 +-
 11 files changed, 270 insertions(+), 528 deletions(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
index fa0714ad36a..aafaecc1fdf 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -75,7 +75,7 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
         this.numRows = numRows;
         this.version = version;
         if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
-            this.columnChunkURI = Path.of(rootURI).resolve(columnChunk.getFile_path()).toUri();
+            this.columnChunkURI = convertToURI(Path.of(rootURI).resolve(columnChunk.getFile_path()), false);
         } else {
             // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
             this.columnChunkURI = rootURI;
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
index 48773dc65da..42a93dc24e1 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetMetadataFileWriter.java
@@ -5,7 +5,6 @@
 
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 
-import java.io.File;
 import java.io.IOException;
 
 /**
diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
index f64eb256bc0..604444ae8be 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetUtils.java
@@ -3,6 +3,7 @@
 //
 package io.deephaven.parquet.base;
 
+import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 
@@ -33,9 +34,10 @@ public static boolean fileNameMatches(final Path path) {
     }
 
     /**
-     * @return the key value derived from the file name, used for storing each file's metadata in the metadata files.
+     * @return the key value derived from the file path, used for storing each file's metadata in the combined
+     *         {@value #METADATA_FILE_NAME} and {@value #COMMON_METADATA_FILE_NAME} files.
      */
-    public static String getKeyForFile(final String fileName) {
-        return "deephaven_per_file_" + fileName;
+    public static String getPerFileMetadataKey(final String filePath) {
+        return "deephaven_per_file_" + filePath.replace(File.separatorChar, '_');
     }
 }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
index 8fdfe446f60..587b50c9b4f 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
@@ -118,6 +118,11 @@ public static int getDefaultTargetPageSize() {
 
     private static final boolean DEFAULT_GENERATE_METADATA_FILES = false;
 
+    static final String UUID_TOKEN = "{uuid}";
+    static final String PARTITIONS_TOKEN = "{partitions}";
+    static final String FILE_INDEX_TOKEN = "{i}";
+    private static final String DEFAULT_BASE_NAME_FOR_PARTITIONED_PARQUET_DATA = UUID_TOKEN;
+
     public ParquetInstructions() {}
 
     public final String getColumnNameFromParquetColumnNameOrDefault(final String parquetColumnName) {
@@ -173,6 +178,14 @@ public final String getColumnNameFromParquetColumnNameOrDefault(final String par
      */
     public abstract boolean generateMetadataFiles();
 
+
+    /**
+     * @return the base name for partitioned parquet data. Check
+     *         {@link Builder#setBaseNameForPartitionedParquetData(String) setBaseNameForPartitionedParquetData} for
+     *         more details about different tokens that can be used in the base name.
+     */
+    public abstract String baseNameForPartitionedParquetData();
+
     @VisibleForTesting
     public static boolean sameColumnNamesAndCodecMappings(final ParquetInstructions i1, final ParquetInstructions i2) {
         if (i1 == EMPTY) {
@@ -252,6 +265,11 @@ public boolean isRefreshing() {
         public boolean generateMetadataFiles() {
             return DEFAULT_GENERATE_METADATA_FILES;
         }
+
+        @Override
+        public String baseNameForPartitionedParquetData() {
+            return DEFAULT_BASE_NAME_FOR_PARTITIONED_PARQUET_DATA;
+        }
     };
 
     private static class ColumnInstructions {
@@ -321,6 +339,7 @@ private static final class ReadOnly extends ParquetInstructions {
         private final boolean isRefreshing;
         private final Object specialInstructions;
         private final boolean generateMetadataFiles;
+        private final String baseNameForPartitionedParquetData;
 
         private ReadOnly(
                 final KeyedObjectHashMap<String, ColumnInstructions> columnNameToInstructions,
@@ -332,7 +351,8 @@ private ReadOnly(
                 final int targetPageSize,
                 final boolean isRefreshing,
                 final Object specialInstructions,
-                final boolean generateMetadataFiles) {
+                final boolean generateMetadataFiles,
+                final String baseNameForPartitionedParquetData) {
             this.columnNameToInstructions = columnNameToInstructions;
             this.parquetColumnNameToInstructions = parquetColumnNameToColumnName;
             this.compressionCodecName = compressionCodecName;
@@ -343,6 +363,7 @@ private ReadOnly(
             this.isRefreshing = isRefreshing;
             this.specialInstructions = specialInstructions;
             this.generateMetadataFiles = generateMetadataFiles;
+            this.baseNameForPartitionedParquetData = baseNameForPartitionedParquetData;
         }
 
         private String getOrDefault(final String columnName, final String defaultValue,
@@ -441,6 +462,11 @@ public boolean generateMetadataFiles() {
             return generateMetadataFiles;
         }
 
+        @Override
+        public String baseNameForPartitionedParquetData() {
+            return baseNameForPartitionedParquetData;
+        }
+
         KeyedObjectHashMap<String, ColumnInstructions> copyColumnNameToInstructions() {
             // noinspection unchecked
             return (columnNameToInstructions == null)
@@ -493,6 +519,7 @@ public static class Builder {
         private boolean isRefreshing = DEFAULT_IS_REFRESHING;
         private Object specialInstructions;
         private boolean generateMetadataFiles = DEFAULT_GENERATE_METADATA_FILES;
+        private String baseNameForPartitionedParquetData = DEFAULT_BASE_NAME_FOR_PARTITIONED_PARQUET_DATA;
 
         public Builder() {}
 
@@ -687,6 +714,30 @@ public Builder setGenerateMetadataFiles(final boolean generateMetadataFiles) {
             return this;
         }
 
+        /**
+         * Set the base name for partitioned parquet data. This is used to generate the file name for partitioned
+         * parquet files, and therefore, this parameter is only used when writing partitioned parquet data. Users can
+         * provide the following tokens to be replaced in the base name:
+         * <ul>
+         * <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for files
+         * in a directory. For example, a base name of "table-{i}" will result in files named like
+         * "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc., where PC is a partitioning
+         * column.</li>
+         * <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
+         * "table-{uuid}" will result in files named like
+         * "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+         * <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated string
+         * of partition values. For example, a base name of "{partitions}-table" will result in files like
+         * "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
+         * partitioning columns.</li>
+         * </ul>
+         * The default value of this parameter is {@value #DEFAULT_BASE_NAME_FOR_PARTITIONED_PARQUET_DATA}.
+         */
+        public Builder setBaseNameForPartitionedParquetData(final String baseNameForPartitionedParquetData) {
+            this.baseNameForPartitionedParquetData = baseNameForPartitionedParquetData;
+            return this;
+        }
+
         public ParquetInstructions build() {
             final KeyedObjectHashMap<String, ColumnInstructions> columnNameToInstructionsOut = columnNameToInstructions;
             columnNameToInstructions = null;
@@ -695,7 +746,7 @@ public ParquetInstructions build() {
             parquetColumnNameToInstructions = null;
             return new ReadOnly(columnNameToInstructionsOut, parquetColumnNameToColumnNameOut, compressionCodecName,
                     maximumDictionaryKeys, maximumDictionarySize, isLegacyParquet, targetPageSize, isRefreshing,
-                    specialInstructions, generateMetadataFiles);
+                    specialInstructions, generateMetadataFiles, baseNameForPartitionedParquetData);
         }
     }
 
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index 477dfc433c2..5357f8f93fc 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -32,11 +32,12 @@
 import static io.deephaven.base.FileUtils.convertToURI;
 import static io.deephaven.parquet.base.ParquetUtils.MAGIC;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
-import static io.deephaven.parquet.base.ParquetUtils.getKeyForFile;
+import static io.deephaven.parquet.base.ParquetUtils.getPerFileMetadataKey;
 
 /**
  * Used to generate a combined {@value ParquetUtils#METADATA_FILE_NAME} and
- * {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file for provided Parquet files.
+ * {@value ParquetUtils#COMMON_METADATA_FILE_NAME} file for provided Parquet files. This class is stateful and therefore
+ * should not be used by multiple threads concurrently.
  */
 final class ParquetMetadataFileWriterImpl implements ParquetMetadataFileWriter {
 
@@ -82,24 +83,6 @@ private static class ParquetFileMetadata {
         }
         this.metadataRootDirAbsPath = metadataRootDir.getAbsoluteFile().toPath();
         final String metadataRootDirAbsPathString = metadataRootDirAbsPath.toString();
-        final File firstDestination = destinations[0];
-        for (int i = 0; i < destinations.length; i++) {
-            final File destination = destinations[i];
-            if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPathString)) {
-                throw new UncheckedDeephavenException("All destinations must be nested under the provided metadata root"
-                        + " directory, provided destination " + destination.getAbsolutePath() + " is not under " +
-                        metadataRootDirAbsPathString);
-            }
-            // TODO How should I change the basename in the API for writeKeyValuePartitioned data for this check?
-            if (i > 0) {
-                // We use filename to generate the key for each file's metadata in the common metadata file, therefore
-                // all files must have unique names.
-                if (destination.getName().equals(firstDestination.getName())) {
-                    throw new UncheckedDeephavenException("When generating common metadata for multiple parquet files, "
-                            + "all files must have unique names, but " + destination.getName() + " is repeated.");
-                }
-            }
-        }
         for (final File destination : destinations) {
             if (!destination.getAbsolutePath().startsWith(metadataRootDirAbsPathString)) {
                 throw new UncheckedDeephavenException("All destinations must be nested under the provided metadata root"
@@ -121,10 +104,7 @@ private static class ParquetFileMetadata {
     }
 
     /**
-     * Add parquet metadata for the provided parquet file the combined metadata file. We store deephaven-specific
-     * metadata for each file individually inside the key-value metadata of the combined metadata file, with keys being
-     * derived from the file names and values being the metadata for the file. Therefore, the provided parquet files
-     * must have unique names.
+     * Add parquet metadata for the provided parquet file the combined metadata file.
      *
      * @param parquetFilePath The parquet file destination path
      * @param metadata The parquet metadata
@@ -170,8 +150,9 @@ private void mergeMetadata() throws IOException {
         for (final ParquetFileMetadata parquetFileMetadata : parquetFileMetadataList) {
             final FileMetaData fileMetaData = parquetFileMetadata.metadata.getFileMetaData();
             mergedSchema = mergeSchemaInto(fileMetaData.getSchema(), mergedSchema);
-            mergeKeyValueMetaData(parquetFileMetadata);
-            mergeBlocksInto(parquetFileMetadata, metadataRootDirAbsPath, mergedBlocks);
+            final String relativePath = getRelativePath(parquetFileMetadata.filePath, metadataRootDirAbsPath);
+            mergeKeyValueMetaData(parquetFileMetadata, relativePath);
+            mergeBlocksInto(parquetFileMetadata, relativePath, mergedBlocks);
             mergedCreatedBy.add(fileMetaData.getCreatedBy());
         }
         if (mergedKeyValueMetaData.size() != parquetFileMetadataList.size()) {
@@ -212,7 +193,8 @@ private static MessageType mergeSchemaInto(final MessageType schema, final Messa
      * well as accumulate the required fields to generate a common table info later once all files are processed.</li>
      * </ul>
      */
-    private void mergeKeyValueMetaData(@NotNull final ParquetFileMetadata parquetFileMetadata) throws IOException {
+    private void mergeKeyValueMetaData(@NotNull final ParquetFileMetadata parquetFileMetadata,
+            @NotNull final String relativePath) throws IOException {
         final Map<String, String> keyValueMetaData =
                 parquetFileMetadata.metadata.getFileMetaData().getKeyValueMetaData();
         for (final Map.Entry<String, String> entry : keyValueMetaData.entrySet()) {
@@ -232,12 +214,11 @@ private void mergeKeyValueMetaData(@NotNull final ParquetFileMetadata parquetFil
                 });
             } else {
                 // Add a separate entry for each file
-                final String fileKey = getKeyForFile(new File(parquetFileMetadata.filePath).getName());
+                final String fileKey = getPerFileMetadataKey(relativePath);
                 // Assuming the keys are unique for each file because file names are unique, verified in the constructor
                 if (mergedKeyValueMetaData.containsKey(fileKey)) {
-                    throw new UncheckedDeephavenException("Could not merge metadata for for file " +
-                            parquetFileMetadata.filePath + " because has conflicting file name with another file. For "
-                            + " generating metadata files, file names should be unique");
+                    throw new IllegalStateException("Could not merge metadata for file " +
+                            parquetFileMetadata.filePath + " because it has conflicting file key: " + fileKey);
                 }
                 mergedKeyValueMetaData.put(fileKey, entry.getValue());
 
@@ -265,21 +246,24 @@ private void mergeKeyValueMetaData(@NotNull final ParquetFileMetadata parquetFil
     }
 
     private static void mergeBlocksInto(final ParquetFileMetadata parquetFileMetadata,
-            final Path metadataRootDirAbsPath, final Collection<BlockMetaData> mergedBlocks) {
-        final Path parquetFileAbsPath = new File(parquetFileMetadata.filePath).getAbsoluteFile().toPath();
-        String fileRelativePathString = metadataRootDirAbsPath.relativize(parquetFileAbsPath).toString();
-        // Remove leading slashes from the relative path
-        int pos = 0;
-        while (pos < fileRelativePathString.length() && fileRelativePathString.charAt(pos) == '/') {
-            pos++;
-        }
-        fileRelativePathString = fileRelativePathString.substring(pos);
+            final String fileRelativePathString, final Collection<BlockMetaData> mergedBlocks) {
         for (final BlockMetaData block : parquetFileMetadata.metadata.getBlocks()) {
             block.setPath(fileRelativePathString);
             mergedBlocks.add(block);
         }
     }
 
+    private static String getRelativePath(final String parquetFilePath, final Path metadataRootDirAbsPath) {
+        final Path parquetFileAbsPath = new File(parquetFilePath).getAbsoluteFile().toPath();
+        final String relativePath = metadataRootDirAbsPath.relativize(parquetFileAbsPath).toString();
+        // Remove leading slashes from the relative path
+        int pos = 0;
+        while (pos < relativePath.length() && relativePath.charAt(pos) == '/') {
+            pos++;
+        }
+        return relativePath.substring(pos);
+    }
+
     private void writeMetadataFile(final ParquetMetadata metadataFooter, final String outputPath) throws IOException {
         final PositionedBufferedOutputStream metadataOutputStream =
                 new PositionedBufferedOutputStream(channelsProvider.getWriteChannel(outputPath, false),
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index e9d6affa735..3559a1b05d9 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -67,6 +67,9 @@
 
 import static io.deephaven.base.FileUtils.convertToURI;
 import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
+import static io.deephaven.parquet.table.ParquetInstructions.FILE_INDEX_TOKEN;
+import static io.deephaven.parquet.table.ParquetInstructions.PARTITIONS_TOKEN;
+import static io.deephaven.parquet.table.ParquetInstructions.UUID_TOKEN;
 import static io.deephaven.parquet.table.ParquetTableWriter.getSchemaForTable;
 import static io.deephaven.parquet.base.ParquetUtils.PARQUET_FILE_EXTENSION;
 import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME;
@@ -85,10 +88,6 @@ private ParquetTools() {}
 
     private static final Logger log = LoggerFactory.getLogger(ParquetTools.class);
 
-    private static final String UUID_TOKEN = "{uuid}";
-    private static final String PARTITIONS_TOKEN = "{partitions}";
-    private static final String FILE_INDEX_TOKEN = "{i}";
-
     /**
      * Reads in a table from a single parquet file, metadata file, or directory with recognized layout. The source
      * provided can be a local file path or a URI to be resolved via the provided
@@ -491,92 +490,18 @@ private static Map<String, ParquetTableWriter.GroupingColumnWritingInfo> groupin
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
-     * table. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid" is a
-     * random UUID. To provide custom file names, use {@link #writeKeyValuePartitionedTable(Table, String, String)}
-     * instead.
-     *
-     * @param sourceTable The table to partition and write
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final String destinationDir) {
-        writeKeyValuePartitionedTable(sourceTable, destinationDir, ParquetInstructions.EMPTY);
-    }
-
-    /**
-     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
-     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
-     * table. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid" is a
-     * random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(Table, String, String, ParquetInstructions)} instead.
-     *
-     * @param sourceTable The table to partition and write
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param writeInstructions Write instructions for customizations while writing
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final String destinationDir,
-            @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(sourceTable, destinationDir, UUID_TOKEN, writeInstructions);
-    }
-
-    /**
-     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
-     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
-     * table.
-     *
-     * @param sourceTable The table to partition and write
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@code {uuid}} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@code {partitions}} will be replaced with an underscore-delimited, concatenated string of
-     *        partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final String destinationDir,
-            @NotNull final String baseName) {
-        writeKeyValuePartitionedTable(sourceTable, destinationDir, baseName, ParquetInstructions.EMPTY);
-    }
-
-    /**
-     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
-     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
-     * table.
+     * table. The generated parquet files will have names of the format provided by
+     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
      *
      * @param sourceTable The table to partition and write
      * @param destinationDir The path to destination root directory to store partitioned data in nested format.
      *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
             @NotNull final String destinationDir,
-            @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(sourceTable, sourceTable.getDefinition(), destinationDir, baseName,
+        writeKeyValuePartitionedTable(sourceTable, sourceTable.getDefinition(), destinationDir,
                 writeInstructions);
     }
 
@@ -584,98 +509,18 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
-     * table definition. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid"
-     * is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, String)} instead.
-     *
-     * @param sourceTable The table to partition and write
-     * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final TableDefinition definition,
-            @NotNull final String destinationDir) {
-        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, ParquetInstructions.EMPTY);
-    }
-
-    /**
-     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
-     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
-     * table definition. The parquet files generated will have names of the format {@code "<uuid>.parquet"} where "uuid"
-     * is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, String, ParquetInstructions)} instead.
-     *
-     * @param sourceTable The table to partition and write
-     * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param writeInstructions Write instructions for customizations while writing
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final TableDefinition definition,
-            @NotNull final String destinationDir,
-            @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, UUID_TOKEN, writeInstructions);
-    }
-
-    /**
-     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
-     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
-     * table definition.
-     *
-     * @param sourceTable The table to partition and write
-     * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
-            @NotNull final TableDefinition definition,
-            @NotNull final String destinationDir,
-            @NotNull final String baseName) {
-        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, baseName, ParquetInstructions.EMPTY);
-    }
-
-    /**
-     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
-     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
-     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
-     * table definition.
+     * table definition. The generated parquet files will have names of the format provided by
+     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
      *
      * @param sourceTable The table to partition and write
      * @param definition table definition to use (instead of the one implied by the table itself)
      * @param destinationDir The path to destination root directory to store partitioned data in nested format.
      *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
             @NotNull final TableDefinition definition,
             @NotNull final String destinationDir,
-            @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final List<ColumnDefinition<?>> partitioningColumns = definition.getPartitioningColumns();
         if (partitioningColumns.isEmpty()) {
@@ -689,236 +534,77 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
         final TableDefinition leafDefinition =
                 getNonKeyTableDefinition(new HashSet<>(Arrays.asList(partitioningColNames)), definition);
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
-                baseName, writeInstructions);
-    }
-
-    /**
-     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
-     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
-     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String)} instead.
-     *
-     * @param partitionedTable The partitioned table to write
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final String destinationDir) {
-        writeKeyValuePartitionedTable(partitionedTable, destinationDir, ParquetInstructions.EMPTY);
-    }
-
-    /**
-     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
-     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
-     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String, ParquetInstructions)} instead.
-     *
-     * @param partitionedTable The partitioned table to write
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param writeInstructions Write instructions for customizations while writing
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final String destinationDir,
-            @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(partitionedTable, destinationDir, UUID_TOKEN, writeInstructions);
-    }
-
-    /**
-     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
-     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
-     *
-     * @param partitionedTable The partitioned table to write
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
-     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final String destinationDir,
-            @NotNull final String baseName) {
-        writeKeyValuePartitionedTable(partitionedTable, destinationDir, baseName, ParquetInstructions.EMPTY);
+                writeInstructions, Optional.of(sourceTable));
     }
 
     /**
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The generated parquet files will have
+     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
      *
      * @param partitionedTable The partitioned table to write
      * @param destinationDir The path to destination root directory to store partitioned data in nested format.
      *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
-     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final String destinationDir,
-            @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(),
                 partitionedTable.table().getDefinition());
         final TableDefinition leafDefinition = getNonKeyTableDefinition(partitionedTable.keyColumnNames(),
                 partitionedTable.constituentDefinition());
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
-                baseName, writeInstructions);
-    }
-
-    /**
-     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
-     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
-     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)}
-     * instead.
-     *
-     * @param partitionedTable The partitioned table to write
-     * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final TableDefinition definition,
-            @NotNull final String destinationDir) {
-        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, ParquetInstructions.EMPTY);
-    }
-
-    /**
-     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
-     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The parquet files generated will have
-     * names of the format {@code "<uuid>.parquet"} where "uuid" is a random UUID. To provide custom file names, use
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)}
-     * instead.
-     *
-     * @param partitionedTable The partitioned table to write
-     * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param writeInstructions Write instructions for customizations while writing
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final TableDefinition definition,
-            @NotNull final String destinationDir,
-            @NotNull final ParquetInstructions writeInstructions) {
-        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, UUID_TOKEN, writeInstructions);
-    }
-
-    /**
-     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
-     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
-     *
-     * @param partitionedTable The partitioned table to write
-     * @param definition table definition to use (instead of the one implied by the table itself)
-     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
-     *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
-     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
-     */
-    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
-            @NotNull final TableDefinition definition,
-            @NotNull final String destinationDir,
-            @NotNull final String baseName) {
-        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, baseName,
-                ParquetInstructions.EMPTY);
+                writeInstructions, Optional.empty());
     }
 
     /**
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
-     * {@link Table#partitionBy(String...) partitionBy} on the required columns.
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The generated parquet files will have
+     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
      *
      * @param partitionedTable The partitioned table to write
      * @param definition table definition to use (instead of the one implied by the table itself)
      * @param destinationDir The path to destination root directory to store partitioned data in nested format.
      *        Non-existing directories are created.
-     * @param baseName The base name for the individual partitioned tables. Users can provide the following tokens to be
-     *        replaced in the base name:
-     *        <ul>
-     *        <li>The token {@value #FILE_INDEX_TOKEN} will be replaced with an automatically incremented integer for
-     *        files in a directory. For example, a base name of "table-{i}" will result in files named like
-     *        "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc.</li>
-     *        <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-     *        "table-{uuid}" will result in files named like
-     *        "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
-     *        <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated
-     *        string of partition values. For example, a base name of "{partitions}-table" will result in files like
-     *        "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-     *        partitioning columns.</li>
-     *        </ul>
      * @param writeInstructions Write instructions for customizations while writing
      */
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition definition,
             @NotNull final String destinationDir,
-            @NotNull final String baseName,
             @NotNull final ParquetInstructions writeInstructions) {
         final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
         final TableDefinition leafDefinition = getNonKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
-                baseName, writeInstructions);
+                writeInstructions, Optional.empty());
     }
 
     /**
      * Write a partitioned table to disk in a key=value partitioning format with the already computed definition for the
      * key table and leaf table.
-     * <p>
-     * Check {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String, ParquetInstructions)},
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)},
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, String, ParquetInstructions)}}, or
-     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, String, ParquetInstructions)}
-     * for more details.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param keyTableDefinition The definition for key columns
+     * @param leafDefinition The definition for leaf parquet files to be written
+     * @param destinationRoot The path to destination root directory to store partitioned data in nested format
+     * @param writeInstructions Write instructions for customizations while writing
+     * @param sourceTable The optional source table, provided when user provides a merged source table to write, like in
+     *        {@link #writeKeyValuePartitionedTable(Table, String, ParquetInstructions)} and
+     *        {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, ParquetInstructions)}
      */
     private static void writeKeyValuePartitionedTableImpl(@NotNull final PartitionedTable partitionedTable,
             @NotNull final TableDefinition keyTableDefinition,
             @NotNull final TableDefinition leafDefinition,
             @NotNull final String destinationRoot,
-            @NotNull final String baseName,
-            @NotNull final ParquetInstructions writeInstructions) {
+            @NotNull final ParquetInstructions writeInstructions,
+            @NotNull final Optional<Table> sourceTable) {
         if (leafDefinition.numColumns() == 0) {
             throw new IllegalArgumentException("Cannot write a partitioned parquet table without any non-partitioning "
                     + "columns");
         }
+        final String baseName = writeInstructions.baseNameForPartitionedParquetData();
         final boolean hasPartitionInName = baseName.contains(PARTITIONS_TOKEN);
         final boolean hasIndexInName = baseName.contains(FILE_INDEX_TOKEN);
         final boolean hasUUIDInName = baseName.contains(UUID_TOKEN);
@@ -927,12 +613,6 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
                     "Cannot write a partitioned parquet table with non-unique keys without {i} or {uuid} in the base " +
                             "name because there can be multiple partitions with the same key values");
         }
-        if (writeInstructions.generateMetadataFiles()) {
-            if (!hasUUIDInName && !hasPartitionInName) {
-                throw new IllegalArgumentException("For generating metadata files, file names for the parquet " +
-                        "files should be unique. Try passing tokens like {uuid} in the file base name");
-            }
-        }
         // Note that there can be multiple constituents with the same key values, so cannot directly use the
         // partitionedTable.constituentFor(keyValues) method, and we need to group them together
         final String[] partitioningColumnNames = partitionedTable.keyColumnNames().toArray(String[]::new);
@@ -964,7 +644,7 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
             while (constituentIterator.hasNext()) {
                 final ObjectVector<? extends Table> constituentVector = constituentIterator.next();
                 final List<String> partitionStrings = partitionStringsList.get(row);
-                final File relativePath = new File(destinationRoot, String.join("/", partitionStrings));
+                final File relativePath = new File(destinationRoot, String.join(File.separator, partitionStrings));
                 int count = 0;
                 for (final Table constituent : constituentVector) {
                     String filename = baseName;
@@ -995,16 +675,10 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
             partitioningColumnsSchema = null;
         }
         final Map<String, Map<ParquetCacheTags, Object>> computedCache =
-                buildComputedCache(partitionedTable::merge, leafDefinition);
-        writeParquetTablesImpl(
-                partitionedData.toArray(Table[]::new),
-                leafDefinition,
-                writeInstructions,
-                destinations.toArray(File[]::new),
-                leafDefinition.getGroupingColumnNamesArray(),
-                partitioningColumnsSchema,
-                new File(destinationRoot),
-                computedCache);
+                buildComputedCache(() -> sourceTable.orElseGet(partitionedTable::merge), leafDefinition);
+        writeParquetTablesImpl(partitionedData.toArray(Table[]::new), leafDefinition, writeInstructions,
+                destinations.toArray(File[]::new), leafDefinition.getGroupingColumnNamesArray(),
+                partitioningColumnsSchema, new File(destinationRoot), computedCache);
     }
 
     /**
@@ -1082,7 +756,7 @@ public static void writeParquetTables(@NotNull final Table[] sources,
         final File metadataRootDir;
         if (writeInstructions.generateMetadataFiles()) {
             // We insist on writing the metadata file in the same directory as the destination files, thus all
-            // destination files should be in the same directory. This also ensures that the file names would be unique.
+            // destination files should be in the same directory.
             final String firstDestinationDir = destinations[0].getAbsoluteFile().getParentFile().getAbsolutePath();
             for (int i = 1; i < destinations.length; i++) {
                 if (!firstDestinationDir.equals(destinations[i].getParentFile().getAbsolutePath())) {
@@ -1097,8 +771,8 @@ public static void writeParquetTables(@NotNull final Table[] sources,
 
         final Map<String, Map<ParquetCacheTags, Object>> computedCache =
                 buildComputedCache(() -> PartitionedTableFactory.ofTables(definition, sources).merge(), definition);
-        // We do not have any additional common schema here. Schema for all columns will be generated at the time of
-        // writing the parquet files and merged to generate the metadata files.
+        // We do not have any additional schema for partitioning columns in this case. Schema for all columns will be
+        // generated at the time of writing the parquet files and merged to generate the metadata files.
         writeParquetTablesImpl(sources, definition, writeInstructions, destinations, groupingColumns,
                 null, metadataRootDir, computedCache);
     }
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
index 1e0014ff86c..af5c6da54fd 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
@@ -41,7 +41,7 @@
 import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_FILE_NAME;
 import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY;
-import static io.deephaven.parquet.base.ParquetUtils.getKeyForFile;
+import static io.deephaven.parquet.base.ParquetUtils.getPerFileMetadataKey;
 import static java.util.stream.Collectors.toMap;
 
 /**
@@ -204,7 +204,7 @@ public ParquetMetadataFileLayout(
             final ParquetTableLocationKey tlk = new ParquetTableLocationKey(partitionFile,
                     partitionOrder.getAndIncrement(), partitions, inputInstructions);
             tlk.setFileReader(metadataFileReader);
-            tlk.setMetadata(getParquetMetadataForFile(partitionFile, metadataFileMetadata));
+            tlk.setMetadata(getParquetMetadataForFile(filePathString, metadataFileMetadata));
             tlk.setRowGroupIndices(rowGroupIndices);
             return tlk;
         }).collect(Collectors.toList());
@@ -214,16 +214,16 @@ public ParquetMetadataFileLayout(
      * This method takes the {@link ParquetMetadata} from the metadata file, extracts the key-value metadata specific to
      * the provided file, and creates a new {@link ParquetMetadata} for this file.
      *
-     * @param parquetFile The parquet file to read metadata for
+     * @param parquetFileRelativePath The parquet file path relative to the root directory containing the metadata file
      * @param metadataFileMetadata The overall metadata in the metadata file
      */
-    private static ParquetMetadata getParquetMetadataForFile(@NotNull final File parquetFile,
+    private static ParquetMetadata getParquetMetadataForFile(@NotNull final String parquetFileRelativePath,
             @NotNull final ParquetMetadata metadataFileMetadata) {
         final String fileMetadataString = metadataFileMetadata.getFileMetaData().getKeyValueMetaData()
-                .get(getKeyForFile(parquetFile.getName()));
+                .get(getPerFileMetadataKey(parquetFileRelativePath));
         final ParquetMetadata fileMetadata;
         if (fileMetadataString != null) {
-            // Create a new file metadata object using the key-value metadata for that file
+            // Create a new file metadata object using the key-value metadata for this file
             final Map<String, String> keyValueMetadata = Map.of(METADATA_KEY, fileMetadataString);
             fileMetadata = new ParquetMetadata(
                     new FileMetaData(metadataFileMetadata.getFileMetaData().getSchema(),
@@ -231,7 +231,7 @@ private static ParquetMetadata getParquetMetadataForFile(@NotNull final File par
                             metadataFileMetadata.getFileMetaData().getCreatedBy()),
                     metadataFileMetadata.getBlocks());
         } else {
-            // File specific metadata not found, the the metadata file's metadata
+            // File specific metadata not found, use the metadata file's metadata
             fileMetadata = metadataFileMetadata;
         }
         return fileMetadata;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
index b2a070a8eb0..650554d3d79 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/OffsetIndexBasedColumnChunkPageStore.java
@@ -59,15 +59,11 @@ private static final class PageState<ATTR extends Any> {
             final long mask,
             @NotNull final ToPage<ATTR, ?> toPage) throws IOException {
         super(pageCache, columnChunkReader, mask, toPage);
-        isInitialized = false;
         numPages = NUM_PAGES_NOT_INITIALIZED;
         fixedPageSize = PAGE_SIZE_NOT_FIXED;
-        offsetIndex = null;
-        pageStates = null;
-        columnPageDirectAccessor = null;
     }
 
-    private void initialize(@Nullable final FillContext fillContext) {
+    private void ensureInitialized(@Nullable final FillContext fillContext) {
         if (isInitialized) {
             return;
         }
@@ -162,20 +158,13 @@ private ChunkPage<ATTR> getPageImpl(@Nullable FillContext fillContext, int pageN
 
     @Override
     @NotNull
-    public ChunkPage<ATTR> getPageContaining(@Nullable final FillContext fillContext, long rowKey) {
-        if (isInitialized) {
-            return getPageContainingImpl(fillContext, rowKey);
-        }
-        // We need to initialize this class
-        if (fillContext != null) {
-            initialize(fillContext);
-            return getPageContainingImpl(fillContext, rowKey);
-        }
-        // Create a fill context and use it to initialize this class as well as read the page
-        // TODO What should be the chunk capacity here?
-        try (final FillContext fillContext1 = makeFillContext(1, null)) {
-            initialize(fillContext1);
-            return getPageContainingImpl(fillContext1, rowKey);
+    public ChunkPage<ATTR> getPageContaining(@Nullable final FillContext fillContext, final long rowKey) {
+        // We don't really use chunk capacity in our FillContext. In practice, however, this method is only invoked with
+        // a null FillContext for single-element "get" methods.
+        try (final FillContext allocatedFillContext = fillContext != null ? null : makeFillContext(1, null)) {
+            final FillContext fillContextToUse = fillContext != null ? fillContext : allocatedFillContext;
+            ensureInitialized(fillContextToUse);
+            return getPageContainingImpl(fillContextToUse, rowKey);
         }
     }
 
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 2602bee676b..f30203a5235 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -617,15 +617,20 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
+                .setBaseNameForPartitionedParquetData("data")
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), "{partitions}-data", writeInstructions);
+        writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), writeInstructions);
+
+        // Verify that metadata files are generated
+        assertTrue(new File(parentDir, "_common_metadata").exists());
+        assertTrue(new File(parentDir, "_metadata").exists());
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
             for (int PC2 = 0; PC2 <= 1; PC2++) {
                 final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
                 assertTrue(dir.exists() && dir.isDirectory());
-                final File dataFile = new File(dir, "PC1=" + PC1 + "_PC2=" + PC2 + "-data.parquet");
+                final File dataFile = new File(dir, "data.parquet");
                 assertTrue(dataFile.exists() && dataFile.isFile());
             }
         }
@@ -638,6 +643,29 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         final Table fromDiskWithMetadata = readTable(commonMetadata);
         fromDiskWithMetadata.where("I == 3").select();
         assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
+
+        // Write the same table without generating metadata files
+        final File parentDirWithoutMetadata = new File(rootFile, "writeKeyValuePartitionedDataWithoutMetadata");
+        writeKeyValuePartitionedTable(inputData, parentDirWithoutMetadata.getAbsolutePath(), EMPTY);
+
+        // Verify that no metadata files are generated
+        assertFalse(new File(parentDirWithoutMetadata, "_common_metadata").exists());
+        assertFalse(new File(parentDirWithoutMetadata, "_metadata").exists());
+
+        // Verify that the partitioned data exists
+        for (int PC1 = 0; PC1 <= 2; PC1++) {
+            for (int PC2 = 0; PC2 <= 1; PC2++) {
+                final File dir = new File(parentDirWithoutMetadata, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
+                assertTrue(dir.exists() && dir.isDirectory());
+                final File[] fileList = dir.listFiles();
+                for (final File dataFile : fileList) {
+                    // hidden grouping data
+                    assertTrue(dataFile.isHidden() || dataFile.getName().endsWith(".parquet"));
+                }
+            }
+        }
+        final Table fromDiskWithoutMetadata = readKeyValuePartitionedTable(parentDirWithoutMetadata, EMPTY);
+        assertTableEquals(fromDisk, fromDiskWithoutMetadata);
     }
 
     @Test
@@ -653,9 +681,10 @@ public void writeKeyValuePartitionedDataWithNoNonPartitioningColumnsTest() {
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
+                .setBaseNameForPartitionedParquetData("data")
                 .build();
         try {
-            writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), "data", writeInstructions);
+            writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), writeInstructions);
             fail("Expected exception when writing the partitioned table with no non-partitioning columns");
         } catch (final RuntimeException expected) {
         }
@@ -676,20 +705,25 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         final PartitionedTable partitionedTableWithDuplicatedKeys = PartitionedTableFactory.of(internalTableDuplicated);
         assertFalse(partitionedTableWithDuplicatedKeys.uniqueKeys());
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataWithNonUniqueKeys");
-        final ParquetInstructions writeInstructions = ParquetInstructions.builder()
+        ParquetInstructions writeInstructions = ParquetInstructions.builder()
                 .setGenerateMetadataFiles(true)
+                .setBaseNameForPartitionedParquetData("{partitions}-data")
                 .build();
         try {
             writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(),
-                    "{partitions}-data", writeInstructions);
+                    writeInstructions);
             fail("Expected exception when writing the partitioned table with non-unique keys without {i} or {uuid} in "
                     + "base name");
         } catch (final RuntimeException expected) {
         }
 
         // Write the partitioned table with non-unique keys with {i} in the base name
+        writeInstructions = ParquetInstructions.builder()
+                .setGenerateMetadataFiles(true)
+                .setBaseNameForPartitionedParquetData("{partitions}-data-{i}")
+                .build();
         writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(),
-                "{partitions}-data-{i}", writeInstructions);
+                writeInstructions);
 
         // Verify that the partitioned data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
@@ -714,7 +748,11 @@ public void writeKeyValuePartitionedDataWithNonUniqueKeys() {
         FileUtils.deleteRecursively(parentDir);
 
         // Write the partitioned table with non-unique keys with {uuid} in the base name
-        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(), "data-{uuid}",
+        writeInstructions = ParquetInstructions.builder()
+                .setGenerateMetadataFiles(true)
+                .setBaseNameForPartitionedParquetData("data-{uuid}")
+                .build();
+        writeKeyValuePartitionedTable(partitionedTableWithDuplicatedKeys, parentDir.getAbsolutePath(),
                 writeInstructions);
 
         // Verify that the partitioned data exists with uuid in names
@@ -965,7 +1003,7 @@ public void testAllNonPartitioningColumnTypes() {
                 .setGenerateMetadataFiles(true)
                 .build();
 
-        // First API we test is passing the table directly without any table definition
+        // The First API we test is passing the table directly without any table definition
         writeKeyValuePartitionedTable(inputData, parentDir.getPath(), writeInstructions);
 
         // Store the big decimal with the precision and scale consistent with what we write to parquet
diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index a40840e50c0..f7799a46721 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -43,7 +43,8 @@ def _build_parquet_instructions(
     is_refreshing: bool = False,
     for_read: bool = True,
     force_build: bool = False,
-    generate_metadata_files: Optional[bool] = False,
+    generate_metadata_files: Optional[bool] = None,
+    base_name: Optional[str] = None,
     special_instructions: Optional[s3.S3Instructions] = None,
 ):
     if not any(
@@ -57,10 +58,11 @@ def _build_parquet_instructions(
             target_page_size is not None,
             is_refreshing,
             generate_metadata_files is not None,
+            base_name is not None,
             special_instructions is not None
         ]
     ):
-        return None
+        return _JParquetInstructions.EMPTY
 
     builder = _JParquetInstructions.builder()
     if col_instructions is not None:
@@ -97,6 +99,9 @@ def _build_parquet_instructions(
     if generate_metadata_files:
         builder.setGenerateMetadataFiles(generate_metadata_files)
 
+    if base_name:
+        builder.setBaseNameForPartitionedParquetData(base_name)
+
     if special_instructions is not None:
         builder.setSpecialInstructions(special_instructions.j_object)
 
@@ -149,8 +154,8 @@ def read(
 
     Args:
         path (str): the file or directory to examine
-        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while reading, None by
-            default.
+        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while reading particular
+            columns, default is None, which means no specialization for any column
         is_legacy_parquet (bool): if the parquet data is legacy
         is_refreshing (bool): if the parquet data represents a refreshing source
         file_layout (Optional[ParquetFileLayout]): the parquet file layout, by default None. When None, the layout is
@@ -239,22 +244,30 @@ def write(
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
-    generate_metadata_files: Optional[bool] = False,
+    generate_metadata_files: Optional[bool] = None,
 ) -> None:
     """ Write a table to a Parquet file.
 
     Args:
         table (Table): the source table
         path (str): the destination file path; the file name should end in a ".parquet" extension. If the path
-            includes non-existing directories they are created. If there is an error, any intermediate directories
+            includes any non-existing directories, they are created. If there is an error, any intermediate directories
             previously created are removed; note this makes this method unsafe for concurrent use
-        col_definitions (Optional[List[Column]]): the column definitions to use, default is None
-        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing, default is None
-        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY
-        max_dictionary_keys (Optional[int]): the maximum dictionary keys allowed, if not specified, defaults to 2^20 (1,048,576)
-        max_dictionary_size (Optional[int]): the maximum dictionary size (in bytes) allowed, defaults to 2^20 (1,048,576)
+        col_definitions (Optional[List[Column]]): the column definitions to use for writing, instead of the definitions
+            implied by the table. Default is None, which means use the column definitions implied by the table
+        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing particular
+            columns, default is None, which means no specialization for any column
+        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY.
+            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "LZ4_RAW", "ZSTD".
+        max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
+            before switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
+        max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
+            switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
         target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
-        generate_metadata_files (Optional[bool]): whether to generate _metadata and _common_metadata files, default is False
+        generate_metadata_files (Optional[bool]): whether to generate parquet _metadata and _common_metadata files,
+            defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
+            files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
+            files.
 
     Raises:
         DHError
@@ -275,20 +288,14 @@ def write(
             table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
 
         if table_definition:
-            if write_instructions:
-                _JParquetTools.writeTable(table.j_table, path, table_definition, write_instructions)
-            else:
-                _JParquetTools.writeTable(table.j_table, _JFile(path), table_definition)
+            _JParquetTools.writeTable(table.j_table, path, table_definition, write_instructions)
         else:
-            if write_instructions:
-                _JParquetTools.writeTable(table.j_table, _JFile(path), write_instructions)
-            else:
-                _JParquetTools.writeTable(table.j_table, path)
+            _JParquetTools.writeTable(table.j_table, _JFile(path), write_instructions)
     except Exception as e:
         raise DHError(e, "failed to write to parquet data.") from e
 
 
-def write_key_value_partitioned_table(
+def write_partitioned(
         table: Union[Table, PartitionedTable],
         destination_dir: str,
         col_definitions: Optional[List[Column]] = None,
@@ -298,19 +305,28 @@ def write_key_value_partitioned_table(
         max_dictionary_size: Optional[int] = None,
         target_page_size: Optional[int] = None,
         base_name: Optional[str] = None,
-        generate_metadata_files: Optional[bool] = False,
+        generate_metadata_files: Optional[bool] = None,
 ) -> None:
-    """ Write a table to a Parquet file in a key-value partitoned format.
+    """ Write table to disk in parquet format with the partitioning columns written as "key=value" format in a nested
+    directory structure. For example, for a partitioned column "date", we will have a directory structure like
+    "date=2021-01-01/<base_name>.parquet", "date=2021-01-02/<base_name>.parquet", etc. where "2021-01-01" and
+    "2021-01-02" are the partition values and "<base_name>" is passed as an optional parameter. All the necessary
+    subdirectories are created if they do not exist.
 
     Args:
         table (Table): the source table or partitioned table
-        destination_dir (str): the destination root directory to store partitioned data in nested format. Non-existing
-            directories are created.
-        col_definitions (Optional[List[Column]]): the column definitions to use, default is None
-        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing, default is None
-        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY
-        max_dictionary_keys (Optional[int]): the maximum dictionary keys allowed, if not specified, defaults to 2^20 (1,048,576)
-        max_dictionary_size (Optional[int]): the maximum dictionary size (in bytes) allowed, defaults to 2^20 (1,048,576)
+        destination_dir (str): The path to destination root directory in which the partitioned parquet data will be stored
+            in a nested directory structure format. Non-existing directories in the provided path will be created.
+        col_definitions (Optional[List[Column]]): the column definitions to use for writing, instead of the definitions
+            implied by the table. Default is None, which means use the column definitions implied by the table
+        col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing particular
+            columns, default is None, which means no specialization for any column
+        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY.
+            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "LZ4_RAW", "ZSTD".
+        max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
+            before switching to non-dictionary encoding; never evaluated for non-String columns , defaults to 2^20 (1,048,576)
+        max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
+            switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
         target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
         base_name (Optional[str]): The base name for the individual partitioned tables, if not specified, defaults to
             `{uuid}`, so files will have names of the format `<uuid>.parquet` where `uuid` is a randomly generated UUID.
@@ -318,13 +334,16 @@ def write_key_value_partitioned_table(
             - The token `{uuid}` will be replaced with a random UUID. For example, a base name of
             "table-{uuid}" will result in files named like "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".
             - The token `{partitions}` will be replaced with an underscore-delimited, concatenated string of
-            partition values. For example, a base name of "{partitions}-table" will result in files like
-            "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
-            partitioning columns.
+            partition values. For example, for a base name of "{partitions}-table" and partitioning columns "PC1" and
+            "PC2", the file name is generated by concatenating the partition values "PC1=pc1" and "PC2=pc2"
+            with an underscore followed by "-table.parquet", like "PC1=pc1_PC2=pc2-table.parquet".
             - The token `{i}` will be replaced with an automatically incremented integer for files in a directory. For
             example, a base name of "table-{i}" will result in files named like "PC=partition1/table-0.parquet",
             "PC=partition1/table-1.parquet", etc.
-        generate_metadata_files (Optional[bool]): whether to generate _metadata and _common_metadata files, default is False
+        generate_metadata_files (Optional[bool]): whether to generate parquet _metadata and _common_metadata files,
+            defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
+            files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
+            files.
 
     Raises:
         DHError
@@ -338,6 +357,7 @@ def write_key_value_partitioned_table(
             target_page_size=target_page_size,
             for_read=False,
             generate_metadata_files=generate_metadata_files,
+            base_name=base_name,
         )
 
         table_definition = None
@@ -345,32 +365,10 @@ def write_key_value_partitioned_table(
             table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
 
         if table_definition:
-            if write_instructions:
-                if base_name:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
-                                                                 base_name, write_instructions)
-                else:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
-                                                                 write_instructions)
-            else:
-                if base_name:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
-                                                                 base_name)
-                else:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir)
+            _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
+                                                         write_instructions)
         else:
-            if write_instructions:
-                if base_name:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, base_name,
-                                                                 write_instructions)
-                else:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir,
-                                                                 write_instructions)
-            else:
-                if base_name:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, base_name)
-                else:
-                    _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir)
+            _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, write_instructions)
     except Exception as e:
         raise DHError(e, "failed to write to parquet data.") from e
 
@@ -385,7 +383,7 @@ def batch_write(
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
     grouping_cols: Optional[List[str]] = None,
-    generate_metadata_files: Optional[bool] = False,
+    generate_metadata_files: Optional[bool] = None,
 ):
     """ Writes tables to disk in parquet format to a supplied set of paths.
 
@@ -399,14 +397,21 @@ def batch_write(
         paths (List[str]): the destinations paths. Any non existing directories in the paths provided are
             created. If there is an error, any intermediate directories previously created are removed; note this makes
             this method unsafe for concurrent use
-        col_definitions (List[Column]): the column definitions to use
+        col_definitions (Optional[List[Column]]): the column definitions to use for writing, instead of the definitions
+            implied by the table. Default is None, which means use the column definitions implied by the table
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing
-        compression_codec_name (Optional[str]): the compression codec to use, if not specified, defaults to SNAPPY
-        max_dictionary_keys (Optional[int]): the maximum dictionary keys allowed, if not specified, defaults to 2^20 (1,048,576)
-        max_dictionary_size (Optional[int]): the maximum dictionary size (in bytes) allowed, defaults to 2^20 (1,048,576)
+        compression_codec_name (Optional[str]): the compression codec to use, if not specified, defaults to SNAPPY.
+            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "LZ4_RAW", "ZSTD".
+        max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
+            before switching to non-dictionary encoding; never evaluated for non-String columns , defaults to 2^20 (1,048,576)
+        max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
+            switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
         target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
         grouping_cols (Optional[List[str]]): the group column names
-        generate_metadata_files (Optional[bool]): whether to generate _metadata and _common_metadata files, default is False
+        generate_metadata_files (Optional[bool]): whether to generate parquet _metadata and _common_metadata files,
+            defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
+            files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
+            files.
 
     Raises:
         DHError
diff --git a/py/server/tests/test_parquet.py b/py/server/tests/test_parquet.py
index c3e253759e5..6e512e88aad 100644
--- a/py/server/tests/test_parquet.py
+++ b/py/server/tests/test_parquet.py
@@ -16,7 +16,7 @@
 from deephaven.column import InputColumn, Column, ColumnType, string_col, int_col
 from deephaven.pandas import to_pandas, to_table
 from deephaven.parquet import (write, batch_write, read, delete, ColumnInstruction, ParquetFileLayout,
-                               write_key_value_partitioned_table)
+                               write_partitioned)
 from tests.testbase import BaseTestCase
 from deephaven.experimental import s3
 
@@ -339,6 +339,12 @@ def test_dictionary_encoding(self):
         self.assertTrue((metadata.row_group(0).column(2).path_in_schema == 'someIntColumn') &
                         ('RLE_DICTIONARY' not in str(metadata.row_group(0).column(2).encodings)))
 
+    def verify_metadata_files(self, root_dir):
+        metadata_file_path = os.path.join(root_dir, '_metadata')
+        self.assertTrue(os.path.exists(metadata_file_path))
+        common_metadata_file_path = os.path.join(root_dir, '_common_metadata')
+        self.assertTrue(os.path.exists(common_metadata_file_path))
+
     def test_dates_and_time(self):
         dh_table = empty_table(10000).update(formulas=[
             "someDateColumn = i % 10 == 0 ? null : java.time.LocalDate.ofEpochDay(i)",
@@ -581,13 +587,7 @@ def test_read_parquet_from_s3(self):
             read("s3://dh-s3-parquet-test1/multiColFile.parquet", special_instructions=s3_instructions).select()
         # TODO(deephaven-core#5064): Add support for local S3 testing
 
-    def verify_metadata_files(self, root_dir):
-        metadata_file_path = os.path.join(root_dir, '_metadata')
-        self.assertTrue(os.path.exists(metadata_file_path))
-        common_metadata_file_path = os.path.join(root_dir, '_common_metadata')
-        self.assertTrue(os.path.exists(common_metadata_file_path))
-
-    def test_writing_partitioned_data(self):
+    def test_write_partitioned_data(self):
         source = new_table([
             string_col("X", ["Aa", "Bb", "Aa", "Cc", "Bb", "Aa", "Bb", "Bb", "Cc"]),
             string_col("Y", ["M", "N", "O", "N", "P", "M", "O", "P", "M"]),
@@ -615,8 +615,8 @@ def verify_file_names():
             self.assertTrue(fnmatch.fnmatch(parquet_file, 'test-*.parquet'))
 
         # Test all different APIs
-        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir, base_name=base_name,
-                                          max_dictionary_keys=max_dictionary_keys)
+        write_partitioned(partitioned_table, destination_dir=root_dir, base_name=base_name,
+                          max_dictionary_keys=max_dictionary_keys)
         from_disk = read(root_dir)
         definition = from_disk.columns
         verify_table_from_disk(from_disk)
@@ -624,40 +624,40 @@ def verify_file_names():
         from_disk = None
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir,
-                                          max_dictionary_keys=max_dictionary_keys, generate_metadata_files=True)
+        write_partitioned(partitioned_table, destination_dir=root_dir,
+                          max_dictionary_keys=max_dictionary_keys, generate_metadata_files=True)
         verify_table_from_disk(read(root_dir))
         self.verify_metadata_files(root_dir)
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir, base_name=base_name)
+        write_partitioned(partitioned_table, destination_dir=root_dir, base_name=base_name)
         verify_table_from_disk(read(root_dir))
         verify_file_names()
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(partitioned_table, destination_dir=root_dir)
+        write_partitioned(partitioned_table, destination_dir=root_dir)
         verify_table_from_disk(read(root_dir))
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
-                                          base_name=base_name, max_dictionary_keys=max_dictionary_keys)
+        write_partitioned(source, col_definitions=definition, destination_dir=root_dir,
+                          base_name=base_name, max_dictionary_keys=max_dictionary_keys)
         verify_table_from_disk(read(root_dir))
         verify_file_names()
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
-                                          max_dictionary_keys=max_dictionary_keys, generate_metadata_files=True)
+        write_partitioned(source, col_definitions=definition, destination_dir=root_dir,
+                          max_dictionary_keys=max_dictionary_keys, generate_metadata_files=True)
         verify_table_from_disk(read(root_dir))
         self.verify_metadata_files(root_dir)
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir,
-                                          base_name=base_name)
+        write_partitioned(source, col_definitions=definition, destination_dir=root_dir,
+                          base_name=base_name)
         verify_table_from_disk(read(root_dir))
         verify_file_names()
 
         shutil.rmtree(root_dir)
-        write_key_value_partitioned_table(source, col_definitions=definition, destination_dir=root_dir)
+        write_partitioned(source, col_definitions=definition, destination_dir=root_dir)
         verify_table_from_disk(read(root_dir))
 
 

From 0d7c62e5a8dd7ecf52bda949f54489e7d38ded7f Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Fri, 22 Mar 2024 10:14:46 -0500
Subject: [PATCH 35/45] Review with Chip part 2

---
 .../parquet/table/ParquetTableWriter.java       |  6 +++---
 .../table/ParquetTableReadWriteTest.java        | 12 ++++++------
 py/server/deephaven/parquet.py                  | 17 +++++++++--------
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 0876118cdc5..0914c2f88bf 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -168,9 +168,9 @@ static void write(
                         }
                         // We don't accumulate metadata from grouping files into the main metadata file
                         write(indexTable, indexTable.getDefinition(), writeInstructionsToUse,
-                                info.destFile.getAbsolutePath(),
-                                info.destFileForMetadata.getAbsolutePath(), Collections.emptyMap(), TableInfo.builder(),
-                                NullParquetMetadataFileWriter.INSTANCE, computedCache);
+                                info.destFile.getAbsolutePath(), info.destFileForMetadata.getAbsolutePath(),
+                                Collections.emptyMap(), TableInfo.builder(), NullParquetMetadataFileWriter.INSTANCE,
+                                computedCache);
                     }
                 }
             }
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 73b604399de..46cb17ca007 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -703,12 +703,12 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
                 ColumnDefinition.ofInt("PC1").withPartitioning(),
                 ColumnDefinition.ofInt("PC2").withPartitioning(),
                 ColumnDefinition.ofLong("I"));
-        final Table inputData = ((QueryTable) TableTools.emptyTable(1_000_000)
+        final Table indexedtable = ((QueryTable) TableTools.emptyTable(1_000_000)
                 .updateView("PC1 = (int)(ii%3)",
                         "PC2 = (int)(ii%2)",
                         "I = ii"))
                 .withDefinitionUnsafe(definition);
-        DataIndexer.getOrCreateDataIndex(inputData, "I");
+        DataIndexer.getOrCreateDataIndex(indexedtable, "I");
         // TODO verify correctness of the indexing data
 
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
@@ -716,7 +716,7 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
                 .setGenerateMetadataFiles(true)
                 .setBaseNameForPartitionedParquetData("data")
                 .build();
-        writeKeyValuePartitionedTable(inputData, parentDir.getAbsolutePath(), writeInstructions);
+        writeKeyValuePartitionedTable(indexedtable, parentDir.getAbsolutePath(), writeInstructions);
 
         // Verify that metadata files are generated
         assertTrue(new File(parentDir, "_common_metadata").exists());
@@ -734,16 +734,16 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
 
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
         fromDisk.where("I == 3").select();
-        assertTableEquals(inputData.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
+        assertTableEquals(indexedtable.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
         final File commonMetadata = new File(parentDir, "_common_metadata");
         final Table fromDiskWithMetadata = readTable(commonMetadata);
         fromDiskWithMetadata.where("I == 3").select();
-        assertTableEquals(inputData.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
+        assertTableEquals(indexedtable.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2"));
 
         // Write the same table without generating metadata files
         final File parentDirWithoutMetadata = new File(rootFile, "writeKeyValuePartitionedDataWithoutMetadata");
-        writeKeyValuePartitionedTable(inputData, parentDirWithoutMetadata.getAbsolutePath(), EMPTY);
+        writeKeyValuePartitionedTable(indexedtable, parentDirWithoutMetadata.getAbsolutePath(), EMPTY);
 
         // Verify that no metadata files are generated
         assertFalse(new File(parentDirWithoutMetadata, "_common_metadata").exists());
diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index f7799a46721..8131a8e790a 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -6,7 +6,7 @@
 Parquet files. """
 from dataclasses import dataclass
 from enum import Enum
-from typing import List, Optional, Union, Dict
+from typing import List, Optional, Union, Dict, Literal
 
 import jpy
 
@@ -22,6 +22,7 @@
 _JParquetInstructions = jpy.get_type("io.deephaven.parquet.table.ParquetInstructions")
 _JTableDefinition = jpy.get_type("io.deephaven.engine.table.TableDefinition")
 
+_COMPRESSION_CODEC_NAMES = Literal["UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD"]
 
 @dataclass
 class ColumnInstruction:
@@ -35,7 +36,7 @@ class ColumnInstruction:
 
 def _build_parquet_instructions(
     col_instructions: Optional[List[ColumnInstruction]] = None,
-    compression_codec_name: Optional[str] = None,
+    compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     is_legacy_parquet: bool = False,
@@ -240,7 +241,7 @@ def write(
     path: str,
     col_definitions: Optional[List[Column]] = None,
     col_instructions: Optional[List[ColumnInstruction]] = None,
-    compression_codec_name: Optional[str] = None,
+    compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
@@ -258,7 +259,7 @@ def write(
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing particular
             columns, default is None, which means no specialization for any column
         compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY.
-            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "LZ4_RAW", "ZSTD".
+            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD".
         max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
             before switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
@@ -300,7 +301,7 @@ def write_partitioned(
         destination_dir: str,
         col_definitions: Optional[List[Column]] = None,
         col_instructions: Optional[List[ColumnInstruction]] = None,
-        compression_codec_name: Optional[str] = None,
+        compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
         max_dictionary_keys: Optional[int] = None,
         max_dictionary_size: Optional[int] = None,
         target_page_size: Optional[int] = None,
@@ -322,7 +323,7 @@ def write_partitioned(
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing particular
             columns, default is None, which means no specialization for any column
         compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY.
-            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "LZ4_RAW", "ZSTD".
+            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD".
         max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
             before switching to non-dictionary encoding; never evaluated for non-String columns , defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
@@ -378,7 +379,7 @@ def batch_write(
     paths: List[str],
     col_definitions: List[Column],
     col_instructions: Optional[List[ColumnInstruction]] = None,
-    compression_codec_name: Optional[str] = None,
+    compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
@@ -401,7 +402,7 @@ def batch_write(
             implied by the table. Default is None, which means use the column definitions implied by the table
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing
         compression_codec_name (Optional[str]): the compression codec to use, if not specified, defaults to SNAPPY.
-            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "LZ4_RAW", "ZSTD".
+            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD".
         max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
             before switching to non-dictionary encoding; never evaluated for non-String columns , defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before

From 5a3de8e4565d9dbe10fdcb5687dce9a064b2b2d3 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Fri, 22 Mar 2024 10:43:56 -0500
Subject: [PATCH 36/45] Review with Chip and Jianfeng Part 3

---
 py/server/deephaven/parquet.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index 8131a8e790a..a4b23807881 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -22,7 +22,6 @@
 _JParquetInstructions = jpy.get_type("io.deephaven.parquet.table.ParquetInstructions")
 _JTableDefinition = jpy.get_type("io.deephaven.engine.table.TableDefinition")
 
-_COMPRESSION_CODEC_NAMES = Literal["UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD"]
 
 @dataclass
 class ColumnInstruction:
@@ -36,7 +35,7 @@ class ColumnInstruction:
 
 def _build_parquet_instructions(
     col_instructions: Optional[List[ColumnInstruction]] = None,
-    compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
+    compression_codec_name: Optional[str] = None,
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     is_legacy_parquet: bool = False,
@@ -241,7 +240,7 @@ def write(
     path: str,
     col_definitions: Optional[List[Column]] = None,
     col_instructions: Optional[List[ColumnInstruction]] = None,
-    compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
+    compression_codec_name: Optional[str] = None,
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
@@ -301,7 +300,7 @@ def write_partitioned(
         destination_dir: str,
         col_definitions: Optional[List[Column]] = None,
         col_instructions: Optional[List[ColumnInstruction]] = None,
-        compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
+        compression_codec_name: Optional[str] = None,
         max_dictionary_keys: Optional[int] = None,
         max_dictionary_size: Optional[int] = None,
         target_page_size: Optional[int] = None,
@@ -379,7 +378,7 @@ def batch_write(
     paths: List[str],
     col_definitions: List[Column],
     col_instructions: Optional[List[ColumnInstruction]] = None,
-    compression_codec_name: Optional[_COMPRESSION_CODEC_NAMES] = None,
+    compression_codec_name: Optional[str] = None,
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,

From f68551f684c12ceef8d09dd3d5c8dc015df51ecd Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Fri, 22 Mar 2024 12:38:25 -0500
Subject: [PATCH 37/45] Review with Chip and Jianfeng continued

---
 .../table/ParquetTableReadWriteTest.java      | 78 ++++++-------------
 py/server/deephaven/parquet.py                | 14 ++--
 2 files changed, 29 insertions(+), 63 deletions(-)

diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 46cb17ca007..072e901548f 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -22,10 +22,8 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.*;
 import io.deephaven.engine.table.impl.QueryTable;
-import io.deephaven.engine.table.impl.SourceTable;
 import io.deephaven.engine.table.impl.dataindex.DataIndexUtils;
 import io.deephaven.engine.table.impl.indexer.DataIndexer;
-import io.deephaven.engine.table.impl.locations.TableDataException;
 import io.deephaven.engine.table.impl.select.FormulaEvaluationException;
 import io.deephaven.engine.table.impl.select.FunctionalColumn;
 import io.deephaven.engine.table.impl.select.SelectColumn;
@@ -345,23 +343,9 @@ public void indexByLongKey() {
         final Table fromDisk = checkSingleTable(testTable, dest);
 
         // Validate the indexes and lookup functions.
-        ColumnSource<?>[] columns = Arrays.stream(new String[] {"someLong"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        DataIndex fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
-
-        columns = Arrays.stream(new String[] {"someInt", "someLong"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
-
-        columns = Arrays.stream(new String[] {"someLong", "someInt"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
+        verifyIndexingInfoExists(fromDisk, "someLong");
+        verifyIndexingInfoExists(fromDisk, "someInt", "someLong");
+        verifyIndexingInfoExists(fromDisk, "someLong", "someInt");
     }
 
     @Test
@@ -382,23 +366,9 @@ public void indexByStringKey() {
         final Table fromDisk = checkSingleTable(testTable, dest);
 
         // Validate the indexes and lookup functions.
-        ColumnSource<?>[] columns = Arrays.stream(new String[] {"someString"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        DataIndex fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
-
-        columns = Arrays.stream(new String[] {"someInt", "someString"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
-
-        columns = Arrays.stream(new String[] {"someString", "someInt"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
+        verifyIndexingInfoExists(fromDisk, "someString");
+        verifyIndexingInfoExists(fromDisk, "someInt", "someString");
+        verifyIndexingInfoExists(fromDisk, "someString", "someInt");
     }
 
     @Test
@@ -419,33 +389,28 @@ public void indexByBigInt() {
         final Table fromDisk = checkSingleTable(testTable, dest);
 
         // Validate the indexes and lookup functions.
-        ColumnSource<?>[] columns = Arrays.stream(new String[] {"someBigInt"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        DataIndex fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
-
-        columns = Arrays.stream(new String[] {"someInt", "someBigInt"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
-        Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
+        verifyIndexingInfoExists(fromDisk, "someBigInt");
+        verifyIndexingInfoExists(fromDisk, "someInt", "someBigInt");
+        verifyIndexingInfoExists(fromDisk, "someBigInt", "someInt");
+    }
 
-        columns = Arrays.stream(new String[] {"someBigInt", "someInt"}).map(fromDisk::getColumnSource)
-                .toArray(ColumnSource[]::new);
-        fullIndex = DataIndexer.of(fromDisk.getRowSet()).getDataIndex(columns);
+    private static void verifyIndexingInfoExists(final Table table, final String... columnNames) {
+        assertTrue(DataIndexer.hasDataIndex(table, columnNames));
+        final DataIndex fullIndex = DataIndexer.getDataIndex(table, columnNames);
         Assert.neqNull(fullIndex, "fullIndex");
-        assertLookupFromTable(fromDisk, fullIndex, columns);
+        assertLookupFromTable(table, fullIndex, columnNames);
     }
 
-    private void assertLookupFromTable(
+    private static void assertLookupFromTable(
             final Table sourceTable,
             final DataIndex fullIndex,
-            final ColumnSource<?>[] columns) {
+            final String... columnNames) {
+        final ColumnSource<?>[] columns = Arrays.stream(columnNames).map(sourceTable::getColumnSource)
+                .toArray(ColumnSource[]::new);
         final DataIndex.RowKeyLookup fullIndexRowKeyLookup = fullIndex.rowKeyLookup(columns);
         final ColumnSource<RowSet> fullIndexRowSetColumn = fullIndex.rowSetColumn();
 
-        ChunkSource.WithPrev<?> tableKeys = DataIndexUtils.makeBoxedKeySource(columns);
+        final ChunkSource.WithPrev<?> tableKeys = DataIndexUtils.makeBoxedKeySource(columns);
 
         // Iterate through the entire source table and verify the lookup row set is valid and contains this row.
         try (final RowSet.Iterator rsIt = sourceTable.getRowSet().iterator();
@@ -594,7 +559,6 @@ public void basicParquetWithMetadataTest() {
 
     @Test
     public void parquetWithIndexingDataAndMetadataTest() {
-        // TODO verify correctness of the indexing data
         final File parentDir = new File(rootFile, "tempDir");
         final int[] data = new int[500 * 4];
         for (int i = 0; i < data.length; i++) {
@@ -611,10 +575,12 @@ public void parquetWithIndexingDataAndMetadataTest() {
 
         final Table fromDisk = readTable(destFile);
         assertTableEquals(indexedTable, fromDisk);
+        verifyIndexingInfoExists(fromDisk, "vvv");
 
         final File metadataFile = new File(parentDir, "_metadata");
         final Table fromDiskWithMetadata = readTable(metadataFile);
         assertTableEquals(indexedTable, fromDiskWithMetadata);
+        verifyIndexingInfoExists(fromDiskWithMetadata, "vvv");
     }
 
     @Test
@@ -1079,7 +1045,7 @@ public void testAllNonPartitioningColumnTypes() {
         Table inputData = ((QueryTable) TableTools.emptyTable(10).updateView(
                 "PC1 =  (ii%2 == 0) ? `AA` : `BB`",
                 "PC2 = (int)(ii%3)",
-                "NPC1 =  (ii%2 == 0) ? `AA` : `BB`",
+                "NPC1 = (ii%2 == 0) ? `AA` : `BB`",
                 "NPC2 = (ii % 2 == 0)",
                 "NPC3 = (char)(65 + (ii % 2))",
                 "NPC4 = (byte)(ii % 2)",
diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index a4b23807881..c743b1abffd 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -6,7 +6,7 @@
 Parquet files. """
 from dataclasses import dataclass
 from enum import Enum
-from typing import List, Optional, Union, Dict, Literal
+from typing import List, Optional, Union, Dict
 
 import jpy
 
@@ -257,8 +257,8 @@ def write(
             implied by the table. Default is None, which means use the column definitions implied by the table
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing particular
             columns, default is None, which means no specialization for any column
-        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY.
-            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD".
+        compression_codec_name (Optional[str]): the compression codec to use. Allowed values include "UNCOMPRESSED",
+            "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD", etc. If not specified, defaults to "SNAPPY".
         max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
             before switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
@@ -321,8 +321,8 @@ def write_partitioned(
             implied by the table. Default is None, which means use the column definitions implied by the table
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing particular
             columns, default is None, which means no specialization for any column
-        compression_codec_name (Optional[str]): the default compression codec to use, if not specified, defaults to SNAPPY.
-            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD".
+        compression_codec_name (Optional[str]): the compression codec to use. Allowed values include "UNCOMPRESSED",
+            "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD", etc. If not specified, defaults to "SNAPPY".
         max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
             before switching to non-dictionary encoding; never evaluated for non-String columns , defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
@@ -400,8 +400,8 @@ def batch_write(
         col_definitions (Optional[List[Column]]): the column definitions to use for writing, instead of the definitions
             implied by the table. Default is None, which means use the column definitions implied by the table
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing
-        compression_codec_name (Optional[str]): the compression codec to use, if not specified, defaults to SNAPPY.
-            Allowed values are "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD".
+        compression_codec_name (Optional[str]): the compression codec to use. Allowed values include "UNCOMPRESSED",
+            "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD", etc. If not specified, defaults to "SNAPPY".
         max_dictionary_keys (Optional[int]): the maximum number of unique keys the writer should add to a dictionary page
             before switching to non-dictionary encoding; never evaluated for non-String columns , defaults to 2^20 (1,048,576)
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before

From b11ebd101e71c477164c40cdd1b722f92b395319 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 25 Mar 2024 16:59:00 -0500
Subject: [PATCH 38/45] Added new APIs for managing indexes

---
 .../deephaven/parquet/table/ParquetTools.java | 169 ++++++++++++++++--
 .../table/ParquetTableReadWriteTest.java      | 120 ++++++++++---
 2 files changed, 243 insertions(+), 46 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index 774d816d897..ba3b03fc04a 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -10,6 +10,7 @@
 import io.deephaven.base.Pair;
 import io.deephaven.base.verify.Require;
 import io.deephaven.engine.context.ExecutionContext;
+import io.deephaven.engine.liveness.LivenessScopeStack;
 import io.deephaven.engine.primitive.iterator.CloseableIterator;
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.ColumnSource;
@@ -22,6 +23,7 @@
 import io.deephaven.engine.updategraph.UpdateSourceRegistrar;
 import io.deephaven.parquet.base.ParquetMetadataFileWriter;
 import io.deephaven.parquet.base.NullParquetMetadataFileWriter;
+import io.deephaven.util.SafeCloseable;
 import io.deephaven.util.channel.SeekableChannelsProvider;
 import io.deephaven.util.channel.SeekableChannelsProviderLoader;
 import io.deephaven.util.channel.SeekableChannelsProviderPlugin;
@@ -88,6 +90,7 @@
 public class ParquetTools {
 
     private static final int MAX_PARTITIONING_LEVELS_INFERENCE = 32;
+    private static final String[][] EMPTY_INDEXES = new String[0][];
 
     private ParquetTools() {}
 
@@ -506,7 +509,9 @@ private static List<ParquetTableWriter.IndexWritingInfo> indexInfoBuilderHelper(
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
      * table. The generated parquet files will have names of the format provided by
-     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}. Any indexing columns present on the source table
+     * will be written as sidecar tables. To write only a subset of the indexes or add additional indexes while writing,
+     * use {@link #writeKeyValuePartitionedTable(Table, String, ParquetInstructions, String[][])}.
      *
      * @param sourceTable The table to partition and write
      * @param destinationDir The path to destination root directory to store partitioned data in nested format.
@@ -517,7 +522,31 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
             @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
         writeKeyValuePartitionedTable(sourceTable, sourceTable.getDefinition(), destinationDir,
-                writeInstructions);
+                writeInstructions, indexedColumnNames(sourceTable));
+    }
+
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns of provided
+     * table. The generated parquet files will have names of the format provided by
+     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     *
+     * @param sourceTable The table to partition and write
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     * @param indexColumnArr Arrays containing the column names for indexes to persist. The write operation will store
+     *        the index info as sidecar tables. This argument is used to narrow the set of indexes to write, or to be
+     *        explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+     *        will be computed on demand.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final String destinationDir,
+            @NotNull final ParquetInstructions writeInstructions,
+            @Nullable final String[][] indexColumnArr) {
+        writeKeyValuePartitionedTable(sourceTable, sourceTable.getDefinition(), destinationDir,
+                writeInstructions, indexColumnArr);
     }
 
     /**
@@ -525,7 +554,9 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
      * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
      * table definition. The generated parquet files will have names of the format provided by
-     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}. Any indexing columns present on the source table
+     * will be written as sidecar tables. To write only a subset of the indexes or add additional indexes while writing,
+     * use {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, ParquetInstructions, String[][])}.
      *
      * @param sourceTable The table to partition and write
      * @param definition table definition to use (instead of the one implied by the table itself)
@@ -537,6 +568,33 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
             @NotNull final TableDefinition definition,
             @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(sourceTable, definition, destinationDir, writeInstructions,
+                indexedColumnNames(sourceTable));
+
+    }
+
+    /**
+     * Write table to disk in parquet format with {@link TableDefinition#getPartitioningColumns() partitioning columns}
+     * written as "key=value" format in a nested directory structure. To generate these individual partitions, this
+     * method will call {@link Table#partitionBy(String...) partitionBy} on all the partitioning columns in the provided
+     * table definition. The generated parquet files will have names of the format provided by
+     * {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     *
+     * @param sourceTable The table to partition and write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     * @param indexColumnArr Arrays containing the column names for indexes to persist. The write operation will store
+     *        the index info as sidecar tables. This argument is used to narrow the set of indexes to write, or to be
+     *        explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+     *        will be computed on demand.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final String destinationDir,
+            @NotNull final ParquetInstructions writeInstructions,
+            @Nullable final String[][] indexColumnArr) {
         final List<ColumnDefinition<?>> partitioningColumns = definition.getPartitioningColumns();
         if (partitioningColumns.isEmpty()) {
             throw new IllegalArgumentException("Table must have partitioning columns to write partitioned data");
@@ -549,7 +607,7 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
         final TableDefinition leafDefinition =
                 getNonKeyTableDefinition(new HashSet<>(Arrays.asList(partitioningColNames)), definition);
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
-                writeInstructions, Optional.of(sourceTable));
+                writeInstructions, indexColumnArr, Optional.of(sourceTable));
     }
 
     /**
@@ -566,12 +624,34 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
     public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
             @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(partitionedTable, destinationDir, writeInstructions, EMPTY_INDEXES);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The generated parquet files will have
+     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     * @param indexColumnArr Arrays containing the column names for indexes to persist. The write operation will store
+     *        the index info as sidecar tables. This argument is used to narrow the set of indexes to write, or to be
+     *        explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+     *        will be computed on demand.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final String destinationDir,
+            @NotNull final ParquetInstructions writeInstructions,
+            @Nullable final String[][] indexColumnArr) {
         final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(),
                 partitionedTable.table().getDefinition());
         final TableDefinition leafDefinition = getNonKeyTableDefinition(partitionedTable.keyColumnNames(),
                 partitionedTable.constituentDefinition());
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
-                writeInstructions, Optional.empty());
+                writeInstructions, indexColumnArr, Optional.empty());
     }
 
     /**
@@ -590,10 +670,34 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
             @NotNull final TableDefinition definition,
             @NotNull final String destinationDir,
             @NotNull final ParquetInstructions writeInstructions) {
+        writeKeyValuePartitionedTable(partitionedTable, definition, destinationDir, writeInstructions, EMPTY_INDEXES);
+    }
+
+    /**
+     * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
+     * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
+     * {@link Table#partitionBy(String...) partitionBy} on the required columns. The generated parquet files will have
+     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     *
+     * @param partitionedTable The partitioned table to write
+     * @param definition table definition to use (instead of the one implied by the table itself)
+     * @param destinationDir The path to destination root directory to store partitioned data in nested format.
+     *        Non-existing directories are created.
+     * @param writeInstructions Write instructions for customizations while writing
+     * @param indexColumnArr Arrays containing the column names for indexes to persist. The write operation will store
+     *        the index info as sidecar tables. This argument is used to narrow the set of indexes to write, or to be
+     *        explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+     *        will be computed on demand.
+     */
+    public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable partitionedTable,
+            @NotNull final TableDefinition definition,
+            @NotNull final String destinationDir,
+            @NotNull final ParquetInstructions writeInstructions,
+            @NotNull final String[][] indexColumnArr) {
         final TableDefinition keyTableDefinition = getKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
         final TableDefinition leafDefinition = getNonKeyTableDefinition(partitionedTable.keyColumnNames(), definition);
         writeKeyValuePartitionedTableImpl(partitionedTable, keyTableDefinition, leafDefinition, destinationDir,
-                writeInstructions, Optional.empty());
+                writeInstructions, indexColumnArr, Optional.empty());
     }
 
     /**
@@ -605,6 +709,10 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      * @param leafDefinition The definition for leaf parquet files to be written
      * @param destinationRoot The path to destination root directory to store partitioned data in nested format
      * @param writeInstructions Write instructions for customizations while writing
+     * @param indexColumnArr Arrays containing the column names for indexes to persist. The write operation will store
+     *        the index info as sidecar tables. This argument is used to narrow the set of indexes to write, or to be
+     *        explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+     *        will be computed on demand.
      * @param sourceTable The optional source table, provided when user provides a merged source table to write, like in
      *        {@link #writeKeyValuePartitionedTable(Table, String, ParquetInstructions)} and
      *        {@link #writeKeyValuePartitionedTable(Table, TableDefinition, String, ParquetInstructions)}
@@ -614,6 +722,7 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
             @NotNull final TableDefinition leafDefinition,
             @NotNull final String destinationRoot,
             @NotNull final ParquetInstructions writeInstructions,
+            @Nullable final String[][] indexColumnArr,
             @NotNull final Optional<Table> sourceTable) {
         if (leafDefinition.numColumns() == 0) {
             throw new IllegalArgumentException("Cannot write a partitioned parquet table without any non-partitioning "
@@ -692,10 +801,24 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
         final Map<String, Map<ParquetCacheTags, Object>> computedCache =
                 buildComputedCache(() -> sourceTable.orElseGet(partitionedTable::merge), leafDefinition);
         final Table[] partitionedDataArray = partitionedData.toArray(Table[]::new);
-        // TODO Verify correctness with Ryan/Larry
-        writeParquetTablesImpl(partitionedDataArray, leafDefinition, writeInstructions,
-                destinations.toArray(File[]::new), indexedColumnNames(partitionedDataArray), partitioningColumnsSchema,
-                new File(destinationRoot), computedCache);
+        try (final SafeCloseable ignored = LivenessScopeStack.open()) {
+            // TODO(deephaven-core#5292): Optimize creating index on constituent tables
+            addIndexesToTables(partitionedDataArray, indexColumnArr);
+            writeParquetTablesImpl(partitionedDataArray, leafDefinition, writeInstructions,
+                    destinations.toArray(File[]::new), indexColumnArr, partitioningColumnsSchema,
+                    new File(destinationRoot), computedCache);
+        }
+    }
+
+    private static void addIndexesToTables(@NotNull final Table[] tables,
+            @Nullable final String[][] indexColumnArr) {
+        if (indexColumnArr != null && indexColumnArr.length != 0) {
+            for (final Table table : tables) {
+                for (final String[] indexCols : indexColumnArr) {
+                    DataIndexer.getOrCreateDataIndex(table, indexCols);
+                }
+            }
+        }
     }
 
     /**
@@ -939,22 +1062,32 @@ private static void writeParquetTablesImpl(@NotNull final Table[] sources,
      * @implNote This only examines the first source table. The writing code will compute missing indexes for the other
      *           source tables.
      */
-    private static String[][] indexedColumnNames(@NotNull Table @NotNull [] sources) {
+    @NotNull
+    private static String[][] indexedColumnNames(@NotNull final Table @NotNull [] sources) {
         if (sources.length == 0) {
-            return new String[0][];
+            return EMPTY_INDEXES;
         }
-
         // Use the first table as the source of indexed columns
-        final Table firstTable = sources[0];
-        final DataIndexer dataIndexer = DataIndexer.existingOf(firstTable.getRowSet());
+        return indexedColumnNames(sources[0]);
+    }
+
+    /**
+     * Examine the source table to retrieve the list of indexes as String[] arrays.
+     *
+     * @param source The table from which to retrieve the indexes
+     * @return An array containing the indexes as String[] arrays.
+     */
+    @NotNull
+    private static String[][] indexedColumnNames(@NotNull final Table source) {
+        final DataIndexer dataIndexer = DataIndexer.existingOf(source.getRowSet());
         if (dataIndexer == null) {
-            return new String[0][];
+            return EMPTY_INDEXES;
         }
         final List<DataIndex> dataIndexes = dataIndexer.dataIndexes(true);
         if (dataIndexes.isEmpty()) {
-            return new String[0][];
+            return EMPTY_INDEXES;
         }
-        final Map<String, ? extends ColumnSource<?>> nameToColumn = firstTable.getColumnSourceMap();
+        final Map<String, ? extends ColumnSource<?>> nameToColumn = source.getColumnSourceMap();
         // We disregard collisions, here; any mapped name is an adequate choice.
         final Map<ColumnSource<?>, String> columnToName = nameToColumn.entrySet().stream()
                 .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 072e901548f..5ad5348025d 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -675,7 +675,6 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
                         "I = ii"))
                 .withDefinitionUnsafe(definition);
         DataIndexer.getOrCreateDataIndex(indexedtable, "I");
-        // TODO verify correctness of the indexing data
 
         final File parentDir = new File(rootFile, "writeKeyValuePartitionedDataTest");
         final ParquetInstructions writeInstructions = ParquetInstructions.builder()
@@ -688,17 +687,20 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         assertTrue(new File(parentDir, "_common_metadata").exists());
         assertTrue(new File(parentDir, "_metadata").exists());
 
-        // Verify that the partitioned data exists
+        // Verify that the partitioning and indexing data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
             for (int PC2 = 0; PC2 <= 1; PC2++) {
                 final File dir = new File(parentDir, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
                 assertTrue(dir.exists() && dir.isDirectory());
                 final File dataFile = new File(dir, "data.parquet");
                 assertTrue(dataFile.exists() && dataFile.isFile());
+                final File indexFile = new File(dir, ".dh_metadata/indexes/I/index_I_data.parquet");
+                assertTrue(indexFile.exists() && indexFile.isFile());
             }
         }
 
         final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+        verifyIndexingInfoExists(fromDisk, "I");
         fromDisk.where("I == 3").select();
         assertTableEquals(indexedtable.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2"));
 
@@ -715,7 +717,7 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
         assertFalse(new File(parentDirWithoutMetadata, "_common_metadata").exists());
         assertFalse(new File(parentDirWithoutMetadata, "_metadata").exists());
 
-        // Verify that the partitioned data exists
+        // Verify that the partitioning and indexing data exists
         for (int PC1 = 0; PC1 <= 2; PC1++) {
             for (int PC2 = 0; PC2 <= 1; PC2++) {
                 final File dir = new File(parentDirWithoutMetadata, "PC1=" + PC1 + File.separator + "PC2=" + PC2);
@@ -723,12 +725,14 @@ public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() {
                 final File[] fileList = dir.listFiles();
                 for (final File dataFile : fileList) {
                     // hidden indexing data
-                    assertTrue(dataFile.isHidden() || dataFile.getName().endsWith(".parquet"));
+                    assertTrue(dataFile.getName().equals(".dh_metadata")
+                            || dataFile.getName().endsWith(".parquet"));
                 }
             }
         }
         final Table fromDiskWithoutMetadata = readKeyValuePartitionedTable(parentDirWithoutMetadata, EMPTY);
         assertTableEquals(fromDisk, fromDiskWithoutMetadata);
+        verifyIndexingInfoExists(fromDiskWithoutMetadata, "I");
     }
 
     @Test
@@ -1070,26 +1074,57 @@ public void testAllNonPartitioningColumnTypes() {
         writeKeyValuePartitionedTable(inputData, parentDir.getPath(), writeInstructions);
 
         // Store the big decimal with the precision and scale consistent with what we write to parquet
-        inputData = maybeFixBigDecimal(inputData);
+        final Table bigDecimalFixedInputData = maybeFixBigDecimal(inputData);
 
         final String[] partitioningColumns = definition.getPartitioningColumns().stream()
                 .map(ColumnDefinition::getName).toArray(String[]::new);
-        Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
-        assertTableEquals(inputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
-        Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
-        assertTableEquals(inputData.sort(partitioningColumns), fromDiskWithMetadata.sort(partitioningColumns));
+        {
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns),
+                    fromDiskWithMetadata.sort(partitioningColumns));
+            FileUtils.deleteRecursively(parentDir);
+        }
 
-        FileUtils.deleteRecursively(parentDir);
+        // Next API we test is to pass additional indexing columns
+        final String indexColumn = "NPC5";
+        final String[][] indexColumns = new String[][] {{indexColumn}};
+        {
+            writeKeyValuePartitionedTable(inputData, parentDir.getPath(), writeInstructions, indexColumns);
+            assertFalse(DataIndexer.hasDataIndex(inputData, indexColumn));
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+            verifyIndexingInfoExists(fromDisk, indexColumn);
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns),
+                    fromDiskWithMetadata.sort(partitioningColumns));
+            FileUtils.deleteRecursively(parentDir);
+        }
 
         // Next API we test is passing the partitioned table without any table definition
         final PartitionedTable partitionedTable = inputData.partitionBy("PC1");
-        writeKeyValuePartitionedTable(partitionedTable, parentDir.getPath(), writeInstructions);
-        fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
-        assertTableEquals(inputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
-        fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
-        assertTableEquals(inputData.sort(partitioningColumns), fromDiskWithMetadata.sort(partitioningColumns));
+        {
+            writeKeyValuePartitionedTable(partitionedTable, parentDir.getPath(), writeInstructions);
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns),
+                    fromDiskWithMetadata.sort(partitioningColumns));
+            FileUtils.deleteRecursively(parentDir);
+        }
 
-        FileUtils.deleteRecursively(parentDir);
+        // Next API we test is to pass additional indexing columns with partitioned table and no definition
+        {
+            writeKeyValuePartitionedTable(partitionedTable, parentDir.getPath(), writeInstructions, indexColumns);
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+            verifyIndexingInfoExists(fromDisk, "NPC5");
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns), fromDisk.sort(partitioningColumns));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(bigDecimalFixedInputData.sort(partitioningColumns),
+                    fromDiskWithMetadata.sort(partitioningColumns));
+            FileUtils.deleteRecursively(parentDir);
+        }
 
         // Next API we test is passing the regular table with an updated table definition where we drop
         // some partitioning columns and non-partitioning columns and add some new non-partitioning columns
@@ -1099,21 +1134,50 @@ public void testAllNonPartitioningColumnTypes() {
                 .collect(Collectors.toList());
         newColumns.add(ColumnDefinition.ofInt("NPC15"));
         final TableDefinition newDefinition = TableDefinition.of(newColumns);
-        writeKeyValuePartitionedTable(inputData, newDefinition, parentDir.getAbsolutePath(), writeInstructions);
-        final Table expected = inputData.dropColumns("PC2", "NPC6").updateView("NPC15 = (int)null");
-        fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
-        assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
-        fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
-        assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+        final Table expected = bigDecimalFixedInputData.dropColumns("PC2", "NPC6").updateView("NPC15 = (int)null");
+        {
+            writeKeyValuePartitionedTable(inputData, newDefinition, parentDir.getAbsolutePath(), writeInstructions);
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+            assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+            FileUtils.deleteRecursively(parentDir);
+        }
 
-        FileUtils.deleteRecursively(parentDir);
+        // Next API to test takes table with updated definition and additional indexing columns
+        {
+            writeKeyValuePartitionedTable(inputData, newDefinition, parentDir.getAbsolutePath(), writeInstructions,
+                    indexColumns);
+            assertFalse(DataIndexer.hasDataIndex(inputData, indexColumn));
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+            verifyIndexingInfoExists(fromDisk, indexColumn);
+            assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+            FileUtils.deleteRecursively(parentDir);
+        }
 
         // Next API we test is passing the partitioned table with an updated table definition
-        writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir.getPath(), writeInstructions);
-        fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
-        assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
-        fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
-        assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+        {
+            writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir.getPath(), writeInstructions);
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY).select();
+            assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+            FileUtils.deleteRecursively(parentDir);
+        }
+
+        // Next API we test is passing the indexing columns with partitioned table and an updated table definition
+        {
+            writeKeyValuePartitionedTable(partitionedTable, newDefinition, parentDir.getPath(), writeInstructions,
+                    indexColumns);
+            final Table fromDisk = readKeyValuePartitionedTable(parentDir, EMPTY);
+            verifyIndexingInfoExists(fromDisk, "NPC5");
+            assertTableEquals(expected.sort("PC1"), fromDisk.sort("PC1"));
+            final Table fromDiskWithMetadata = readTable(new File(parentDir, "_common_metadata"));
+            assertTableEquals(expected.sort("PC1"), fromDiskWithMetadata.sort("PC1"));
+            FileUtils.deleteRecursively(parentDir);
+        }
     }
 
 

From 3bdc92b3804d4ae85ebf60ecab325671657d38eb Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 25 Mar 2024 18:22:23 -0500
Subject: [PATCH 39/45] Trigger CI jobs


From 3dd60974327f0f823c94155f24ce1c53e8ad694b Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 26 Mar 2024 15:56:55 -0500
Subject: [PATCH 40/45] Review with Ryan

---
 .../parquet/base/RowGroupReaderImpl.java      |  3 --
 .../parquet/table/ParquetInstructions.java    |  3 +-
 .../table/ParquetMetadataFileWriterImpl.java  | 10 +----
 .../parquet/table/ParquetTableWriter.java     |  1 -
 .../deephaven/parquet/table/ParquetTools.java | 37 +++++++++++++------
 5 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
index 7015a9004ff..e50f17f2dae 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java
@@ -12,14 +12,11 @@
 import org.jetbrains.annotations.Nullable;
 
 import java.net.URI;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static io.deephaven.parquet.base.ParquetFileReader.FILE_URI_SCHEME;
-
 final class RowGroupReaderImpl implements RowGroupReader {
     private final RowGroup rowGroup;
     private final SeekableChannelsProvider channelsProvider;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
index 587b50c9b4f..9e0cc929229 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
@@ -724,8 +724,7 @@ public Builder setGenerateMetadataFiles(final boolean generateMetadataFiles) {
          * "PC=partition1/table-0.parquet", "PC=partition1/table-1.parquet", etc., where PC is a partitioning
          * column.</li>
          * <li>The token {@value #UUID_TOKEN} will be replaced with a random UUID. For example, a base name of
-         * "table-{uuid}" will result in files named like
-         * "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet.parquet".</li>
+         * "table-{uuid}" will result in files named like "table-8e8ab6b2-62f2-40d1-8191-1c5b70c5f330.parquet".</li>
          * <li>The token {@value #PARTITIONS_TOKEN} will be replaced with an underscore-delimited, concatenated string
          * of partition values. For example, a base name of "{partitions}-table" will result in files like
          * "PC1=partition1/PC2=partitionA/PC1=partition1_PC2=partitionA-table.parquet", where "PC1" and "PC2" are
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
index 5357f8f93fc..90da731eb0c 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetMetadataFileWriterImpl.java
@@ -104,7 +104,7 @@ private static class ParquetFileMetadata {
     }
 
     /**
-     * Add parquet metadata for the provided parquet file the combined metadata file.
+     * Add parquet metadata for the provided parquet file to the combined metadata file.
      *
      * @param parquetFilePath The parquet file destination path
      * @param metadata The parquet metadata
@@ -255,13 +255,7 @@ private static void mergeBlocksInto(final ParquetFileMetadata parquetFileMetadat
 
     private static String getRelativePath(final String parquetFilePath, final Path metadataRootDirAbsPath) {
         final Path parquetFileAbsPath = new File(parquetFilePath).getAbsoluteFile().toPath();
-        final String relativePath = metadataRootDirAbsPath.relativize(parquetFileAbsPath).toString();
-        // Remove leading slashes from the relative path
-        int pos = 0;
-        while (pos < relativePath.length() && relativePath.charAt(pos) == '/') {
-            pos++;
-        }
-        return relativePath.substring(pos);
+        return metadataRootDirAbsPath.relativize(parquetFileAbsPath).toString();
     }
 
     private void writeMetadataFile(final ParquetMetadata metadataFooter, final String outputPath) throws IOException {
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 0914c2f88bf..9afb4c358e5 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -166,7 +166,6 @@ static void write(
                                     .addColumnNameMapping(INDEX_ROW_SET_COLUMN_NAME, dataIndex.rowSetColumnName())
                                     .build();
                         }
-                        // We don't accumulate metadata from grouping files into the main metadata file
                         write(indexTable, indexTable.getDefinition(), writeInstructionsToUse,
                                 info.destFile.getAbsolutePath(), info.destFileForMetadata.getAbsolutePath(),
                                 Collections.emptyMap(), TableInfo.builder(), NullParquetMetadataFileWriter.INSTANCE,
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index ba3b03fc04a..ec91efcd1bf 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -614,7 +614,9 @@ public static void writeKeyValuePartitionedTable(@NotNull final Table sourceTabl
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
      * {@link Table#partitionBy(String...) partitionBy} on the required columns. The generated parquet files will have
-     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}. This method does
+     * not write any indexes as sidecar tables to disk. To write indexes, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, String, ParquetInstructions, String[][])}.
      *
      * @param partitionedTable The partitioned table to write
      * @param destinationDir The path to destination root directory to store partitioned data in nested format.
@@ -658,7 +660,9 @@ public static void writeKeyValuePartitionedTable(@NotNull final PartitionedTable
      * Write a partitioned table to disk in parquet format with all the {@link PartitionedTable#keyColumnNames() key
      * columns} as "key=value" format in a nested directory structure. To generate the partitioned table, users can call
      * {@link Table#partitionBy(String...) partitionBy} on the required columns. The generated parquet files will have
-     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}.
+     * names of the format provided by {@link ParquetInstructions#baseNameForPartitionedParquetData()}. This method does
+     * not write any indexes as sidecar tables to disk. To write indexes, use
+     * {@link #writeKeyValuePartitionedTable(PartitionedTable, TableDefinition, String, ParquetInstructions, String[][])}.
      *
      * @param partitionedTable The partitioned table to write
      * @param definition table definition to use (instead of the one implied by the table itself)
@@ -798,27 +802,38 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
         } else {
             partitioningColumnsSchema = null;
         }
-        final Map<String, Map<ParquetCacheTags, Object>> computedCache =
-                buildComputedCache(() -> sourceTable.orElseGet(partitionedTable::merge), leafDefinition);
         final Table[] partitionedDataArray = partitionedData.toArray(Table[]::new);
         try (final SafeCloseable ignored = LivenessScopeStack.open()) {
             // TODO(deephaven-core#5292): Optimize creating index on constituent tables
-            addIndexesToTables(partitionedDataArray, indexColumnArr);
+            final Map<String, Map<ParquetCacheTags, Object>> computedCache =
+                    buildComputedCache(() -> sourceTable.orElseGet(partitionedTable::merge), leafDefinition);
+            // Store hard reference to prevent indexes from being garbage collected
+            final List<DataIndex> dataIndexes = addIndexesToTables(partitionedDataArray, indexColumnArr);
             writeParquetTablesImpl(partitionedDataArray, leafDefinition, writeInstructions,
                     destinations.toArray(File[]::new), indexColumnArr, partitioningColumnsSchema,
                     new File(destinationRoot), computedCache);
+            if (dataIndexes != null) {
+                dataIndexes.clear();
+            }
         }
     }
 
-    private static void addIndexesToTables(@NotNull final Table[] tables,
+    /**
+     * Add data indexes to provided tables, if not present, and return a list of hard references to the indexes.
+     */
+    @Nullable
+    private static List<DataIndex> addIndexesToTables(@NotNull final Table[] tables,
             @Nullable final String[][] indexColumnArr) {
-        if (indexColumnArr != null && indexColumnArr.length != 0) {
-            for (final Table table : tables) {
-                for (final String[] indexCols : indexColumnArr) {
-                    DataIndexer.getOrCreateDataIndex(table, indexCols);
-                }
+        if (indexColumnArr == null || indexColumnArr.length == 0) {
+            return null;
+        }
+        final List<DataIndex> dataIndexes = new ArrayList<>(indexColumnArr.length * tables.length);
+        for (final Table table : tables) {
+            for (final String[] indexCols : indexColumnArr) {
+                dataIndexes.add(DataIndexer.getOrCreateDataIndex(table, indexCols));
             }
         }
+        return dataIndexes;
     }
 
     /**

From d8bc0a531d777cc5015b2a18a4f46ccc86d609b6 Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 26 Mar 2024 19:24:52 -0500
Subject: [PATCH 41/45] Added python support for writing indexes

---
 .../parquet/table/ParquetTableWriter.java     |  6 +-
 py/server/deephaven/parquet.py                | 67 +++++++++++++------
 py/server/tests/test_parquet.py               | 36 ++++++++++
 3 files changed, 87 insertions(+), 22 deletions(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
index 9afb4c358e5..eed80903ae0 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTableWriter.java
@@ -91,8 +91,8 @@ static class IndexWritingInfo {
                 final File destFile) {
             this.indexColumnNames = indexColumnNames;
             this.parquetColumnNames = parquetColumnNames;
-            this.destFileForMetadata = destFileForMetadata;
-            this.destFile = destFile;
+            this.destFileForMetadata = destFileForMetadata.getAbsoluteFile();
+            this.destFile = destFile.getAbsoluteFile();
         }
     }
 
@@ -143,7 +143,7 @@ static void write(
         try {
             if (indexInfoList != null) {
                 cleanupFiles = new ArrayList<>(indexInfoList.size());
-                final Path destDirPath = Paths.get(destFilePath).getParent();
+                final Path destDirPath = new File(destFilePath).getAbsoluteFile().getParentFile().toPath();
                 for (final ParquetTableWriter.IndexWritingInfo info : indexInfoList) {
                     try (final SafeCloseable ignored = t.isRefreshing() ? LivenessScopeStack.open() : null) {
                         // This will retrieve an existing index if one exists, or create a new one if not
diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index c743b1abffd..d26345d05b1 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -220,6 +220,10 @@ def _j_file_array(paths: List[str]):
     return jpy.array("java.io.File", [_JFile(el) for el in paths])
 
 
+def _j_array_of_array_of_string(data_indexes: List[List[str]]):
+    return jpy.array("[Ljava.lang.String;", [jpy.array("java.lang.String", index_cols) for index_cols in data_indexes])
+
+
 def delete(path: str) -> None:
     """ Deletes a Parquet table on disk.
 
@@ -245,6 +249,7 @@ def write(
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
     generate_metadata_files: Optional[bool] = None,
+    index_columns: Optional[List[List[str]]] = None
 ) -> None:
     """ Write a table to a Parquet file.
 
@@ -268,6 +273,11 @@ def write(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
+        index_columns (Optional[List[List[str]]]): Lists containing the column names for indexes to persist. The write
+            operation will store the index info as sidecar tables. By default, data indexes to write are determined by
+            those present on the source table. This argument is used to narrow the set of indexes to write, or to be
+            explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+            will be computed on demand.
 
     Raises:
         DHError
@@ -282,15 +292,18 @@ def write(
             for_read=False,
             generate_metadata_files=generate_metadata_files,
         )
-
-        table_definition = None
         if col_definitions is not None:
             table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
+        else:
+            table_definition = table._definition
 
-        if table_definition:
-            _JParquetTools.writeTable(table.j_table, path, table_definition, write_instructions)
+        if index_columns:
+            table_array = jpy.array("io.deephaven.engine.table.Table", [table.j_table])
+            index_columns_array = _j_array_of_array_of_string(index_columns)
+            _JParquetTools.writeParquetTables(table_array, table_definition, write_instructions,
+                                              _j_file_array([path]), index_columns_array)
         else:
-            _JParquetTools.writeTable(table.j_table, _JFile(path), write_instructions)
+            _JParquetTools.writeTable(table.j_table, path, table_definition, write_instructions)
     except Exception as e:
         raise DHError(e, "failed to write to parquet data.") from e
 
@@ -306,6 +319,7 @@ def write_partitioned(
         target_page_size: Optional[int] = None,
         base_name: Optional[str] = None,
         generate_metadata_files: Optional[bool] = None,
+        index_columns: Optional[List[List[str]]] = None
 ) -> None:
     """ Write table to disk in parquet format with the partitioning columns written as "key=value" format in a nested
     directory structure. For example, for a partitioned column "date", we will have a directory structure like
@@ -344,6 +358,11 @@ def write_partitioned(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
+        index_columns (Optional[List[List[str]]]): Lists containing the column names for indexes to persist. The write
+            operation will store the index info as sidecar tables. By default, data indexes to write are determined by
+            those present on the source table. This argument is used to narrow the set of indexes to write, or to be
+            explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+            will be computed on demand.
 
     Raises:
         DHError
@@ -364,11 +383,20 @@ def write_partitioned(
         if col_definitions is not None:
             table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
 
-        if table_definition:
-            _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
-                                                         write_instructions)
+        if index_columns:
+            index_columns_array = _j_array_of_array_of_string(index_columns)
+            if table_definition:
+                _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
+                                                             write_instructions, index_columns_array)
+            else:
+                _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, write_instructions,
+                                                             index_columns_array)
         else:
-            _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, write_instructions)
+            if table_definition:
+                _JParquetTools.writeKeyValuePartitionedTable(table.j_object, table_definition, destination_dir,
+                                                             write_instructions)
+            else:
+                _JParquetTools.writeKeyValuePartitionedTable(table.j_object, destination_dir, write_instructions)
     except Exception as e:
         raise DHError(e, "failed to write to parquet data.") from e
 
@@ -382,23 +410,19 @@ def batch_write(
     max_dictionary_keys: Optional[int] = None,
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
-    grouping_cols: Optional[List[str]] = None,
     generate_metadata_files: Optional[bool] = None,
+    index_columns: Optional[List[List[str]]] = None
 ):
     """ Writes tables to disk in parquet format to a supplied set of paths.
 
-    If you specify grouping columns, there must already be grouping information for those columns in the sources.
-    This can be accomplished with .groupBy(<grouping columns>).ungroup() or .sort(<grouping column>).
-
     Note that either all the tables are written out successfully or none is.
 
     Args:
         tables (List[Table]): the source tables
-        paths (List[str]): the destinations paths. Any non existing directories in the paths provided are
+        paths (List[str]): the destination paths. Any non-existing directories in the paths provided are
             created. If there is an error, any intermediate directories previously created are removed; note this makes
             this method unsafe for concurrent use
-        col_definitions (Optional[List[Column]]): the column definitions to use for writing, instead of the definitions
-            implied by the table. Default is None, which means use the column definitions implied by the table
+        col_definitions (List[Column]): the column definitions to use for writing.
         col_instructions (Optional[List[ColumnInstruction]]): instructions for customizations while writing
         compression_codec_name (Optional[str]): the compression codec to use. Allowed values include "UNCOMPRESSED",
             "SNAPPY", "GZIP", "LZO", "LZ4", "LZ4_RAW", "ZSTD", etc. If not specified, defaults to "SNAPPY".
@@ -407,11 +431,15 @@ def batch_write(
         max_dictionary_size (Optional[int]): the maximum number of bytes the writer should add to the dictionary before
             switching to non-dictionary encoding, never evaluated for non-String columns, defaults to 2^20 (1,048,576)
         target_page_size (Optional[int]): the target page size in bytes, if not specified, defaults to 2^20 bytes (1 MiB)
-        grouping_cols (Optional[List[str]]): the group column names
         generate_metadata_files (Optional[bool]): whether to generate parquet _metadata and _common_metadata files,
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
+        index_columns (Optional[List[List[str]]]): Lists containing the column names for indexes to persist. The write
+            operation will store the index info as sidecar tables. By default, data indexes to write are determined by
+            those present on the source table. This argument is used to narrow the set of indexes to write, or to be
+            explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
+            will be computed on demand.
 
     Raises:
         DHError
@@ -429,9 +457,10 @@ def batch_write(
 
         table_definition = _JTableDefinition.of([col.j_column_definition for col in col_definitions])
 
-        if grouping_cols:
+        if index_columns:
+            index_columns_array = _j_array_of_array_of_string(index_columns)
             _JParquetTools.writeParquetTables([t.j_table for t in tables], table_definition, write_instructions,
-                                              _j_file_array(paths), grouping_cols)
+                                              _j_file_array(paths), index_columns_array)
         else:
             _JParquetTools.writeTables([t.j_table for t in tables], table_definition,
                                        _j_file_array(paths))
diff --git a/py/server/tests/test_parquet.py b/py/server/tests/test_parquet.py
index 6e512e88aad..04627e1c392 100644
--- a/py/server/tests/test_parquet.py
+++ b/py/server/tests/test_parquet.py
@@ -587,6 +587,10 @@ def test_read_parquet_from_s3(self):
             read("s3://dh-s3-parquet-test1/multiColFile.parquet", special_instructions=s3_instructions).select()
         # TODO(deephaven-core#5064): Add support for local S3 testing
 
+    def verify_index_files(self, index_dir_path, expected_num_index_files=1):
+        self.assertTrue(os.path.exists(index_dir_path))
+        self.assertTrue(len(os.listdir(index_dir_path)) == expected_num_index_files)
+
     def test_write_partitioned_data(self):
         source = new_table([
             string_col("X", ["Aa", "Bb", "Aa", "Cc", "Bb", "Aa", "Bb", "Bb", "Cc"]),
@@ -638,6 +642,12 @@ def verify_file_names():
         write_partitioned(partitioned_table, destination_dir=root_dir)
         verify_table_from_disk(read(root_dir))
 
+        shutil.rmtree(root_dir)
+        write_partitioned(partitioned_table, destination_dir=root_dir, index_columns=[["Y"], ["Y", "Number"]])
+        verify_table_from_disk(read(root_dir))
+        self.verify_index_files(os.path.join(root_dir, "X=Aa/.dh_metadata/indexes/Y"))
+        self.verify_index_files(os.path.join(root_dir, "X=Aa/.dh_metadata/indexes/Y,Number"))
+
         shutil.rmtree(root_dir)
         write_partitioned(source, col_definitions=definition, destination_dir=root_dir,
                           base_name=base_name, max_dictionary_keys=max_dictionary_keys)
@@ -660,6 +670,32 @@ def verify_file_names():
         write_partitioned(source, col_definitions=definition, destination_dir=root_dir)
         verify_table_from_disk(read(root_dir))
 
+        shutil.rmtree(root_dir)
+        write_partitioned(source, col_definitions=definition, destination_dir=root_dir,
+                          index_columns=[["Y"], ["Y", "Number"]])
+        verify_table_from_disk(read(root_dir))
+        self.verify_index_files(os.path.join(root_dir, "X=Aa/.dh_metadata/indexes/Y"))
+        self.verify_index_files(os.path.join(root_dir, "X=Aa/.dh_metadata/indexes/Y,Number"))
+
+    def test_write_with_index_columns(self):
+        first_table = empty_table(10).update(formulas=["x=i", "y=(double)(i/10.0)", "z=(double)(i*i)"])
+        write(first_table, "data_from_dh.parquet", index_columns=[["x"], ["y", "z"]])
+        from_disk = read("data_from_dh.parquet")
+        self.assert_table_equals(first_table, from_disk)
+        self.verify_index_files(".dh_metadata/indexes/x")
+        self.verify_index_files(".dh_metadata/indexes/y,z")
+        shutil.rmtree(".dh_metadata")
+
+        second_table = empty_table(10).update(formulas=["x=i*5", "y=(double)(i/5.0)", "z=(double)(i*i*i)"])
+        batch_write([first_table, second_table], ["X.parquet", "Y.parquet"], index_columns=[["x"], ["y", "z"]],
+                    col_definitions=first_table.columns)
+        from_disk_first_table = read("X.parquet")
+        self.assert_table_equals(first_table, from_disk_first_table)
+        from_disk_second_table = read("Y.parquet")
+        self.assert_table_equals(second_table, from_disk_second_table)
+        self.verify_index_files(".dh_metadata/indexes/x", expected_num_index_files=2)
+        self.verify_index_files(".dh_metadata/indexes/y,z", expected_num_index_files=2)
+
 
 if __name__ == '__main__':
     unittest.main()

From f700883e8df0aa5ddb88f39ddf7274864f684a2d Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Tue, 26 Mar 2024 19:37:43 -0500
Subject: [PATCH 42/45] Reordered comments

---
 .../src/main/java/io/deephaven/parquet/table/ParquetTools.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index ec91efcd1bf..a8b77754fb8 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -804,9 +804,9 @@ private static void writeKeyValuePartitionedTableImpl(@NotNull final Partitioned
         }
         final Table[] partitionedDataArray = partitionedData.toArray(Table[]::new);
         try (final SafeCloseable ignored = LivenessScopeStack.open()) {
-            // TODO(deephaven-core#5292): Optimize creating index on constituent tables
             final Map<String, Map<ParquetCacheTags, Object>> computedCache =
                     buildComputedCache(() -> sourceTable.orElseGet(partitionedTable::merge), leafDefinition);
+            // TODO(deephaven-core#5292): Optimize creating index on constituent tables
             // Store hard reference to prevent indexes from being garbage collected
             final List<DataIndex> dataIndexes = addIndexesToTables(partitionedDataArray, indexColumnArr);
             writeParquetTablesImpl(partitionedDataArray, leafDefinition, writeInstructions,

From 624628992782978162a9870e4c1be55f9cd0ad5c Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Wed, 27 Mar 2024 09:26:58 -0500
Subject: [PATCH 43/45] Added more details to python comments

---
 py/server/deephaven/parquet.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index d26345d05b1..ace76a226c7 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -273,11 +273,12 @@ def write(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
-        index_columns (Optional[List[List[str]]]): Lists containing the column names for indexes to persist. The write
-            operation will store the index info as sidecar tables. By default, data indexes to write are determined by
-            those present on the source table. This argument is used to narrow the set of indexes to write, or to be
-            explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
-            will be computed on demand.
+        index_columns (Optional[List[List[str]]]): list of list containing the column names for indexes to persist. The
+            write operation will store the index info for the provided columns as sidecar tables. For example, if the
+            input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for ["Col1"] and for
+            ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the source table.
+            This argument can be used to narrow the set of indexes to write, or to be explicit about the expected set of
+            indexes present on all sources. Indexes that are specified but missing will be computed on demand.
 
     Raises:
         DHError
@@ -358,11 +359,12 @@ def write_partitioned(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
-        index_columns (Optional[List[List[str]]]): Lists containing the column names for indexes to persist. The write
-            operation will store the index info as sidecar tables. By default, data indexes to write are determined by
-            those present on the source table. This argument is used to narrow the set of indexes to write, or to be
-            explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
-            will be computed on demand.
+        index_columns (Optional[List[List[str]]]): list of list containing the column names for indexes to persist. The
+            write operation will store the index info for the provided columns as sidecar tables. For example, if the
+            input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for ["Col1"] and for
+            ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the source table.
+            This argument can be used to narrow the set of indexes to write, or to be explicit about the expected set of
+            indexes present on all sources. Indexes that are specified but missing will be computed on demand.
 
     Raises:
         DHError
@@ -435,11 +437,12 @@ def batch_write(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
-        index_columns (Optional[List[List[str]]]): Lists containing the column names for indexes to persist. The write
-            operation will store the index info as sidecar tables. By default, data indexes to write are determined by
-            those present on the source table. This argument is used to narrow the set of indexes to write, or to be
-            explicit about the expected set of indexes present on all sources. Indexes that are specified but missing
-            will be computed on demand.
+        index_columns (Optional[List[List[str]]]): list of list containing the column names for indexes to persist. The
+            write operation will store the index info for the provided columns as sidecar tables. For example, if the
+            input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for ["Col1"] and for
+            ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the source table.
+            This argument can be used to narrow the set of indexes to write, or to be explicit about the expected set of
+            indexes present on all sources. Indexes that are specified but missing will be computed on demand.
 
     Raises:
         DHError

From fa2b0c5c4ecdfd0a7435cfb0520d34133745428c Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Wed, 27 Mar 2024 10:10:28 -0500
Subject: [PATCH 44/45] Moved from list to sequence

---
 py/server/deephaven/parquet.py | 52 ++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 25 deletions(-)

diff --git a/py/server/deephaven/parquet.py b/py/server/deephaven/parquet.py
index ace76a226c7..ea6b36504a4 100644
--- a/py/server/deephaven/parquet.py
+++ b/py/server/deephaven/parquet.py
@@ -6,7 +6,7 @@
 Parquet files. """
 from dataclasses import dataclass
 from enum import Enum
-from typing import List, Optional, Union, Dict
+from typing import List, Optional, Union, Dict, Sequence
 
 import jpy
 
@@ -220,8 +220,8 @@ def _j_file_array(paths: List[str]):
     return jpy.array("java.io.File", [_JFile(el) for el in paths])
 
 
-def _j_array_of_array_of_string(data_indexes: List[List[str]]):
-    return jpy.array("[Ljava.lang.String;", [jpy.array("java.lang.String", index_cols) for index_cols in data_indexes])
+def _j_array_of_array_of_string(index_columns: Sequence[Sequence[str]]):
+    return jpy.array("[Ljava.lang.String;", [jpy.array("java.lang.String", index_cols) for index_cols in index_columns])
 
 
 def delete(path: str) -> None:
@@ -249,7 +249,7 @@ def write(
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
     generate_metadata_files: Optional[bool] = None,
-    index_columns: Optional[List[List[str]]] = None
+    index_columns: Optional[Sequence[Sequence[str]]] = None
 ) -> None:
     """ Write a table to a Parquet file.
 
@@ -273,13 +273,13 @@ def write(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
-        index_columns (Optional[List[List[str]]]): list of list containing the column names for indexes to persist. The
-            write operation will store the index info for the provided columns as sidecar tables. For example, if the
-            input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for ["Col1"] and for
-            ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the source table.
-            This argument can be used to narrow the set of indexes to write, or to be explicit about the expected set of
-            indexes present on all sources. Indexes that are specified but missing will be computed on demand.
-
+        index_columns (Optional[Sequence[Sequence[str]]]): sequence of sequence containing the column names for indexes
+            to persist. The write operation will store the index info for the provided columns as sidecar tables. For
+            example, if the input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for
+            ["Col1"] and for ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the
+            source table. This argument can be used to narrow the set of indexes to write, or to be explicit about the
+            expected set of indexes present on all sources. Indexes that are specified but missing will be computed on
+            demand.
     Raises:
         DHError
     """
@@ -320,7 +320,7 @@ def write_partitioned(
         target_page_size: Optional[int] = None,
         base_name: Optional[str] = None,
         generate_metadata_files: Optional[bool] = None,
-        index_columns: Optional[List[List[str]]] = None
+        index_columns: Optional[Sequence[Sequence[str]]] = None
 ) -> None:
     """ Write table to disk in parquet format with the partitioning columns written as "key=value" format in a nested
     directory structure. For example, for a partitioned column "date", we will have a directory structure like
@@ -359,12 +359,13 @@ def write_partitioned(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
-        index_columns (Optional[List[List[str]]]): list of list containing the column names for indexes to persist. The
-            write operation will store the index info for the provided columns as sidecar tables. For example, if the
-            input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for ["Col1"] and for
-            ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the source table.
-            This argument can be used to narrow the set of indexes to write, or to be explicit about the expected set of
-            indexes present on all sources. Indexes that are specified but missing will be computed on demand.
+        index_columns (Optional[Sequence[Sequence[str]]]): sequence of sequence containing the column names for indexes
+            to persist. The write operation will store the index info for the provided columns as sidecar tables. For
+            example, if the input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for
+            ["Col1"] and for ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the
+            source table. This argument can be used to narrow the set of indexes to write, or to be explicit about the
+            expected set of indexes present on all sources. Indexes that are specified but missing will be computed on
+            demand.
 
     Raises:
         DHError
@@ -413,7 +414,7 @@ def batch_write(
     max_dictionary_size: Optional[int] = None,
     target_page_size: Optional[int] = None,
     generate_metadata_files: Optional[bool] = None,
-    index_columns: Optional[List[List[str]]] = None
+    index_columns: Optional[Sequence[Sequence[str]]] = None
 ):
     """ Writes tables to disk in parquet format to a supplied set of paths.
 
@@ -437,12 +438,13 @@ def batch_write(
             defaults to False. Generating these files can help speed up reading of partitioned parquet data because these
             files contain metadata (including schema) about the entire dataset, which can be used to skip reading some
             files.
-        index_columns (Optional[List[List[str]]]): list of list containing the column names for indexes to persist. The
-            write operation will store the index info for the provided columns as sidecar tables. For example, if the
-            input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for ["Col1"] and for
-            ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the source table.
-            This argument can be used to narrow the set of indexes to write, or to be explicit about the expected set of
-            indexes present on all sources. Indexes that are specified but missing will be computed on demand.
+        index_columns (Optional[Sequence[Sequence[str]]]): sequence of sequence containing the column names for indexes
+            to persist. The write operation will store the index info for the provided columns as sidecar tables. For
+            example, if the input is [["Col1"], ["Col1", "Col2"]], the write operation will store the index info for
+            ["Col1"] and for ["Col1", "Col2"]. By default, data indexes to write are determined by those present on the
+            source table. This argument can be used to narrow the set of indexes to write, or to be explicit about the
+            expected set of indexes present on all sources. Indexes that are specified but missing will be computed on
+            demand.
 
     Raises:
         DHError

From 7909ea096f73cc6ccffcea1be93a35e543377a6f Mon Sep 17 00:00:00 2001
From: Shivam Malhotra <malhotraashivam@gmail.com>
Date: Mon, 1 Apr 2024 11:53:02 -0500
Subject: [PATCH 45/45] Added fixes and tests for windows

---
 .../parquet/base/ColumnChunkReaderImpl.java   |   4 ++-
 .../deephaven/parquet/table/ParquetTools.java |  11 +++----
 .../layout/ParquetMetadataFileLayout.java     |  28 +++++++++---------
 .../location/ParquetTableLocationKey.java     |   4 ++-
 .../table/ParquetTableReadWriteTest.java      |  14 ++++++++-
 .../_common_metadata                          | Bin 0 -> 454 bytes
 .../_metadata                                 | Bin 0 -> 1565 bytes
 ...f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet |   3 ++
 ...f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet |   3 ++
 ...f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet |   3 ++
 ...f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet |   3 ++
 11 files changed, 51 insertions(+), 22 deletions(-)
 create mode 100644 extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/_common_metadata
 create mode 100644 extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/_metadata
 create mode 100644 extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2019/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
 create mode 100644 extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2020/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
 create mode 100644 extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2021/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
 create mode 100644 extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2022/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet

diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
index aafaecc1fdf..ce4030b895a 100644
--- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
+++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java
@@ -10,6 +10,7 @@
 import io.deephaven.parquet.compress.DeephavenCompressorAdapterFactory;
 import io.deephaven.util.channel.SeekableChannelContext.ContextHolder;
 import io.deephaven.util.datastructures.LazyCachingFunction;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.parquet.bytes.BytesInput;
 import org.apache.parquet.column.ColumnDescriptor;
 import org.apache.parquet.column.Dictionary;
@@ -75,7 +76,8 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader {
         this.numRows = numRows;
         this.version = version;
         if (columnChunk.isSetFile_path() && FILE_URI_SCHEME.equals(rootURI.getScheme())) {
-            this.columnChunkURI = convertToURI(Path.of(rootURI).resolve(columnChunk.getFile_path()), false);
+            final String relativePath = FilenameUtils.separatorsToSystem(columnChunk.getFile_path());
+            this.columnChunkURI = convertToURI(Path.of(rootURI).resolve(relativePath), false);
         } else {
             // TODO(deephaven-core#5066): Add support for reading metadata files from non-file URIs
             this.columnChunkURI = rootURI;
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
index a8b77754fb8..f76af02d108 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java
@@ -319,12 +319,12 @@ public static void writeTable(
         writeTables(new Table[] {sourceTable}, definition, new File[] {destFile}, writeInstructions);
     }
 
-    private static File getShadowFile(File destFile) {
+    private static File getShadowFile(final File destFile) {
         return new File(destFile.getParent(), ".NEW_" + destFile.getName());
     }
 
     @VisibleForTesting
-    static File getBackupFile(File destFile) {
+    static File getBackupFile(final File destFile) {
         return new File(destFile.getParent(), ".OLD_" + destFile.getName());
     }
 
@@ -339,15 +339,16 @@ private static String minusParquetSuffix(@NotNull final String s) {
      * Generates the index file path relative to the table destination file path.
      *
      * @param tableDest Destination path for the main table containing these indexing columns
-     * @param columnNames Array of names of the indexing columns
+     * @param columnNames Array of indexing column names
      *
      * @return The relative index file path. For example, for table with destination {@code "table.parquet"} and
      *         indexing column {@code "IndexingColName"}, the method will return
-     *         {@code ".dh_metadata/indexes/IndexingColName/index_IndexingColName_table.parquet"}
+     *         {@code ".dh_metadata/indexes/IndexingColName/index_IndexingColName_table.parquet"} on unix systems.
      */
     public static String getRelativeIndexFilePath(@NotNull final File tableDest, @NotNull final String[] columnNames) {
         final String columns = String.join(",", columnNames);
-        return String.format(".dh_metadata/indexes/%s/index_%s_%s", columns, columns, tableDest.getName());
+        return String.format(".dh_metadata%sindexes%s%s%sindex_%s_%s", File.separator, File.separator, columns,
+                File.separator, columns, tableDest.getName());
     }
 
     /**
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
index af5c6da54fd..6f279f4651c 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java
@@ -16,6 +16,7 @@
 import io.deephaven.parquet.table.location.ParquetTableLocationKey;
 import io.deephaven.parquet.table.ParquetInstructions;
 import io.deephaven.parquet.base.ParquetFileReader;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.parquet.format.converter.ParquetMetadataConverter;
 import io.deephaven.util.type.TypeUtils;
 import org.apache.commons.lang3.mutable.MutableInt;
@@ -143,21 +144,20 @@ public ParquetMetadataFileLayout(
         final Map<String, PartitionParser> partitionKeyToParser = partitioningColumns.stream().collect(toMap(
                 ColumnDefinition::getName,
                 cd -> PartitionParser.lookupSupported(cd.getDataType(), cd.getComponentType())));
-        final Map<String, TIntList> fileNameToRowGroupIndices = new LinkedHashMap<>();
+        final Map<String, TIntList> filePathToRowGroupIndices = new LinkedHashMap<>();
         final List<RowGroup> rowGroups = metadataFileReader.fileMetaData.getRow_groups();
         final int numRowGroups = rowGroups.size();
         for (int rgi = 0; rgi < numRowGroups; ++rgi) {
-            fileNameToRowGroupIndices
-                    .computeIfAbsent(rowGroups.get(rgi).getColumns().get(0).getFile_path(), fn -> new TIntArrayList())
-                    .add(rgi);
+            final String relativePath =
+                    FilenameUtils.separatorsToSystem(rowGroups.get(rgi).getColumns().get(0).getFile_path());
+            filePathToRowGroupIndices.computeIfAbsent(relativePath, fn -> new TIntArrayList()).add(rgi);
         }
         final File directory = metadataFile.getParentFile();
         final MutableInt partitionOrder = new MutableInt(0);
-        keys = fileNameToRowGroupIndices.entrySet().stream().map(entry -> {
-            final String filePathString = entry.getKey();
+        keys = filePathToRowGroupIndices.entrySet().stream().map(entry -> {
+            final String relativePathString = entry.getKey();
             final int[] rowGroupIndices = entry.getValue().toArray();
-
-            if (filePathString == null || filePathString.isEmpty()) {
+            if (relativePathString == null || relativePathString.isEmpty()) {
                 throw new TableDataException(String.format(
                         "Missing parquet file name for row groups %s in %s",
                         Arrays.toString(rowGroupIndices), metadataFile));
@@ -165,12 +165,12 @@ public ParquetMetadataFileLayout(
             final LinkedHashMap<String, Comparable<?>> partitions =
                     partitioningColumns.isEmpty() ? null : new LinkedHashMap<>();
             if (partitions != null) {
-                final Path filePath = Paths.get(filePathString);
+                final Path filePath = Paths.get(relativePathString);
                 final int numPartitions = filePath.getNameCount() - 1;
                 if (numPartitions != partitioningColumns.size()) {
                     throw new TableDataException(String.format(
                             "Unexpected number of path elements in %s for partitions %s",
-                            filePathString, partitions.keySet()));
+                            relativePathString, partitions.keySet()));
                 }
                 final boolean useHiveStyle = filePath.getName(0).toString().contains("=");
                 for (int pi = 0; pi < numPartitions; ++pi) {
@@ -182,7 +182,7 @@ public ParquetMetadataFileLayout(
                         if (pathComponents.length != 2) {
                             throw new TableDataException(String.format(
                                     "Unexpected path format found for hive-style partitioning from %s for %s",
-                                    filePathString, metadataFile));
+                                    relativePathString, metadataFile));
                         }
                         partitionKey = instructions.getColumnNameFromParquetColumnNameOrDefault(pathComponents[0]);
                         partitionValueRaw = pathComponents[1];
@@ -195,16 +195,16 @@ public ParquetMetadataFileLayout(
                     if (partitions.containsKey(partitionKey)) {
                         throw new TableDataException(String.format(
                                 "Unexpected duplicate partition key %s when parsing %s for %s",
-                                partitionKey, filePathString, metadataFile));
+                                partitionKey, relativePathString, metadataFile));
                     }
                     partitions.put(partitionKey, partitionValue);
                 }
             }
-            final File partitionFile = new File(directory, filePathString);
+            final File partitionFile = new File(directory, relativePathString);
             final ParquetTableLocationKey tlk = new ParquetTableLocationKey(partitionFile,
                     partitionOrder.getAndIncrement(), partitions, inputInstructions);
             tlk.setFileReader(metadataFileReader);
-            tlk.setMetadata(getParquetMetadataForFile(filePathString, metadataFileMetadata));
+            tlk.setMetadata(getParquetMetadataForFile(relativePathString, metadataFileMetadata));
             tlk.setRowGroupIndices(rowGroupIndices);
             return tlk;
         }).collect(Collectors.toList());
diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
index 0aea3207f17..a044c309cea 100644
--- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
+++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java
@@ -9,6 +9,7 @@
 import io.deephaven.engine.table.impl.locations.TableDataException;
 import io.deephaven.engine.table.impl.locations.TableLocationKey;
 import io.deephaven.parquet.base.ParquetFileReader;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.parquet.format.converter.ParquetMetadataConverter;
 import org.apache.parquet.format.RowGroup;
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
@@ -190,7 +191,8 @@ public synchronized int[] getRowGroupIndices() {
             // While it seems that row group *could* have column chunks splayed out into multiple files,
             // we're not expecting that in this code path. To support it, discovery tools should figure out
             // the row groups for a partition themselves and call setRowGroupReaders.
-            final String filePath = rowGroups.get(rgi).getColumns().get(0).getFile_path();
+            final String filePath =
+                    FilenameUtils.separatorsToSystem(rowGroups.get(rgi).getColumns().get(0).getFile_path());
             return filePath == null || convertToURI(filePath, false).equals(uri);
         }).toArray();
     }
diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
index 5ad5348025d..531348f2d17 100644
--- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
+++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java
@@ -1433,7 +1433,7 @@ private Table maybeFixBigDecimal(Table toFix) {
 
     private static Table readParquetFileFromGitLFS(final File dest) {
         try {
-            return readSingleFileTable(dest, EMPTY);
+            return readTable(dest, EMPTY);
         } catch (final RuntimeException e) {
             if (e.getCause() instanceof InvalidParquetFileException) {
                 final String InvalidParquetFileErrorMsgString = "Invalid parquet file detected, please ensure the " +
@@ -1623,6 +1623,18 @@ public void basicWriteTests() {
         basicWriteTestsImpl(MULTI_WRITER);
     }
 
+    @Test
+    public void readPartitionedDataGeneratedOnWindows() {
+        final String path = ParquetTableReadWriteTest.class
+                .getResource("/referencePartitionedDataFromWindows").getFile();
+        final Table partitionedDataFromWindows = readParquetFileFromGitLFS(new File(path)).select();
+        final Table expected = TableTools.newTable(
+                longCol("year", 2019, 2020, 2021, 2021, 2022, 2022),
+                longCol("n_legs", 5, 2, 4, 100, 2, 4),
+                stringCol("animal", "Brittle stars", "Flamingo", "Dog", "Centipede", "Parrot", "Horse"));
+        assertTableEquals(expected, partitionedDataFromWindows.sort("year"));
+    }
+
     private static void basicWriteTestsImpl(TestParquetTableWriter writer) {
         // Create an empty parent directory
         final File parentDir = new File(rootFile, "tempDir");
diff --git a/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/_common_metadata b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/_common_metadata
new file mode 100644
index 0000000000000000000000000000000000000000..439dd5c41b394a8b4c32a5a88706978112d86596
GIT binary patch
literal 454
zcmYjO!A^rf5M3L>i7N(@9=P;mT5XlY7<-sStH!og4r0BSm29Aqwg^`A&x~K^%&a6%
zVD`;h-rJdF0rAka{AnmWyH0bxR<12vtJm?YFRdyA?agzcvs$<g^0wNn)~e`<X;%ob
zFMP-EdJe?#U2<j|yzkB-#Q{*<a|XBtXh3E#0mvZAn3WP>MkoTfp%v?P#frA%3Gs7E
zY!$tdH5O9FOs14k*TxlrWfGPz$)rYjNjl!2U!d}os0c|EaT5%wL>0br_c;12o{sVC
zS^GUm9s*6UdF*cqvN3(LIcxEVoB^Bu>xcby@{p%yX4E${?={`SUl@NFkA)YMs(Rb$
m_kLQIeN|Q4Ciu`*z1nVqVK_J%jQn%Fck_s!Js}qO2Y&%_pL4hX

literal 0
HcmV?d00001

diff --git a/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/_metadata b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/_metadata
new file mode 100644
index 0000000000000000000000000000000000000000..d474961e0cb8b7e9a36d2327335e2821a27c12e9
GIT binary patch
literal 1565
zcmb`IKX21O7{>1$H!@O?iiA&R$wQV}9g^a>Ntz%soYO`@6i6dbDMCU{;<%BVKiDpT
zZ@@?Bz|euAp8zo+lnGT#9hp!E1|}wkPKdj6oQPs1Qt?W;d*1h)dmsOlyEI5MCyUCh
z0>DYXy>Is|Sp>4UBH-lUscSnvkW(zN2ED%Ju7GV70Dw};wvxg5m$v0q4Bf~*=@wgh
zv1^w%+T~8ZST<UgZnfIoe5qTqTkA$QtFMhL@7aMJpwCdH4GSQ-tVBIz0g31YqEo2V
zHA%%%hc;n>B{38Lfz_BQP%^S816crcSf&&G%H>>5L{&~juELk)u8tr&hT2&|J4~oW
zu$tH$I`~S}9t6FS-LYf(qi^(OGzpyx*8yFg*~yg`fpiiJNkH!)nn10@AuUbOU@1}z
zQl#hA7?YQD^)VKbfZjs%0cxKT+A9v}s_FHDz_rtUV0k|NJ&kdLz6(g8$bMcxX6XML
z8T)~-z*3|bvPjRXF(zlo7ISs{7i5PJ9YJjpA(LvZ)$a|QA)cui59l-mb#@Vyu?VPt
zIe^izn<HVM{;bBBoP%1()%#dbB6<zc8>oGW4q&ojdEPL<OZB1W$DY5h;raW)8L2o$
zn$7#WH^aLPpTiq0HVBIyJ`!?=kTEeGQX|A6RfktilaRYiRSCJxW|~Jc&6qT|rgcK@
zJ)oPGXu4Gki(|ILB@=|~(T};`>bt?-E?e7S`|));JY;=o|2p##EiyqLsq%-Y36}7Y
z4}8CaN5%{H@jA2VkWFr?&HPS-zRoNv6@b&z8#vn@jk1j5IQ`uA{N8Yo&K1`5HABhJ
P%bnft*OCA<=pVx`N!m{Q

literal 0
HcmV?d00001

diff --git a/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2019/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2019/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
new file mode 100644
index 00000000000..000b0d8799f
--- /dev/null
+++ b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2019/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07c780c6a7092f19dd25f0c36070ca5a1e754e8fe8cf54305b56a6546fe7e4c0
+size 950
diff --git a/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2020/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2020/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
new file mode 100644
index 00000000000..2d110f68cf1
--- /dev/null
+++ b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2020/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e76b0d962781c58d9cb5e9229b66ff2165b326e2a21bbc80f2fd71235468b14
+size 915
diff --git a/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2021/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2021/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
new file mode 100644
index 00000000000..58c883c132f
--- /dev/null
+++ b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2021/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:100ee2b1f38680e5536e40ee7eda1aaf5c0def26219fa993c3ccfd7b35fa2507
+size 919
diff --git a/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2022/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2022/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
new file mode 100644
index 00000000000..0850d74f0f0
--- /dev/null
+++ b/extensions/parquet/table/src/test/resources/referencePartitionedDataFromWindows/year=2022/f6b06fe97c9d4692ba0abcf48f8eb32f-0.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:819cb36fd6d76c59de6c6ef11b424f1f4d42c88749d07d257b7d599a85a1c94e
+size 915