From 16c96168def2080e12056cd34b9971b553023457 Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Fri, 23 Jun 2023 15:39:32 -0600 Subject: [PATCH] Java Arrow: Upgrade to 12.0.1; Use CommonsCompressionFactory for Feather (#4061) Fixes #3793. Apache Arrow in Java was failing to parse python generated feather files as python compresses by default. Release Notes: Upgraded apache arrow for Java from 11.0.0 to version 12.0.1. --- buildSrc/src/main/groovy/Classpaths.groovy | 2 +- extensions/arrow/build.gradle | 1 + .../java/io/deephaven/extensions/arrow/ArrowWrapperTools.java | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/Classpaths.groovy b/buildSrc/src/main/groovy/Classpaths.groovy index fbe8f05c96f..c5b806dd1f6 100644 --- a/buildSrc/src/main/groovy/Classpaths.groovy +++ b/buildSrc/src/main/groovy/Classpaths.groovy @@ -46,7 +46,7 @@ class Classpaths { static final String COMMONS_GROUP = 'org.apache.commons' static final String ARROW_GROUP = 'org.apache.arrow' - static final String ARROW_VERSION = '11.0.0' + static final String ARROW_VERSION = '12.0.1' static final String SLF4J_GROUP = 'org.slf4j' static final String SLF4J_VERSION = '2.0.6' diff --git a/extensions/arrow/build.gradle b/extensions/arrow/build.gradle index 453c2456ede..93996ff889a 100644 --- a/extensions/arrow/build.gradle +++ b/extensions/arrow/build.gradle @@ -13,6 +13,7 @@ dependencies { Classpaths.inheritArrow(project, 'arrow-format', 'implementation') Classpaths.inheritArrow(project, 'arrow-vector', 'implementation') + Classpaths.inheritArrow(project, 'arrow-compression', 'implementation') testImplementation TestTools.projectDependency(project, 'engine-table'), TestTools.projectDependency(project, 'Util') diff --git a/extensions/arrow/src/main/java/io/deephaven/extensions/arrow/ArrowWrapperTools.java b/extensions/arrow/src/main/java/io/deephaven/extensions/arrow/ArrowWrapperTools.java index ad9c1c8f957..2c92c8a3410 100644 --- a/extensions/arrow/src/main/java/io/deephaven/extensions/arrow/ArrowWrapperTools.java +++ b/extensions/arrow/src/main/java/io/deephaven/extensions/arrow/ArrowWrapperTools.java @@ -52,6 +52,7 @@ import io.deephaven.util.annotations.TestUseOnly; import io.deephaven.util.datastructures.LongSizedDataStructure; import io.deephaven.util.datastructures.SizeException; +import org.apache.arrow.compression.CommonsCompressionFactory; import org.apache.arrow.flatbuf.Message; import org.apache.arrow.flatbuf.RecordBatch; import org.apache.arrow.memory.BufferAllocator; @@ -403,7 +404,8 @@ private Shareable newShareable() { } catch (IOException e) { throw new UncheckedIOException(e); } - return new Shareable(this, new ArrowFileReader(channel, rootAllocator)); + return new Shareable(this, new ArrowFileReader( + channel, rootAllocator, CommonsCompressionFactory.INSTANCE)); } @NotNull