Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Raise exception when reading non-UTC adjusted timestamps in parquet #4421

Merged
merged 13 commits into from
Sep 19, 2023
Merged

Raise exception when reading non-UTC adjusted timestamps in parquet #4421

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
a4c752b
Initial commit
malhotrashivam Aug 31, 2023
cd77c6c
Fixed the failing tests
malhotrashivam Aug 31, 2023
c28dbfc
Merge branch 'main' into sm-pq-time-fail
malhotrashivam Aug 31, 2023
c69e038
Regenerated reference files
malhotrashivam Aug 31, 2023
5a23cd8
Merging with other change
malhotrashivam Aug 31, 2023
b7d846e
Improved the exception message
malhotrashivam Sep 15, 2023
1339e69
Removing some unnecessary changes
malhotrashivam Sep 15, 2023
531bb68
Moved binary parquet files to Git LFS
malhotrashivam Sep 15, 2023
2e14fc1
Merge branch 'main' into sm-pq-time-fail
malhotrashivam Sep 15, 2023
b457d73
Added Git LFS tag to Github checkout actions
malhotrashivam Sep 18, 2023
c50cc25
Merge branch 'main' into sm-pq-time-fail
malhotrashivam Sep 18, 2023
1860c7e
Reverted LFS pulls to just test workflows on Github
malhotrashivam Sep 19, 2023
6391820
Move all LFS gitattributes to the repo root
malhotrashivam Sep 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion extensions/parquet/table/src/test/e0.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
"c": np.arange(3, 6).astype("u1"),
"d": np.arange(4.0, 7.0, dtype="float64"),
"e": [True, False, True],
"f": pd.date_range("20130101", periods=3),
# TODO(deephaven-core#976): Unable to read parquet TimestampLogicalTypeAnnotation that is not adjusted to UTC
# "f": pd.date_range("20130101", periods=3),
"g": pd.date_range("20130101", periods=3, tz="US/Eastern"),
"h": pd.Categorical(list("abc")),
"i": pd.Categorical(list("abc"), ordered=True),
Expand Down
3 changes: 2 additions & 1 deletion extensions/parquet/table/src/test/e1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
"c": np.arange(3, 6).astype("u1"),
"d": np.arange(4.0, 7.0, dtype="float64"),
"e": [True, False, True],
"f": pd.date_range("20130101", periods=3),
# TODO(deephaven-core#976): Unable to read parquet TimestampLogicalTypeAnnotation that is not adjusted to UTC
# "f": pd.date_range("20130101", periods=3),
"g": pd.date_range("20130101", periods=3, tz="US/Eastern"),
"h": pd.Categorical(list("abc")),
"i": pd.Categorical(list("abc"), ordered=True),
Expand Down
3 changes: 2 additions & 1 deletion extensions/parquet/table/src/test/e2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
"c": np.arange(3, 6).astype("u1"),
"d": np.arange(4.0, 7.0, dtype="float64"),
"e": [True, False, True],
"f": pd.date_range("20130101", periods=3),
# TODO(deephaven-core#976): Unable to read parquet TimestampLogicalTypeAnnotation that is not adjusted to UTC
# "f": pd.date_range("20130101", periods=3),
"g": pd.date_range("20130101", periods=3, tz="US/Eastern"),
"h": pd.Categorical(list("abc")),
"i": pd.Categorical(list("abc"), ordered=True),
Expand Down
43 changes: 21 additions & 22 deletions extensions/parquet/table/src/test/java/io/deephaven/parquet/table/TestParquetTools.java
View file
Open in desktop
malhotrashivam marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@

import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.nio.file.Files;
import java.util.ArrayList;
Expand Down Expand Up @@ -432,8 +430,9 @@ public void e0() {
final Table gzip = ParquetTools.readTable(TestParquetTools.class.getResource("/e0/gzip.parquet").getFile());
assertTableEquals(uncompressed, gzip);

final Table lz4 = ParquetTools.readTable(TestParquetTools.class.getResource("/e0/lz4.parquet").getFile());
assertTableEquals(uncompressed, lz4);
// TODO(deephaven-core#3585): LZ4_RAW parquet support
// final Table lz4 = ParquetTools.readTable(TestParquetTools.class.getResource("/e0/lz4.parquet").getFile());
// assertTableEquals(uncompressed, lz4);

final Table snappy = ParquetTools.readTable(TestParquetTools.class.getResource("/e0/snappy.parquet").getFile());
assertTableEquals(uncompressed, snappy);
Expand Down Expand Up @@ -461,24 +460,24 @@ public void e1() {
assertTableEquals(uncompressed, zstd);
}

// TODO(deephaven-core#3588): Unable to read parquet TimestampLogicalTypeAnnotation that is not adjusted to UTC
// @Test
// public void e2() {
// final Table uncompressed =
// ParquetTools.readTable(TestParquetTools.class.getResource("/e2/uncompressed.parquet").getFile());
//
// final Table gzip = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/gzip.parquet").getFile());
// assertTableEquals(uncompressed, gzip);
//
// final Table lz4 = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/lz4.parquet").getFile());
// assertTableEquals(uncompressed, lz4);
//
// final Table snappy = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/snappy.parquet").getFile());
// assertTableEquals(uncompressed, snappy);
//
// final Table zstd = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/zstd.parquet").getFile());
// assertTableEquals(uncompressed, zstd);
// }
@Test
public void e2() {
final Table uncompressed =
ParquetTools.readTable(TestParquetTools.class.getResource("/e2/uncompressed.parquet").getFile());

final Table gzip = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/gzip.parquet").getFile());
assertTableEquals(uncompressed, gzip);

// TODO(deephaven-core#3585): LZ4_RAW parquet support
// final Table lz4 = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/lz4.parquet").getFile());
// assertTableEquals(uncompressed, lz4);

final Table snappy = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/snappy.parquet").getFile());
assertTableEquals(uncompressed, snappy);

final Table zstd = ParquetTools.readTable(TestParquetTools.class.getResource("/e2/zstd.parquet").getFile());
assertTableEquals(uncompressed, zstd);
}

private void testWriteRead(Table source, Function<Table, Table> transform) {
final File f2w = new File(testRoot, "testWriteRead.parquet");
Expand Down
Binary file modified extensions/parquet/table/src/test/resources/e0/brotli.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e0/gzip.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e0/lz4.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e0/snappy.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e0/uncompressed.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e0/zstd.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e1/brotli.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e1/gzip.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e1/lz4.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e1/snappy.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e1/uncompressed.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e1/zstd.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e2/brotli.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e2/gzip.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e2/lz4.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e2/snappy.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e2/uncompressed.parquet
View file
Open in desktop
Binary file not shown.
Binary file modified extensions/parquet/table/src/test/resources/e2/zstd.parquet
View file
Open in desktop
Binary file not shown.
Loading