test: add data acceptance tests

rust/Cargo.toml

@@ -109,13 +109,16 @@ dynamodb_lock = { version = "0", default-features = false, optional = true }
 
 [dev-dependencies]
 dotenvy = "0"
+flate2 = "1.0.26"
 maplit = "1"
 pretty_assertions = "1.2.1"
 rand = "0.8"
 serial_test = "2"
+tar = "0.4.39"
 tempdir = "0"
 tempfile = "3"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+ureq = "2.7.1"
 utime = "0.3"
 
 [features]
@@ -131,18 +134,7 @@ arrow = [
     "arrow-buffer",
 ]
 default = ["arrow", "parquet"]
-datafusion = [
-    "dep:datafusion",
-    "datafusion-expr",
-    "datafusion-common",
-    "datafusion-proto",
-    "datafusion-physical-expr",
-    "datafusion-sql",
-    "sqlparser",
-    "arrow",
-    "parquet",
-    "dashmap",
-]
+datafusion = ["dep:datafusion", "datafusion-expr", "datafusion-common", "datafusion-proto", "datafusion-physical-expr", "datafusion-sql", "sqlparser", "arrow", "parquet", "dashmap"]
 datafusion-ext = ["datafusion"]
 gcs = ["object_store/gcp"]
 glue = ["s3", "rusoto_glue/rustls"]
@@ -152,22 +144,8 @@ hdfs = ["datafusion-objectstore-hdfs"]
 integration_test = ["fs_extra", "tempdir"]
 json = ["parquet/json"]
 python = ["arrow/pyarrow"]
-s3-native-tls = [
-    "rusoto_core/native-tls",
-    "rusoto_credential",
-    "rusoto_sts/native-tls",
-    "rusoto_dynamodb/native-tls",
-    "dynamodb_lock/native-tls",
-    "object_store/aws",
-]
-s3 = [
-    "rusoto_core/rustls",
-    "rusoto_credential",
-    "rusoto_sts/rustls",
-    "rusoto_dynamodb/rustls",
-    "dynamodb_lock/rustls",
-    "object_store/aws",
-]
+s3-native-tls = ["rusoto_core/native-tls", "rusoto_credential", "rusoto_sts/native-tls", "rusoto_dynamodb/native-tls", "dynamodb_lock/native-tls", "object_store/aws"]
+s3 = ["rusoto_core/rustls", "rusoto_credential", "rusoto_sts/rustls", "rusoto_dynamodb/rustls", "dynamodb_lock/rustls", "object_store/aws"]
 unity-experimental = ["reqwest", "reqwest-middleware", "reqwest-retry"]
 
 [[bench]]

rust/tests/dat.rs

@@ -0,0 +1,184 @@
+use datafusion::common::{DFField, DFSchema};
+use datafusion::prelude::*; //{DataFrame, ParquetReadOptions, SessionContext};
+use deltalake::open_table;
+use serde::Deserialize;
+use std::path::Path;
+use std::sync::Arc;
+pub type TestResult = Result<(), Box<dyn std::error::Error + 'static>>;
+use arrow::datatypes::{DataType, TimeUnit};
+use std::sync::Once;
+
+static INIT: Once = Once::new();
+
+fn initialize() {
+    INIT.call_once(setup::run);
+}
+
+pub mod setup {
+    //! Build script for DAT
+    use std::fs::File;
+    use std::io::{BufReader, BufWriter, Write};
+    use std::path::Path;
+
+    use flate2::read::GzDecoder;
+    use tar::Archive;
+
+    const DAT_EXISTS_FILE_CHECK: &str = "tests/data/dat/v0.0.2/.done";
+    const VERSION: &str = "0.0.2";
+    pub const OUTPUT_FOLDER: &str = "tests/data/dat/v0.0.2";
+
+    pub fn run() {
+        if dat_exists() {
+            return;
+        }
+
+        let tarball_data = download_dat_files();
+        extract_tarball(tarball_data);
+        write_done_file();
+    }
+
+    fn dat_exists() -> bool {
+        Path::new(DAT_EXISTS_FILE_CHECK).exists()
+    }
+
+    fn download_dat_files() -> Vec<u8> {
+        let tarball_url = format!(
+        "https://github.com/delta-incubator/dat/releases/download/v{version}/deltalake-dat-v{version}.tar.gz",
+        version = VERSION
+    );
+
+        let response = ureq::get(&tarball_url).call().unwrap();
+        let mut tarball_data: Vec<u8> = Vec::new();
+        response
+            .into_reader()
+            .read_to_end(&mut tarball_data)
+            .unwrap();
+
+        tarball_data
+    }
+
+    fn extract_tarball(tarball_data: Vec<u8>) {
+        let tarball = GzDecoder::new(BufReader::new(&tarball_data[..]));
+        let mut archive = Archive::new(tarball);
+        std::fs::create_dir_all(OUTPUT_FOLDER).expect("Failed to create output directory");
+        archive
+            .unpack(OUTPUT_FOLDER)
+            .expect("Failed to unpack tarball");
+    }
+
+    fn write_done_file() {
+        let mut done_file = BufWriter::new(
+            File::create(DAT_EXISTS_FILE_CHECK).expect("Failed to create .done file"),
+        );
+        write!(done_file, "done").expect("Failed to write .done file");
+    }
+}
+
+/// Utility for comparing a delta table
+/// with a dataframe.
+async fn deltaeq(
+    ctx: &SessionContext,
+    delta_ctx_name: &str,
+    expected: DataFrame,
+) -> datafusion::common::Result<bool> {
+    let delta_df = ctx.table(delta_ctx_name).await?;
+    let delta_df_count = delta_df.clone().count().await?;
+    let counts_eq = delta_df_count == expected.clone().count().await?;
+    if counts_eq {
+        let intersecting_table = &delta_df.intersect(expected)?;
+        Ok(intersecting_table.clone().count().await? == delta_df_count)
+    } else {
+        Ok(false)
+    }
+}
+#[derive(Deserialize, Debug)]
+#[allow(dead_code)]
+struct TableVersionMetadata {
+    version: i64,
+    properties: serde_json::Value,
+    min_reader_version: i32,
+    min_writer_version: i32,
+}
+
+#[macro_export]
+macro_rules! dat_test {
+    ($( $test_name:ident $test:literal),*) => {
+        $(
+#[tokio::test]
+#[cfg(feature = "datafusion")]
+async fn $test_name() -> TestResult {
+    initialize();
+    let test_case = Path::new($test);
+    let root = &format!("{output_folder}/out/reader_tests/generated", output_folder=setup::OUTPUT_FOLDER);
+    let root = Path::new(root).join(test_case);
+    let actual_path = root.join(Path::new("delta"));
+    let expected_path_root = root.join(Path::new("expected"));
+    let actual = open_table(&actual_path.to_str().unwrap()).await?;
+    let max_verison = actual.version();
+    for version in (0..=max_verison).rev() {
+        let ctx: SessionContext = SessionContext::new();
+        let vstring = match max_verison == version {
+            true => "latest".to_owned(),
+            false => "v".to_owned() + &version.to_string(),
+        };
+
+        let expected_path = expected_path_root
+            .join(Path::new(&vstring))
+            .join(Path::new("table_content"));
+        if !expected_path.exists(){
+            continue;
+        }
+        let expected_metadata_path = expected_path_root.join(Path::new(&vstring)).join(Path::new("table_version_metadata.json"));
+        let expected_metadata_rdr = std::fs::File::open(expected_metadata_path.to_str().unwrap())?;
+
+        let expected_metadata: TableVersionMetadata = serde_json::from_reader(expected_metadata_rdr)?;
+        let expected = ctx.read_parquet(expected_path.to_str().unwrap(), ParquetReadOptions::default()).await?;
+
+        let mut actual = open_table(&actual_path.to_str().unwrap()).await?;
+
+        let expected_schema_fields = &expected.schema().fields();
+        let expected = expected.clone().select(expected_schema_fields.iter().map(
+            |f| {
+                if f.data_type().is_temporal()
+                {
+                    cast(col(f.name()), DataType::Timestamp(TimeUnit::Millisecond, None))
+                }
+                else {
+                    col(f.name())
+                }
+            }
+        ).collect::<Vec<_>>()
+    )?;
+
+
+
+
+        if actual.version() != version{
+            actual.load_version(version).await?;
+        }
+        assert!(expected_metadata.version == actual.version());
+        assert!(expected_metadata.min_reader_version  == actual.get_min_reader_version());
+        assert!(expected_metadata.min_writer_version  == actual.get_min_writer_version());
+        ctx.register_table("actual", Arc::new(actual))?;
+        assert!(deltaeq(&ctx, "actual", expected).await?);
+    }
+    Ok(())
+}
+
+        )*
+    }
+}
+
+dat_test!(
+    test_all_primitive_types "all_primitive_types",
+    test_basic_append "basic_append",
+    test_basic_partitioned "basic_partitioned",
+    // test_multi_partitioned "multi_partitioned",
+    // test_multi_partitioned_2 "multi_partitioned_2",
+    // test_nested_types "nested_types",
+    test_no_replay "no_replay",
+    // test_no_stats "no_stats",
+    test_stats_as_struct "stats_as_struct",
+    test_with_checkpoint "with_checkpoint",
+    test_with_schema_change "with_schema_change"
+);

rust/tests/data/dat/v0.0.2/.done

@@ -0,0 +1 @@
+done

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/all_primitive_types/delta/_delta_log/00000000000000000000.json

@@ -0,0 +1,4 @@
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"metaData":{"id":"31f7f2bf-079b-4edd-8da3-aee322aafae7","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"utf8\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int64\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int32\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int16\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int8\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float32\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float64\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(5,3)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date32\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1674611450140}}
+{"add":{"path":"part-00000-7a2970c5-4141-4f83-8d2e-993361e592b5-c000.snappy.parquet","partitionValues":{},"size":3203,"modificationTime":1674611450397,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"utf8\":\"0\",\"int64\":0,\"int32\":0,\"int16\":0,\"int8\":0,\"float32\":0.0,\"float64\":0.0,\"decimal\":10.000,\"date32\":\"1970-01-01\",\"timestamp\":\"1970-01-01T00:00:00.000Z\"},\"maxValues\":{\"utf8\":\"4\",\"int64\":4,\"int32\":4,\"int16\":4,\"int8\":4,\"float32\":4.0,\"float64\":4.0,\"decimal\":14.000,\"date32\":\"1970-01-05\",\"timestamp\":\"1970-01-01T04:00:00.000Z\"},\"nullCount\":{\"utf8\":0,\"int64\":0,\"int32\":0,\"int16\":0,\"int8\":0,\"float32\":0,\"float64\":0,\"bool\":0,\"binary\":0,\"decimal\":0,\"date32\":0,\"timestamp\":0}}"}}
+{"commitInfo":{"timestamp":1674611450410,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"3203"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"282b2c57-6935-492a-b793-16e6c131e2f5"}}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/all_primitive_types/expected/latest/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 0,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/all_primitive_types/test_case_info.json

@@ -0,0 +1,4 @@
+{
+  "name": "all_primitive_types",
+  "description": "Table containing all non-nested types"
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000000.json

@@ -0,0 +1,4 @@
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"metaData":{"id":"9f760748-1ae1-4ac5-bc9f-b8905ba72d80","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1674611410834}}
+{"add":{"path":"part-00000-ef42f28f-e8e8-4d54-b51f-c3af96c72a44-c000.snappy.parquet","partitionValues":{},"size":996,"modificationTime":1674611413765,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}}
+{"commitInfo":{"timestamp":1674611413970,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"996"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"92148ecd-7478-49eb-b824-3360b0e37bb1"}}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000001.json

@@ -0,0 +1,2 @@
+{"add":{"path":"part-00000-c156ac8b-f738-4479-803d-750072dd4c51-c000.snappy.parquet","partitionValues":{},"size":984,"modificationTime":1674611422849,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}}
+{"commitInfo":{"timestamp":1674611422877,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"984"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"acf16676-bf8e-4fac-942f-3cae8e384d2d"}}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_append/expected/latest/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 1,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_append/expected/v0/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 0,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_append/expected/v1/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 1,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_append/test_case_info.json

@@ -0,0 +1,4 @@
+{
+  "name": "basic_append",
+  "description": "A basic table with two append writes."
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000000.json

@@ -0,0 +1,6 @@
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"metaData":{"id":"ced0baf6-aa13-4871-af26-91e6e2787052","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["letter"],"configuration":{},"createdTime":1674611426764}}
+{"add":{"path":"letter=a/part-00000-a08d296a-d2c5-4a99-bea9-afcea42ba2e9.c000.snappy.parquet","partitionValues":{"letter":"a"},"size":751,"modificationTime":1674611427093,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":1,\"a_float\":1.1},\"maxValues\":{\"number\":1,\"a_float\":1.1},\"nullCount\":{\"number\":0,\"a_float\":0}}"}}
+{"add":{"path":"letter=b/part-00000-41954fb0-ef91-47e5-bd41-b75169c41c17.c000.snappy.parquet","partitionValues":{"letter":"b"},"size":751,"modificationTime":1674611427109,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":2,\"a_float\":2.2},\"maxValues\":{\"number\":2,\"a_float\":2.2},\"nullCount\":{\"number\":0,\"a_float\":0}}"}}
+{"add":{"path":"letter=c/part-00000-27a17b8f-be68-485c-9c49-70c742be30c0.c000.snappy.parquet","partitionValues":{"letter":"c"},"size":751,"modificationTime":1674611427117,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":3,\"a_float\":3.3},\"maxValues\":{\"number\":3,\"a_float\":3.3},\"nullCount\":{\"number\":0,\"a_float\":0}}"}}
+{"commitInfo":{"timestamp":1674611427131,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"letter\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"2253"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"6814b5cf-8e72-4add-83a3-3f79dda94dd6"}}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000001.json

@@ -0,0 +1,4 @@
+{"add":{"path":"letter=__HIVE_DEFAULT_PARTITION__/part-00000-8eb7f29a-e6a1-436e-a638-bbf0a7953f09.c000.snappy.parquet","partitionValues":{"letter":null},"size":751,"modificationTime":1674611429929,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":6,\"a_float\":6.6},\"maxValues\":{\"number\":6,\"a_float\":6.6},\"nullCount\":{\"number\":0,\"a_float\":0}}"}}
+{"add":{"path":"letter=a/part-00000-0dbe0cc5-e3bf-4fb0-b36a-b5fdd67fe843.c000.snappy.parquet","partitionValues":{"letter":"a"},"size":751,"modificationTime":1674611429937,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":4,\"a_float\":4.4},\"maxValues\":{\"number\":4,\"a_float\":4.4},\"nullCount\":{\"number\":0,\"a_float\":0}}"}}
+{"add":{"path":"letter=e/part-00000-847cf2d1-1247-4aa0-89ef-2f90c68ea51e.c000.snappy.parquet","partitionValues":{"letter":"e"},"size":750,"modificationTime":1674611429949,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":5,\"a_float\":5.5},\"maxValues\":{\"number\":5,\"a_float\":5.5},\"nullCount\":{\"number\":0,\"a_float\":0}}"}}
+{"commitInfo":{"timestamp":1674611429957,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"letter\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"2252"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"c6d7e8e6-bc66-46fb-b44a-58e0d47414f5"}}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_partitioned/expected/latest/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 1,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_partitioned/expected/v0/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 0,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_partitioned/expected/v1/table_version_metadata.json

@@ -0,0 +1,6 @@
+{
+  "version": 1,
+  "properties": {},
+  "min_reader_version": 1,
+  "min_writer_version": 2
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/basic_partitioned/test_case_info.json

@@ -0,0 +1,4 @@
+{
+  "name": "basic_partitioned",
+  "description": "A basic partitioned table"
+}

rust/tests/data/dat/v0.0.2/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000000.json

@@ -0,0 +1,6 @@
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"metaData":{"id":"df7a26ab-ccdf-4677-887d-a745216cae72","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["letter","date","data"],"configuration":{},"createdTime":1674611433276}}
+{"add":{"path":"letter=a/date=1970-01-01/data=hello/part-00000-31d90fef-d8a8-47da-8176-4ee8ff04c5ff.c000.snappy.parquet","partitionValues":{"letter":"a","date":"1970-01-01","data":"hello"},"size":490,"modificationTime":1674611433501,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":1},\"maxValues\":{\"number\":1},\"nullCount\":{\"number\":0}}"}}
+{"add":{"path":"letter=b/date=1970-01-01/data=world/part-00000-c3fade85-0652-48eb-90cf-a42d188ca9d3.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-01","data":"world"},"size":490,"modificationTime":1674611433521,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":2},\"maxValues\":{\"number\":2},\"nullCount\":{\"number\":0}}"}}
+{"add":{"path":"letter=b/date=1970-01-02/data=world/part-00000-4b3fc81d-2403-4c36-a793-73e5af9b0e90.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-02","data":"world"},"size":490,"modificationTime":1674611433533,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":3},\"maxValues\":{\"number\":3},\"nullCount\":{\"number\":0}}"}}
+{"commitInfo":{"timestamp":1674611433544,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"letter\",\"date\",\"data\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1470"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"258cd190-fac2-4ff0-92f3-dca562a7ade1"}}