forked from delta-io/delta
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'upstream/master' into cm-removal-check-…
…invalid-col-names
- Loading branch information
Showing
52 changed files
with
2,725 additions
and
764 deletions.
There are no files selected for viewing
Binary file renamed
BIN
+232 Bytes
...b3a4-2b56a6bdee04-c000.snappy.parquet.crc → ...a657-3ba905ccae36-c000.snappy.parquet.crc
Binary file not shown.
Binary file added
BIN
+44 Bytes
...les/src/main/resources/golden/parquet-all-types/_delta_log/.00000000000000000000.json.crc
Binary file not shown.
4 changes: 4 additions & 0 deletions
4
...n-tables/src/main/resources/golden/parquet-all-types/_delta_log/00000000000000000000.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
{"commitInfo":{"timestamp":1708108025792,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"200","numOutputBytes":"28420"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"59b6eac4-ecda-4440-8268-a18cac973ba1"}} | ||
{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708108023726}} | ||
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} | ||
{"add":{"path":"part-00000-4b3cf091-231f-4d77-a657-3ba905ccae36-c000.snappy.parquet","partitionValues":{},"size":28420,"modificationTime":1708108025717,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"ByteType\":-128,\"ShortType\":1,\"IntegerType\":1,\"LongType\":2,\"FloatType\":0.234,\"DoubleType\":234234.23,\"decimal\":123.52,\"StringType\":\"1\",\"DateType\":\"1970-01-01\",\"TimestampType\":\"1970-01-01T06:30:23.523Z\",\"nested_struct\":{\"aa\":\"1\",\"ac\":{\"aca\":1}}},\"maxValues\":{\"ByteType\":127,\"ShortType\":199,\"IntegerType\":199,\"LongType\":200,\"FloatType\":46.566,\"DoubleType\":4.661261177E7,\"decimal\":24580.48,\"StringType\":\"99\",\"DateType\":\"1970-02-16\",\"TimestampType\":\"1970-02-23T22:48:01.077Z\",\"nested_struct\":{\"aa\":\"99\",\"ac\":{\"aca\":199}}},\"nullCount\":{\"ByteType\":3,\"ShortType\":4,\"IntegerType\":9,\"LongType\":8,\"FloatType\":8,\"DoubleType\":4,\"decimal\":3,\"BooleanType\":3,\"StringType\":4,\"BinaryType\":4,\"DateType\":4,\"TimestampType\":4,\"nested_struct\":{\"aa\":14,\"ac\":{\"aca\":22}},\"array_of_prims\":8,\"array_of_arrays\":25,\"array_of_structs\":0,\"map_of_prims\":8,\"map_of_rows\":0,\"map_of_arrays\":7}}"}} |
Binary file renamed
BIN
+27.8 KB
...126-b3a4-2b56a6bdee04-c000.snappy.parquet → ...d77-a657-3ba905ccae36-c000.snappy.parquet
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
87 changes: 87 additions & 0 deletions
87
examples/scala/src/main/scala/example/IcebergCompatV2.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
/* | ||
* Copyright (2021) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package example | ||
|
||
import java.io.{File, IOException} | ||
import java.net.ServerSocket | ||
|
||
import org.apache.commons.io.FileUtils | ||
|
||
import org.apache.spark.sql.SparkSession | ||
/** | ||
* This example relies on an external Hive metastore (HMS) instance to run. | ||
* | ||
* A standalone HMS can be created using the following docker command. | ||
* ************************************************************ | ||
* docker run -d -p 9083:9083 --env SERVICE_NAME=metastore \ | ||
* --name metastore-standalone apache/hive:4.0.0-beta-1 | ||
* ************************************************************ | ||
* The URL of this standalone HMS is thrift://localhost:9083 | ||
* | ||
* By default this hms will use `/opt/hive/data/warehouse` as warehouse path. | ||
* Please make sure this path exists or change it prior to running the example. | ||
*/ | ||
object IcebergCompatV2 { | ||
|
||
def main(args: Array[String]): Unit = { | ||
// Update this according to the metastore config | ||
val port = 9083 | ||
val warehousePath = "/opt/hive/data/warehouse/" | ||
|
||
if (!UniForm.hmsReady(port)) { | ||
print("HMS not available. Exit.") | ||
return | ||
} | ||
|
||
val testTableName = "uniform_table3" | ||
FileUtils.deleteDirectory(new File(s"${warehousePath}${testTableName}")) | ||
|
||
val deltaSpark = SparkSession | ||
.builder() | ||
.appName("UniForm-Delta") | ||
.master("local[*]") | ||
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") | ||
.config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") | ||
.config("hive.metastore.uris", s"thrift://localhost:$port") | ||
.config("spark.sql.catalogImplementation", "hive") | ||
.getOrCreate() | ||
|
||
deltaSpark.sql(s"DROP TABLE IF EXISTS ${testTableName}") | ||
deltaSpark.sql( | ||
s"""CREATE TABLE `${testTableName}` | ||
| (id INT, ts TIMESTAMP, array_data array<int>, map_data map<int, int>) | ||
| using DELTA""".stripMargin) | ||
deltaSpark.sql( | ||
s""" | ||
|INSERT INTO `$testTableName` (id, ts, array_data, map_data) | ||
| VALUES (123, '2024-01-01 00:00:00', array(2, 3, 4, 5), map(3, 6, 8, 7))""".stripMargin) | ||
deltaSpark.sql( | ||
s"""REORG TABLE `$testTableName` APPLY (UPGRADE UNIFORM | ||
| (ICEBERG_COMPAT_VERSION = 2))""".stripMargin) | ||
|
||
val icebergSpark = SparkSession.builder() | ||
.master("local[*]") | ||
.appName("UniForm-Iceberg") | ||
.config("spark.sql.extensions", | ||
"org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") | ||
.config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") | ||
.config("hive.metastore.uris", s"thrift://localhost:$port") | ||
.config("spark.sql.catalogImplementation", "hive") | ||
.getOrCreate() | ||
|
||
icebergSpark.sql(s"SELECT * FROM ${testTableName}").show() | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
45 changes: 45 additions & 0 deletions
45
kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
/* | ||
* Copyright (2024) The Delta Lake Project Authors. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.delta.kernel.internal; | ||
|
||
public final class DeltaErrors { | ||
private DeltaErrors() {} | ||
|
||
// TODO update to be user-facing exception with future exception framework | ||
// (see delta-io/delta#2231) & document in method docs as needed (Table::getSnapshotAtVersion) | ||
public static RuntimeException nonReconstructableStateException( | ||
String tablePath, long version) { | ||
String message = String.format( | ||
"%s: Unable to reconstruct state at version %s as the transaction log has been " + | ||
"truncated due to manual deletion or the log retention policy and checkpoint " + | ||
"retention policy.", | ||
tablePath, | ||
version); | ||
return new RuntimeException(message); | ||
} | ||
|
||
// TODO update to be user-facing exception with future exception framework | ||
// (see delta-io/delta#2231) & document in method docs as needed (Table::getSnapshotAtVersion) | ||
public static RuntimeException nonExistentVersionException( | ||
String tablePath, long versionToLoad, long latestVersion) { | ||
String message = String.format( | ||
"%s: Trying to load a non-existent version %s. The latest version available is %s", | ||
tablePath, | ||
versionToLoad, | ||
latestVersion); | ||
return new RuntimeException(message); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.