From 3c452b4c16e5ff6dfa52b3becddde02a40202f2c Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 17 Aug 2024 19:36:39 +0200 Subject: [PATCH] Introduce `JDBC2` version store type, deprecate `JDBC` version store type The current `JDBC` version store type uses a lot of columns in the `objs` table, which cause quite some overhead in storage. The new `JDBC2` version store type works like `JDBC`, but uses a single column for all object values and also serializes object-ids as binary values. --- CHANGELOG.md | 6 + bom/build.gradle.kts | 2 + gradle/projects.main.properties | 2 + helm/nessie/values.yaml | 7 +- servers/jax-rs-tests/build.gradle.kts | 2 +- .../jaxrs/tests/TestRestH2Persist.java | 2 +- servers/quarkus-common/build.gradle.kts | 1 + .../quarkus/config/QuarkusJdbcConfig.java | 14 +- .../datasource/DataSourceActivator.java | 3 +- .../storage/Jdbc2BackendBuilder.java | 130 ++++ .../providers/storage/JdbcBackendBuilder.java | 6 +- .../quarkus/config/VersionStoreConfig.java | 5 +- servers/quarkus-server/build.gradle.kts | 1 + .../server/configchecks/ConfigChecks.java | 6 +- .../src/main/resources/application.properties | 9 +- servers/quarkus-tests/build.gradle.kts | 1 + .../MariaDBTestResourceLifecycleManager.java | 2 +- .../MySQLTestResourceLifecycleManager.java | 2 +- .../PostgresTestResourceLifecycleManager.java | 2 +- .../profiles/QuarkusTestProfilePersistH2.java | 4 +- .../QuarkusTestProfilePersistMariaDB.java | 4 +- .../QuarkusTestProfilePersistMySQL.java | 4 +- .../QuarkusTestProfilePersistPostgres.java | 4 +- servers/services-bench/build.gradle.kts | 1 + servers/services/build.gradle.kts | 1 + .../services/impl/TestApiImplsPersistH2.java | 2 +- site/in-dev/configuration.md | 33 +- tools/doc-generator/doclet/build.gradle.kts | 1 + tools/doc-generator/site-gen/build.gradle.kts | 1 + tools/server-admin/build.gradle.kts | 2 + .../tools/admin/cli/ITExportImport.java | 3 +- .../cli/NessieServerAdminTestBackends.java | 12 +- .../jdbc/ITCockroachDBBackendFactory.java | 2 + .../storage/jdbc/ITCockroachDBPersist.java | 2 + .../jdbc/ITCockroachDBVersionStore.java | 2 + .../ITCockroachDBVersionStoreCaching.java | 2 + .../storage/jdbc/ITMySQLBackendFactory.java | 2 + .../storage/jdbc/ITMySQLPersist.java | 2 + .../storage/jdbc/ITMySQLVersionStore.java | 2 + .../storage/jdbc2-tests/build.gradle.kts | 47 ++ .../AbstractJdbc2BackendTestFactory.java | 60 ++ .../CockroachBackendTestFactory.java | 48 ++ .../ContainerBackendTestFactory.java | 111 +++ .../jdbc2tests/DataSourceProducer.java | 114 +++ .../jdbc2tests/H2BackendTestFactory.java | 47 ++ .../jdbc2tests/MariaDBBackendTestFactory.java | 49 ++ .../jdbc2tests/MySQLBackendTestFactory.java | 66 ++ .../PostgreSQLBackendTestFactory.java | 47 ++ ...d.storage.testextension.BackendTestFactory | 20 + .../jdbc2tests/Dockerfile-cockroach-version | 3 + .../jdbc2tests/Dockerfile-mariadb-version | 3 + .../jdbc2tests/Dockerfile-mysql-version | 3 + .../jdbc2tests/Dockerfile-postgres-version | 3 + versioned/storage/jdbc2/build.gradle.kts | 70 ++ .../jdbc2/ITCockroachDBBackendFactory.java | 27 + .../storage/jdbc2/ITCockroachDBPersist.java | 23 + .../jdbc2/ITCockroachDBVersionStore.java | 23 + .../ITCockroachDBVersionStoreCaching.java | 21 + .../jdbc2/ITMariaDBBackendFactory.java | 27 + .../storage/jdbc2/ITMariaDBPersist.java | 23 + .../storage/jdbc2/ITMariaDBVersionStore.java | 23 + .../storage/jdbc2/ITMySQLBackendFactory.java | 27 + .../storage/jdbc2/ITMySQLPersist.java | 23 + .../storage/jdbc2/ITMySQLVersionStore.java | 23 + .../jdbc2/ITPostgreSQLBackendFactory.java | 27 + .../storage/jdbc2/ITPostgreSQLPersist.java | 23 + .../jdbc2/ITPostgreSQLVersionStore.java | 23 + .../storage/jdbc2/AbstractJdbc2Persist.java | 667 ++++++++++++++++++ .../storage/jdbc2/DatabaseSpecific.java | 36 + .../storage/jdbc2/DatabaseSpecifics.java | 244 +++++++ .../versioned/storage/jdbc2/Jdbc2Backend.java | 370 ++++++++++ .../storage/jdbc2/Jdbc2BackendBaseConfig.java | 43 ++ .../storage/jdbc2/Jdbc2BackendConfig.java | 29 + .../storage/jdbc2/Jdbc2BackendFactory.java | 43 ++ .../storage/jdbc2/Jdbc2ColumnType.java | 31 + .../versioned/storage/jdbc2/Jdbc2Persist.java | 266 +++++++ .../storage/jdbc2/Jdbc2PersistFactory.java | 36 + .../versioned/storage/jdbc2/Jdbc2Serde.java | 63 ++ .../versioned/storage/jdbc2/SqlConstants.java | 202 ++++++ ...oned.storage.common.persist.BackendFactory | 16 + .../storage/jdbc2/TestH2BackendFactory.java | 27 + .../storage/jdbc2/TestH2CachingPersist.java | 21 + .../storage/jdbc2/TestH2Persist.java | 23 + .../storage/jdbc2/TestH2VersionStore.java | 23 + .../jdbc2/TestH2VersionStoreCaching.java | 21 + .../AbstractTestJdbc2BackendFactory.java | 258 +++++++ versioned/transfer/build.gradle.kts | 1 + 87 files changed, 3654 insertions(+), 59 deletions(-) create mode 100644 servers/quarkus-common/src/main/java/org/projectnessie/quarkus/providers/storage/Jdbc2BackendBuilder.java create mode 100644 versioned/storage/jdbc2-tests/build.gradle.kts create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/AbstractJdbc2BackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/CockroachBackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/ContainerBackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/DataSourceProducer.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/H2BackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/MariaDBBackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/MySQLBackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/java/org/projectnessie/versioned/storage/jdbc2tests/PostgreSQLBackendTestFactory.java create mode 100644 versioned/storage/jdbc2-tests/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.testextension.BackendTestFactory create mode 100644 versioned/storage/jdbc2-tests/src/main/resources/org/projectnessie/versioned/storage/jdbc2tests/Dockerfile-cockroach-version create mode 100644 versioned/storage/jdbc2-tests/src/main/resources/org/projectnessie/versioned/storage/jdbc2tests/Dockerfile-mariadb-version create mode 100644 versioned/storage/jdbc2-tests/src/main/resources/org/projectnessie/versioned/storage/jdbc2tests/Dockerfile-mysql-version create mode 100644 versioned/storage/jdbc2-tests/src/main/resources/org/projectnessie/versioned/storage/jdbc2tests/Dockerfile-postgres-version create mode 100644 versioned/storage/jdbc2/build.gradle.kts create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITCockroachDBBackendFactory.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITCockroachDBPersist.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITCockroachDBVersionStore.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITCockroachDBVersionStoreCaching.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITMariaDBBackendFactory.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITMariaDBPersist.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITMariaDBVersionStore.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITMySQLBackendFactory.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITMySQLPersist.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITMySQLVersionStore.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITPostgreSQLBackendFactory.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITPostgreSQLPersist.java create mode 100644 versioned/storage/jdbc2/src/intTest/java/org/projectnessie/versioned/storage/jdbc2/ITPostgreSQLVersionStore.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/AbstractJdbc2Persist.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/DatabaseSpecific.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/DatabaseSpecifics.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2Backend.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2BackendBaseConfig.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2BackendConfig.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2BackendFactory.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2ColumnType.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2Persist.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2PersistFactory.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/Jdbc2Serde.java create mode 100644 versioned/storage/jdbc2/src/main/java/org/projectnessie/versioned/storage/jdbc2/SqlConstants.java create mode 100644 versioned/storage/jdbc2/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.BackendFactory create mode 100644 versioned/storage/jdbc2/src/test/java/org/projectnessie/versioned/storage/jdbc2/TestH2BackendFactory.java create mode 100644 versioned/storage/jdbc2/src/test/java/org/projectnessie/versioned/storage/jdbc2/TestH2CachingPersist.java create mode 100644 versioned/storage/jdbc2/src/test/java/org/projectnessie/versioned/storage/jdbc2/TestH2Persist.java create mode 100644 versioned/storage/jdbc2/src/test/java/org/projectnessie/versioned/storage/jdbc2/TestH2VersionStore.java create mode 100644 versioned/storage/jdbc2/src/test/java/org/projectnessie/versioned/storage/jdbc2/TestH2VersionStoreCaching.java create mode 100644 versioned/storage/jdbc2/src/testFixtures/java/org/projectnessie/versioned/storage/jdbc2/AbstractTestJdbc2BackendFactory.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 25dd1a97da6..9261b1355b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,11 +33,17 @@ as necessary. Empty sections will not end in the release notes. - Nessie commit author(s) and "signed off by" can now be configured for both Nessie clients and Iceberg REST clients. More info on [projectnessie.org](https://projectnessie.org/guides/iceberg-rest/#customizing-nessie-commit-author-et-al). +- Introduce new `JDBC2` version store type, which is has the same functionality as the `JDBC` version + store type, but uses way less columns, which reduces storage overhead for example in PostgreSQL a lot. ### Changes ### Deprecations +- The current version store type `JDBC` is deprecated, please migrate to the new `JDBC2` version store + type. Please use the [Nessie Server Admin Tool](https://projectnessie.org/nessie-latest/export_import) + to migrate from the `JDBC` version store type to `JDBC2`. + ### Fixes ### Commits diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index e195fa2a65a..bd0a11ba85d 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -91,6 +91,8 @@ dependencies { api(project(":nessie-versioned-storage-inmemory-tests")) api(project(":nessie-versioned-storage-jdbc")) api(project(":nessie-versioned-storage-jdbc-tests")) + api(project(":nessie-versioned-storage-jdbc2")) + api(project(":nessie-versioned-storage-jdbc2-tests")) api(project(":nessie-versioned-storage-mongodb")) api(project(":nessie-versioned-storage-mongodb-tests")) api(project(":nessie-versioned-storage-rocksdb")) diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index c082937c3e8..3617bdd4b4e 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -78,6 +78,8 @@ nessie-versioned-storage-inmemory=versioned/storage/inmemory nessie-versioned-storage-inmemory-tests=versioned/storage/inmemory-tests nessie-versioned-storage-jdbc=versioned/storage/jdbc nessie-versioned-storage-jdbc-tests=versioned/storage/jdbc-tests +nessie-versioned-storage-jdbc2=versioned/storage/jdbc2 +nessie-versioned-storage-jdbc2-tests=versioned/storage/jdbc2-tests nessie-versioned-storage-mongodb=versioned/storage/mongodb nessie-versioned-storage-mongodb-tests=versioned/storage/mongodb-tests nessie-versioned-storage-rocksdb=versioned/storage/rocksdb diff --git a/helm/nessie/values.yaml b/helm/nessie/values.yaml index 3e066c45a27..705adca9060 100644 --- a/helm/nessie/values.yaml +++ b/helm/nessie/values.yaml @@ -28,11 +28,8 @@ imagePullSecrets: [] # `quarkus.log.category."io.smallrye.config".level: DEBUG` logLevel: INFO -# -- Which type of version store to use: IN_MEMORY, ROCKSDB, DYNAMODB, MONGODB, CASSANDRA, JDBC, BIGTABLE. -# (Legacy version store types are: INMEMORY, ROCKS, DYNAMO, MONGO, TRANSACTIONAL. If you are using -# one of these legacy version store types, migrate your existing repositories to the new version -# store types using the nessie-server-admin-tool's export/import functionality; support for these -# legacy version store types has been removed in Nessie 0.75.0.) +# -- Which type of version store to use: IN_MEMORY, ROCKSDB, DYNAMODB, MONGODB, CASSANDRA, JDBC2, BIGTABLE. +# Note: the version store type JDBC is deprecated, please use the Nessie Server Admin Tool to migrate to JDBC2. versionStoreType: IN_MEMORY # Cassandra settings. Only required when using CASSANDRA version store type; ignored otherwise. diff --git a/servers/jax-rs-tests/build.gradle.kts b/servers/jax-rs-tests/build.gradle.kts index abf359348ab..6d8b51fa088 100644 --- a/servers/jax-rs-tests/build.gradle.kts +++ b/servers/jax-rs-tests/build.gradle.kts @@ -41,7 +41,7 @@ dependencies { compileOnly("com.fasterxml.jackson.core:jackson-annotations") testImplementation(project(":nessie-versioned-storage-inmemory-tests")) - testImplementation(project(":nessie-versioned-storage-jdbc-tests")) + testImplementation(project(":nessie-versioned-storage-jdbc2-tests")) testRuntimeOnly(libs.agroal.pool) testImplementation(project(":nessie-jaxrs-testextension")) diff --git a/servers/jax-rs-tests/src/test/java/org/projectnessie/jaxrs/tests/TestRestH2Persist.java b/servers/jax-rs-tests/src/test/java/org/projectnessie/jaxrs/tests/TestRestH2Persist.java index 658f8e97838..e661d8418c5 100644 --- a/servers/jax-rs-tests/src/test/java/org/projectnessie/jaxrs/tests/TestRestH2Persist.java +++ b/servers/jax-rs-tests/src/test/java/org/projectnessie/jaxrs/tests/TestRestH2Persist.java @@ -15,7 +15,7 @@ */ package org.projectnessie.jaxrs.tests; -import org.projectnessie.versioned.storage.jdbctests.H2BackendTestFactory; +import org.projectnessie.versioned.storage.jdbc2tests.H2BackendTestFactory; import org.projectnessie.versioned.storage.testextension.NessieBackend; @NessieBackend(H2BackendTestFactory.class) diff --git a/servers/quarkus-common/build.gradle.kts b/servers/quarkus-common/build.gradle.kts index 3ed5278b61f..d552b0163bc 100644 --- a/servers/quarkus-common/build.gradle.kts +++ b/servers/quarkus-common/build.gradle.kts @@ -47,6 +47,7 @@ dependencies { implementation(project(":nessie-versioned-storage-dynamodb")) implementation(project(":nessie-versioned-storage-inmemory")) implementation(project(":nessie-versioned-storage-jdbc")) + implementation(project(":nessie-versioned-storage-jdbc2")) implementation(project(":nessie-versioned-storage-mongodb")) implementation(project(":nessie-versioned-storage-rocksdb")) implementation(project(":nessie-versioned-storage-store")) diff --git a/servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/QuarkusJdbcConfig.java b/servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/QuarkusJdbcConfig.java index e55942c435b..93e14322453 100644 --- a/servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/QuarkusJdbcConfig.java +++ b/servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/QuarkusJdbcConfig.java @@ -19,10 +19,10 @@ import io.smallrye.config.ConfigMapping; import io.smallrye.config.WithName; import java.util.Optional; -import org.projectnessie.versioned.storage.jdbc.JdbcBackendBaseConfig; +import org.projectnessie.versioned.storage.jdbc2.Jdbc2BackendBaseConfig; /** - * Setting {@code nessie.version.store.type=JDBC} enables transactional/RDBMS as the version store + * Setting {@code nessie.version.store.type=JDBC2} enables transactional/RDBMS as the version store * used by the Nessie server. * *

Configuration of the datastore will be done by Quarkus and depends on many factors, such as @@ -34,7 +34,7 @@ *

For example, to configure a PostgresQL connection, the following configuration should be used: * *