Introduce MONGODB2 version store type, deprecate MONGODB version …

…store type The current `MONGODB` version store type uses a lot of attributes in the `objs` collection, which cause quite some overhead in storage. The new `MONGODB2` version store type works like `MONGODB`, but uses a single attributes for all object values.
projectnessie · Aug 17, 2024 · f160180 · f160180
1 parent 3c452b4
commit f160180
Show file tree

Hide file tree

Showing 42 changed files with 1,751 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,8 @@ as necessary. Empty sections will not end in the release notes.
   [projectnessie.org](https://projectnessie.org/guides/iceberg-rest/#customizing-nessie-commit-author-et-al). 
 - Introduce new `JDBC2` version store type, which is has the same functionality as the `JDBC` version
   store type, but uses way less columns, which reduces storage overhead for example in PostgreSQL a lot.
+- Introduce new `MONGODB2` version store type, which is has the same functionality as the `MONGODB` version
+  store type, but uses way less attributes, which reduces storage overhead.
 
 ### Changes
 
@@ -43,6 +45,9 @@ as necessary. Empty sections will not end in the release notes.
 - The current version store type `JDBC` is deprecated, please migrate to the new `JDBC2` version store
   type. Please use the [Nessie Server Admin Tool](https://projectnessie.org/nessie-latest/export_import)
   to migrate from the `JDBC` version store type to `JDBC2`.
+- The current version store type `MONGODB` is deprecated, please migrate to the new `MONGODB2` version store
+  type. Please use the [Nessie Server Admin Tool](https://projectnessie.org/nessie-latest/export_import)
+  to migrate from the `MONGODB` version store type to `MONGODB2`.
 
 ### Fixes
 

diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
@@ -95,6 +95,8 @@ dependencies {
     api(project(":nessie-versioned-storage-jdbc2-tests"))
     api(project(":nessie-versioned-storage-mongodb"))
     api(project(":nessie-versioned-storage-mongodb-tests"))
+    api(project(":nessie-versioned-storage-mongodb2"))
+    api(project(":nessie-versioned-storage-mongodb2-tests"))
     api(project(":nessie-versioned-storage-rocksdb"))
     api(project(":nessie-versioned-storage-rocksdb-tests"))
     api(project(":nessie-versioned-storage-store"))

diff --git a/build-tools-integration-tests/build.gradle b/build-tools-integration-tests/build.gradle
@@ -30,7 +30,7 @@ dependencies {
   implementation("org.projectnessie.nessie:nessie-versioned-storage-cache")
   implementation("org.projectnessie.nessie:nessie-versioned-storage-common-serialize")
   implementation("org.projectnessie.nessie:nessie-versioned-storage-inmemory-tests")
-  implementation("org.projectnessie.nessie:nessie-versioned-storage-mongodb")
+  implementation("org.projectnessie.nessie:nessie-versioned-storage-mongodb2")
   implementation("org.projectnessie.nessie:nessie-versioned-storage-rocksdb-tests")
   implementation("org.projectnessie.nessie:nessie-versioned-storage-store")
   implementation("org.projectnessie.nessie:nessie-versioned-storage-testextension")

diff --git a/build-tools-integration-tests/pom.xml b/build-tools-integration-tests/pom.xml
@@ -68,7 +68,7 @@
     </dependency>
     <dependency>
       <groupId>org.projectnessie.nessie</groupId>
-      <artifactId>nessie-versioned-storage-mongodb</artifactId>
+      <artifactId>nessie-versioned-storage-mongodb2</artifactId>
     </dependency>
     <dependency>
       <groupId>org.projectnessie.nessie</groupId>

diff --git a/...integration-tests/src/test/java/org/projectnessie/buildtools/TestClassesAvailability.java b/...integration-tests/src/test/java/org/projectnessie/buildtools/TestClassesAvailability.java
@@ -34,8 +34,8 @@ public class TestClassesAvailability {
     "org.projectnessie.versioned.storage.inmemorytests.InmemoryBackendTestFactory",
     // from nessie-versioned-storage-rocksdb-tests
     "org.projectnessie.versioned.storage.rocksdbtests.RocksDBBackendTestFactory",
-    // from nessie-versioned-storage-mongodb
-    "org.projectnessie.versioned.storage.mongodb.MongoDBBackendFactory",
+    // from nessie-versioned-storage-mongodb2
+    "org.projectnessie.versioned.storage.mongodb2.MongoDB2BackendFactory",
     // from nessie-versioned-storage-cache
     "org.projectnessie.versioned.storage.cache.PersistCaches",
     // from nessie-versioned-storage-testextension

diff --git a/...tools-integration-tests/src/test/java/org/projectnessie/buildtools/TestPublishedPoms.java b/...tools-integration-tests/src/test/java/org/projectnessie/buildtools/TestPublishedPoms.java
@@ -100,7 +100,7 @@ static void setup() {
     "nessie-versioned-storage-common-serialize",
     "nessie-versioned-storage-common-tests",
     "nessie-versioned-storage-inmemory",
-    "nessie-versioned-storage-mongodb",
+    "nessie-versioned-storage-mongodb2",
     "nessie-versioned-storage-rocksdb",
     "nessie-versioned-storage-store",
     "nessie-versioned-storage-testextension",

diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties
@@ -82,6 +82,8 @@ nessie-versioned-storage-jdbc2=versioned/storage/jdbc2
 nessie-versioned-storage-jdbc2-tests=versioned/storage/jdbc2-tests
 nessie-versioned-storage-mongodb=versioned/storage/mongodb
 nessie-versioned-storage-mongodb-tests=versioned/storage/mongodb-tests
+nessie-versioned-storage-mongodb2=versioned/storage/mongodb2
+nessie-versioned-storage-mongodb2-tests=versioned/storage/mongodb2-tests
 nessie-versioned-storage-rocksdb=versioned/storage/rocksdb
 nessie-versioned-storage-rocksdb-tests=versioned/storage/rocksdb-tests
 nessie-versioned-storage-store=versioned/storage/store

diff --git a/helm/nessie/values.yaml b/helm/nessie/values.yaml
@@ -28,8 +28,9 @@ imagePullSecrets: []
 # `quarkus.log.category."io.smallrye.config".level: DEBUG`
 logLevel: INFO
 
-# -- Which type of version store to use: IN_MEMORY, ROCKSDB, DYNAMODB, MONGODB, CASSANDRA, JDBC2, BIGTABLE.
+# -- Which type of version store to use: IN_MEMORY, ROCKSDB, DYNAMODB, MONGODB2, CASSANDRA, JDBC2, BIGTABLE.
 # Note: the version store type JDBC is deprecated, please use the Nessie Server Admin Tool to migrate to JDBC2.
+# Note: the version store type MONGODB is deprecated, please use the Nessie Server Admin Tool to migrate to MONGODB2.
 versionStoreType: IN_MEMORY
 
 # Cassandra settings. Only required when using CASSANDRA version store type; ignored otherwise.

diff --git a/servers/quarkus-common/build.gradle.kts b/servers/quarkus-common/build.gradle.kts
@@ -49,6 +49,7 @@ dependencies {
   implementation(project(":nessie-versioned-storage-jdbc"))
   implementation(project(":nessie-versioned-storage-jdbc2"))
   implementation(project(":nessie-versioned-storage-mongodb"))
+  implementation(project(":nessie-versioned-storage-mongodb2"))
   implementation(project(":nessie-versioned-storage-rocksdb"))
   implementation(project(":nessie-versioned-storage-store"))
 

diff --git a/...mon/src/main/java/org/projectnessie/quarkus/providers/storage/MongoDB2BackendBuilder.java b/...mon/src/main/java/org/projectnessie/quarkus/providers/storage/MongoDB2BackendBuilder.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2022 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.projectnessie.quarkus.providers.storage;
+
+import static org.projectnessie.quarkus.config.VersionStoreConfig.VersionStoreType.MONGODB2;
+
+import com.mongodb.client.MongoClient;
+import io.quarkus.arc.Arc;
+import io.quarkus.mongodb.runtime.MongoClientBeanUtil;
+import io.quarkus.mongodb.runtime.MongoClients;
+import jakarta.enterprise.context.Dependent;
+import jakarta.inject.Inject;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.projectnessie.quarkus.providers.versionstore.StoreType;
+import org.projectnessie.versioned.storage.common.persist.Backend;
+import org.projectnessie.versioned.storage.mongodb2.MongoDB2BackendConfig;
+import org.projectnessie.versioned.storage.mongodb2.MongoDB2BackendFactory;
+
+@StoreType(MONGODB2)
+@Dependent
+public class MongoDB2BackendBuilder implements BackendBuilder {
+
+  @Inject
+  @ConfigProperty(name = "quarkus.mongodb.database")
+  String databaseName;
+
+  @Override
+  public Backend buildBackend() {
+    MongoClients mongoClients = Arc.container().instance(MongoClients.class).get();
+    MongoClient client =
+        mongoClients.createMongoClient(MongoClientBeanUtil.DEFAULT_MONGOCLIENT_NAME);
+
+    MongoDB2BackendFactory factory = new MongoDB2BackendFactory();
+    MongoDB2BackendConfig c =
+        MongoDB2BackendConfig.builder().databaseName(databaseName).client(client).build();
+    return factory.buildBackend(c);
+  }
+}
diff --git a/...ers/quarkus-config/src/main/java/org/projectnessie/quarkus/config/VersionStoreConfig.java b/...ers/quarkus-config/src/main/java/org/projectnessie/quarkus/config/VersionStoreConfig.java
@@ -29,7 +29,10 @@ enum VersionStoreType {
     IN_MEMORY,
     ROCKSDB,
     DYNAMODB,
+    /** MongoDB variant using many distinct attributes. */
     MONGODB,
+    /** MongoDB variant using few attributes, saves storage overhead. */
+    MONGODB2,
     CASSANDRA,
     /** JDBC variant using many distinct columns. */
     JDBC,

diff --git a/servers/quarkus-server/src/main/resources/application.properties b/servers/quarkus-server/src/main/resources/application.properties
@@ -146,8 +146,9 @@ nessie.server.send-stacktrace-to-client=false
 # nessie.server.authorization.rules.allow_listing_reflog=\
 #   op=='VIEW_REFLOG' && role=='admin_user'
 
-### which type of version store to use: IN_MEMORY, ROCKSDB, DYNAMODB, MONGODB, CASSANDRA, JDBC2, BIGTABLE.
+### which type of version store to use: IN_MEMORY, ROCKSDB, DYNAMODB, MONGODB2, CASSANDRA, JDBC2, BIGTABLE.
 # Note: the version store type JDBC is deprecated, please use the Nessie Server Admin Tool to migrate to JDBC2.
+# Note: the version store type MONGODB is deprecated, please use the Nessie Server Admin Tool to migrate to MONGODB2.
 nessie.version.store.type=IN_MEMORY
 
 # Object cache size as a value relative to the JVM's max heap size. The `cache-capacity-fraction-adjust-mb`

diff --git a/servers/quarkus-tests/build.gradle.kts b/servers/quarkus-tests/build.gradle.kts
@@ -35,6 +35,7 @@ dependencies {
   implementation(project(":nessie-versioned-storage-jdbc-tests"))
   implementation(project(":nessie-versioned-storage-jdbc2-tests"))
   implementation(project(":nessie-versioned-storage-mongodb-tests"))
+  implementation(project(":nessie-versioned-storage-mongodb2-tests"))
   implementation(project(":nessie-versioned-storage-rocksdb-tests"))
   implementation(project(":nessie-versioned-storage-testextension"))
   implementation(project(":nessie-container-spec-helper"))

diff --git a/...main/java/org/projectnessie/quarkus/tests/profiles/MongoTestResourceLifecycleManager.java b/...main/java/org/projectnessie/quarkus/tests/profiles/MongoTestResourceLifecycleManager.java
@@ -19,12 +19,12 @@
 import io.quarkus.test.common.QuarkusTestResourceLifecycleManager;
 import java.util.Map;
 import java.util.Optional;
-import org.projectnessie.versioned.storage.mongodbtests.MongoDBBackendTestFactory;
+import org.projectnessie.versioned.storage.mongodbtests2.MongoDB2BackendTestFactory;
 
 public class MongoTestResourceLifecycleManager
     implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware {
 
-  private MongoDBBackendTestFactory mongo;
+  private MongoDB2BackendTestFactory mongo;
 
   private Optional<String> containerNetworkId;
 
@@ -35,7 +35,7 @@ public void setIntegrationTestContext(DevServicesContext context) {
 
   @Override
   public Map<String, String> start() {
-    mongo = new MongoDBBackendTestFactory();
+    mongo = new MongoDB2BackendTestFactory();
 
     try {
       mongo.start(containerNetworkId);

diff --git a/...rc/main/java/org/projectnessie/quarkus/tests/profiles/QuarkusTestProfilePersistMongo.java b/...rc/main/java/org/projectnessie/quarkus/tests/profiles/QuarkusTestProfilePersistMongo.java
@@ -15,7 +15,7 @@
  */
 package org.projectnessie.quarkus.tests.profiles;
 
-import static org.projectnessie.quarkus.config.VersionStoreConfig.VersionStoreType.MONGODB;
+import static org.projectnessie.quarkus.config.VersionStoreConfig.VersionStoreType.MONGODB2;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -28,7 +28,7 @@ public class QuarkusTestProfilePersistMongo extends BaseConfigProfile {
   public Map<String, String> getConfigOverrides() {
     return ImmutableMap.<String, String>builder()
         .putAll(super.getConfigOverrides())
-        .put("nessie.version.store.type", MONGODB.name())
+        .put("nessie.version.store.type", MONGODB2.name())
         .put("quarkus.mongodb.write-concern.journal", "false")
         .build();
   }

diff --git a/servers/services-bench/build.gradle.kts b/servers/services-bench/build.gradle.kts
@@ -47,6 +47,7 @@ dependencies {
   jmhRuntimeOnly(project(":nessie-versioned-storage-cassandra"))
   jmhRuntimeOnly(project(":nessie-versioned-storage-rocksdb"))
   jmhRuntimeOnly(project(":nessie-versioned-storage-mongodb"))
+  jmhRuntimeOnly(project(":nessie-versioned-storage-mongodb2"))
   jmhRuntimeOnly(project(":nessie-versioned-storage-dynamodb"))
   jmhRuntimeOnly(project(":nessie-versioned-storage-jdbc"))
   jmhRuntimeOnly(project(":nessie-versioned-storage-jdbc2"))

diff --git a/servers/services/build.gradle.kts b/servers/services/build.gradle.kts
@@ -71,6 +71,7 @@ dependencies {
   intTestImplementation(project(":nessie-versioned-storage-cassandra-tests"))
   intTestImplementation(project(":nessie-versioned-storage-rocksdb-tests"))
   intTestImplementation(project(":nessie-versioned-storage-mongodb-tests"))
+  intTestImplementation(project(":nessie-versioned-storage-mongodb2-tests"))
   intTestImplementation(project(":nessie-versioned-storage-dynamodb-tests"))
   intTestRuntimeOnly(platform(libs.testcontainers.bom))
   intTestRuntimeOnly("org.testcontainers:testcontainers")

diff --git a/...s/services/src/intTest/java/org/projectnessie/services/impl/ITApiImplsPersistMongoDB.java b/...s/services/src/intTest/java/org/projectnessie/services/impl/ITApiImplsPersistMongoDB.java
@@ -18,11 +18,11 @@
 import org.junit.jupiter.api.condition.DisabledOnOs;
 import org.junit.jupiter.api.condition.OS;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.projectnessie.versioned.storage.mongodbtests.MongoDBBackendTestFactory;
+import org.projectnessie.versioned.storage.mongodbtests2.MongoDB2BackendTestFactory;
 import org.projectnessie.versioned.storage.testextension.NessieBackend;
 import org.projectnessie.versioned.storage.testextension.PersistExtension;
 
 @ExtendWith(PersistExtension.class)
-@NessieBackend(MongoDBBackendTestFactory.class)
+@NessieBackend(MongoDB2BackendTestFactory.class)
 @DisabledOnOs(OS.WINDOWS) // testcontainers does not support Windows
 public class ITApiImplsPersistMongoDB extends BaseTestApis {}
diff --git a/site/in-dev/configuration.md b/site/in-dev/configuration.md
@@ -194,7 +194,7 @@ Related Quarkus settings:
 | "in memory"      | only for development and local testing           | `IN_MEMORY`                                         | Do not use for any serious use case.                                                                                                                                                                                            |
 | RocksDB          | production, single node only                     | `ROCKSDB`                                           |                                                                                                                                                                                                                                 |
 | Google BigTable  | production                                       | `BIGTABLE`                                          |                                                                                                                                                                                                                                 |
-| MongoDB          | production                                       | `MONGODB`                                           |                                                                                                                                                                                                                                 |
+| MongoDB          | production                                       | `MONGODB2` & `MONGODB` (deprecated)                 |                                                                                                                                                                                                                                 |
 | Amazon DynamoDB  | beta, only tested against the simulator          | `DYNAMODB`                                          | Not recommended for use with Nessie Catalog (Iceberg REST) due to its restrictive row-size limit.                                                                                                                               |
 | PostgreSQL       | production                                       | `JDBC2` & `JDBC` (deprecated)                       |                                                                                                                                                                                                                                 |
 | H2               | only for development and local testing           | `JDBC2` & `JDBC` (deprecated)                       | Do not use for any serious use case.                                                                                                                                                                                            |
@@ -204,6 +204,11 @@ Related Quarkus settings:
 | Apache Cassandra | experimental, known issues                       | `CASSANDRA`                                         | Known to raise user-facing errors due to Cassandra's concept of letting the driver timeout too early, or database timeouts.                                                                                                     |
 | ScyllaDB         | experimental, known issues                       | `CASSANDRA`                                         | Known to raise user-facing errors due to Cassandra's concept of letting the driver timeout too early, or database timeouts. Known to be slow in container based testing. Unclear how good Scylla's LWT implementation performs. |
 
+!!! warn
+    Prefer the `MONGODB2` version store type over the `MONGODB` version store type, because it has way less storage overhead.
+    The `MONGODB` version store type is _deprecated for removal_, please use the
+    [Nessie Server Admin Tool](export_import.md) to migrate from the `MONGODB` version store type to `MONGODB2`.
+
 !!! warn
     Prefer the `JDBC2` version store type over the `JDBC` version store type, because it has way less storage overhead.
     The `JDBC` version store type is _deprecated for removal_, please use the
@@ -273,7 +278,7 @@ Related Quarkus settings:
 
 #### MongoDB Version Store Settings
 
-When setting `nessie.version.store.type=MONGODB` which enables MongoDB as the version store used by the Nessie server, the following configurations are applicable in combination with `nessie.version.store.type`.
+When setting `nessie.version.store.type=MONGODB2` which enables MongoDB as the version store used by the Nessie server, the following configurations are applicable in combination with `nessie.version.store.type`.
 
 Related Quarkus settings:
 

diff --git a/tools/doc-generator/doclet/build.gradle.kts b/tools/doc-generator/doclet/build.gradle.kts
@@ -35,7 +35,7 @@ val genProjectPaths =
     ":nessie-versioned-storage-inmemory",
     ":nessie-versioned-storage-jdbc",
     ":nessie-versioned-storage-jdbc2",
-    ":nessie-versioned-storage-mongodb",
+    ":nessie-versioned-storage-mongodb2",
     ":nessie-versioned-storage-rocksdb"
   )
 

diff --git a/tools/doc-generator/site-gen/build.gradle.kts b/tools/doc-generator/site-gen/build.gradle.kts
@@ -44,6 +44,7 @@ val genProjectPaths = listOf(
   ":nessie-versioned-storage-jdbc",
   ":nessie-versioned-storage-jdbc2",
   ":nessie-versioned-storage-mongodb",
+  ":nessie-versioned-storage-mongodb2",
   ":nessie-versioned-storage-rocksdb",
   ":nessie-catalog-files-impl",
   ":nessie-catalog-service-common"

diff --git a/tools/server-admin/build.gradle.kts b/tools/server-admin/build.gradle.kts
@@ -62,6 +62,7 @@ dependencies {
   implementation(project(":nessie-versioned-storage-jdbc"))
   implementation(project(":nessie-versioned-storage-jdbc2"))
   implementation(project(":nessie-versioned-storage-mongodb"))
+  implementation(project(":nessie-versioned-storage-mongodb2"))
   implementation(project(":nessie-versioned-storage-rocksdb"))
 
   implementation(project(":nessie-catalog-service-common"))
@@ -92,6 +93,7 @@ dependencies {
   testFixturesApi(project(":nessie-quarkus-tests"))
   testFixturesApi(project(":nessie-versioned-tests"))
   intTestImplementation(project(":nessie-versioned-storage-mongodb-tests"))
+  intTestImplementation(project(":nessie-versioned-storage-mongodb2-tests"))
   intTestImplementation(project(":nessie-versioned-storage-jdbc-tests"))
   intTestImplementation(project(":nessie-versioned-storage-jdbc2-tests"))
   intTestImplementation(project(":nessie-versioned-storage-cassandra-tests"))

diff --git a/tools/server-admin/src/intTest/java/org/projectnessie/tools/admin/cli/ITExportImport.java b/tools/server-admin/src/intTest/java/org/projectnessie/tools/admin/cli/ITExportImport.java
@@ -77,7 +77,7 @@
 public class ITExportImport {
 
   private static final String VERSION_STORES =
-      "(ROCKSDB|DYNAMODB|MONGODB|CASSANDRA|JDBC2|BIGTABLE)";
+      "(ROCKSDB|DYNAMODB|MONGODB2|CASSANDRA|JDBC2|BIGTABLE)";
 
   @InjectSoftAssertions private SoftAssertions soft;
 

diff --git a/...min/src/intTest/java/org/projectnessie/tools/admin/cli/NessieServerAdminTestBackends.java b/...min/src/intTest/java/org/projectnessie/tools/admin/cli/NessieServerAdminTestBackends.java
@@ -23,19 +23,19 @@
 import org.projectnessie.versioned.storage.jdbc2tests.MariaDBBackendTestFactory;
 import org.projectnessie.versioned.storage.jdbc2tests.MySQLBackendTestFactory;
 import org.projectnessie.versioned.storage.jdbc2tests.PostgreSQLBackendTestFactory;
-import org.projectnessie.versioned.storage.mongodbtests.MongoDBBackendTestFactory;
+import org.projectnessie.versioned.storage.mongodbtests2.MongoDB2BackendTestFactory;
 import org.projectnessie.versioned.storage.testextension.BackendTestFactory;
 
 public enum NessieServerAdminTestBackends {
   mongo {
     @Override
     BackendTestFactory backendFactory() {
-      return new MongoDBBackendTestFactory();
+      return new MongoDB2BackendTestFactory();
     }
 
     @Override
     Map<String, String> quarkusConfig() {
-      return Map.of("nessie.version.store.type", VersionStoreType.MONGODB.name());
+      return Map.of("nessie.version.store.type", VersionStoreType.MONGODB2.name());
     }
   },