Add Queue Size in Configuration (#18672)

open-metadata · Nov 18, 2024 · 66c253e · 66c253e
1 parent 6f1df37
commit 66c253e
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 6 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
@@ -279,14 +279,19 @@ public Integer reIndex(
           boolean recreateIndexes,
       @Option(
               names = {"--producer-threads"},
-              defaultValue = "10",
+              defaultValue = "5",
               description = "Number of threads to use for processing.")
           int producerThreads,
       @Option(
               names = {"--consumer-threads"},
-              defaultValue = "10",
+              defaultValue = "5",
               description = "Number of threads to use for processing.")
           int consumerThreads,
+      @Option(
+              names = {"--queue-size"},
+              defaultValue = "300",
+              description = "Queue Size to use internally for reindexing.")
+          int queueSize,
       @Option(
               names = {"--back-off"},
               defaultValue = "1000",
@@ -309,10 +314,17 @@ public Integer reIndex(
           int retries) {
     try {
       LOG.info(
-          "Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}",
+          "Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}, Producer threads: {}, Consumer threads: {}, Queue Size: {}, Back-off: {}, Max Back-off: {}, Max Requests: {}, Retries: {}",
           batchSize,
           payloadSize,
-          recreateIndexes);
+          recreateIndexes,
+          producerThreads,
+          consumerThreads,
+          queueSize,
+          backOff,
+          maxBackOff,
+          maxRequests,
+          retries);
       parseConfig();
       CollectionRegistry.initialize();
       ApplicationHandler.initialize(config);
@@ -327,6 +339,7 @@ public Integer reIndex(
           recreateIndexes,
           producerThreads,
           consumerThreads,
+          queueSize,
           backOff,
           maxBackOff,
           maxRequests,
@@ -344,6 +357,7 @@ private int executeSearchReindexApp(
       boolean recreateIndexes,
       int producerThreads,
       int consumerThreads,
+      int queueSize,
       int backOff,
       int maxBackOff,
       int maxRequests,
@@ -363,6 +377,7 @@ private int executeSearchReindexApp(
         .withRecreateIndex(recreateIndexes)
         .withProducerThreads(producerThreads)
         .withConsumerThreads(consumerThreads)
+        .withQueueSize(queueSize)
         .withInitialBackoff(backOff)
         .withMaxBackoff(maxBackOff)
         .withMaxConcurrentRequests(maxRequests)

diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
@@ -122,7 +122,7 @@
     },
     "queueSize": {
       "title": "Queue Size to use.",
-      "description": "Queue Size to user internally for reindexing.",
+      "description": "Queue Size to use internally for reindexing.",
       "type": "integer",
       "default": 100
     },

diff --git a/...data-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/...data-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
@@ -30,7 +30,7 @@
     },
     "queueSize": {
       "title": "Queue Size to use.",
-      "description": "Queue Size to user internally for reindexing.",
+      "description": "Queue Size to use internally for reindexing.",
       "type": "integer",
       "default": 100
     },