From 66c253ef105b093f73bf56a7a357a58ba4a9aa62 Mon Sep 17 00:00:00 2001
From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com>
Date: Mon, 18 Nov 2024 17:15:08 +0530
Subject: [PATCH] Add Queue Size in Configuration (#18672)

---
 .../service/util/OpenMetadataOperations.java  | 23 +++++++++++++++----
 .../json/schema/system/eventPublisherJob.json |  2 +-
 .../SearchIndexingApplication.json            |  2 +-
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
index 9fe3eaaf4db7..47d521bfe04d 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
@@ -279,14 +279,19 @@ public Integer reIndex(
           boolean recreateIndexes,
       @Option(
               names = {"--producer-threads"},
-              defaultValue = "10",
+              defaultValue = "5",
               description = "Number of threads to use for processing.")
           int producerThreads,
       @Option(
               names = {"--consumer-threads"},
-              defaultValue = "10",
+              defaultValue = "5",
               description = "Number of threads to use for processing.")
           int consumerThreads,
+      @Option(
+              names = {"--queue-size"},
+              defaultValue = "300",
+              description = "Queue Size to use internally for reindexing.")
+          int queueSize,
       @Option(
               names = {"--back-off"},
               defaultValue = "1000",
@@ -309,10 +314,17 @@ public Integer reIndex(
           int retries) {
     try {
       LOG.info(
-          "Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}",
+          "Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}, Producer threads: {}, Consumer threads: {}, Queue Size: {}, Back-off: {}, Max Back-off: {}, Max Requests: {}, Retries: {}",
           batchSize,
           payloadSize,
-          recreateIndexes);
+          recreateIndexes,
+          producerThreads,
+          consumerThreads,
+          queueSize,
+          backOff,
+          maxBackOff,
+          maxRequests,
+          retries);
       parseConfig();
       CollectionRegistry.initialize();
       ApplicationHandler.initialize(config);
@@ -327,6 +339,7 @@ public Integer reIndex(
           recreateIndexes,
           producerThreads,
           consumerThreads,
+          queueSize,
           backOff,
           maxBackOff,
           maxRequests,
@@ -344,6 +357,7 @@ private int executeSearchReindexApp(
       boolean recreateIndexes,
       int producerThreads,
       int consumerThreads,
+      int queueSize,
       int backOff,
       int maxBackOff,
       int maxRequests,
@@ -363,6 +377,7 @@ private int executeSearchReindexApp(
         .withRecreateIndex(recreateIndexes)
         .withProducerThreads(producerThreads)
         .withConsumerThreads(consumerThreads)
+        .withQueueSize(queueSize)
         .withInitialBackoff(backOff)
         .withMaxBackoff(maxBackOff)
         .withMaxConcurrentRequests(maxRequests)
diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
index 069ddbb62431..3305971858dd 100644
--- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
+++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json
@@ -122,7 +122,7 @@
     },
     "queueSize": {
       "title": "Queue Size to use.",
-      "description": "Queue Size to user internally for reindexing.",
+      "description": "Queue Size to use internally for reindexing.",
       "type": "integer",
       "default": 100
     },
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
index 9c3989ce0fba..4957d1979fa9 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
@@ -30,7 +30,7 @@
     },
     "queueSize": {
       "title": "Queue Size to use.",
-      "description": "Queue Size to user internally for reindexing.",
+      "description": "Queue Size to use internally for reindexing.",
       "type": "integer",
       "default": 100
     },