Issue #LR-773 fix: Added the OCI support for content_index_batch.json

Sunbird-Ed · Jun 17, 2024 · fb708e6 · fb708e6
1 parent 901a544
commit fb708e6
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 14 deletions.
diff --git a/etl-jobs/ingestion-spec/content_index_batch.json b/etl-jobs/ingestion-spec/content_index_batch.json
@@ -785,13 +785,18 @@
     "ioConfig":{
       "type":"index",
       "firehose" : {
-        "type" : "static-azure-blobstore",
-        "blobs": [
-          {
-            "container": "telemetry-data-store",
-            "path": "/druid-content-snapshot/snapshot.txt"
-          }
-        ],
+{% if cloud_storage.provider == "azure" %}
+    "type": "static-azure-blobstore",
+    "blobs": [
+      {
+        "container": "{{cloud_storage.container}}",
+        "path": "/{{cloud_storage.object_key}}"
+      }
+    ],
+{% else %}
+    "type": "static-s3",
+    "uris": [ "s3://{{cloud_storage.container}}/{{cloud_storage.object_key}}"],
+{% endif %}
         "fetchTimeout": 300000
       }
     },

diff --git a/etl-jobs/ingestion-spec/vdn_content_index_batch.json b/etl-jobs/ingestion-spec/vdn_content_index_batch.json
@@ -847,13 +847,18 @@
       "ioConfig":{
         "type":"index",
         "firehose" : {
-          "type" : "static-azure-blobstore",
-          "blobs": [
-            {
-              "container": "telemetry-data-store",
-              "path": "/druid-content-snapshot/vdn-snapshot.txt"
-            }
-          ],
+{% if cloud_storage.provider == "azure" %}
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{cloud_storage.container}}",
+            "path": "/{{cloud_storage.object_key}}"
+          }
+        ],
+{% else %}
+        "type": "static-s3",
+        "uris": [ "s3://{{cloud_storage.container}}/{{cloud_storage.object_key}}"],
+{% endif %}
           "fetchTimeout": 3000000
         }
       },