Merge pull request #570 from AmiableAnil/release-5.1.1

Issue #LR-773 fix: Added the OCI support
Sunbird-Ed · Jun 25, 2024 · 6cb12ea · 6cb12ea
2 parents 2bd3c20 + fb708e6
commit 6cb12ea
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 17 deletions.
diff --git a/etl-jobs/ingestion-spec/content_index_batch.json b/etl-jobs/ingestion-spec/content_index_batch.json
@@ -785,13 +785,18 @@
     "ioConfig":{
       "type":"index",
       "firehose" : {
-        "type" : "static-azure-blobstore",
-        "blobs": [
-          {
-            "container": "telemetry-data-store",
-            "path": "/druid-content-snapshot/snapshot.txt"
-          }
-        ],
+{% if cloud_storage.provider == "azure" %}
+    "type": "static-azure-blobstore",
+    "blobs": [
+      {
+        "container": "{{cloud_storage.container}}",
+        "path": "/{{cloud_storage.object_key}}"
+      }
+    ],
+{% else %}
+    "type": "static-s3",
+    "uris": [ "s3://{{cloud_storage.container}}/{{cloud_storage.object_key}}"],
+{% endif %}
         "fetchTimeout": 300000
       }
     },

diff --git a/etl-jobs/ingestion-spec/vdn_content_index_batch.json b/etl-jobs/ingestion-spec/vdn_content_index_batch.json
@@ -847,13 +847,18 @@
       "ioConfig":{
         "type":"index",
         "firehose" : {
-          "type" : "static-azure-blobstore",
-          "blobs": [
-            {
-              "container": "telemetry-data-store",
-              "path": "/druid-content-snapshot/vdn-snapshot.txt"
-            }
-          ],
+{% if cloud_storage.provider == "azure" %}
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{cloud_storage.container}}",
+            "path": "/{{cloud_storage.object_key}}"
+          }
+        ],
+{% else %}
+        "type": "static-s3",
+        "uris": [ "s3://{{cloud_storage.container}}/{{cloud_storage.object_key}}"],
+{% endif %}
           "fetchTimeout": 3000000
         }
       },

diff --git a/etl-jobs/pom.xml b/etl-jobs/pom.xml
@@ -24,7 +24,7 @@
             <exclusions>
                 <exclusion>
                     <artifactId>jets3t</artifactId>
-                    <groupId>net.java.dev.jets3t</groupId>
+                    <groupId>org.jets3t</groupId>
                 </exclusion>
                 <exclusion>
                     <groupId>org.apache.xbean</groupId>
@@ -153,7 +153,7 @@
         <dependency>
             <groupId>org.sunbird</groupId>
             <artifactId>cloud-store-sdk_${scala.maj.version}</artifactId>
-            <version>1.3.0</version>
+            <version>1.4.6</version>
             <exclusions>
                 <exclusion>
                     <groupId>com.microsoft.azure</groupId>

diff --git a/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala b/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala
@@ -46,7 +46,7 @@ object ESCloudUploader {
             .saveAsTextFile(outputFilePath)
 
         // backup the output file to cloud
-        val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey")))
+        val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey"),Option(config.getString("cloudStorage.accountEndpoint")),Option("")))
         storageService.upload(config.getString("cloudStorage.container"), outputFilePath + "/part-00000", config.getString("cloudStorage.objectKey"), isDirectory = Option(false))
         println("successfully backed up file to cloud!")
         System.exit(0)