diff --git a/etl-jobs/ingestion-spec/content_index_batch.json b/etl-jobs/ingestion-spec/content_index_batch.json index 9a24c8efd..a9e5a77c5 100644 --- a/etl-jobs/ingestion-spec/content_index_batch.json +++ b/etl-jobs/ingestion-spec/content_index_batch.json @@ -785,13 +785,18 @@ "ioConfig":{ "type":"index", "firehose" : { - "type" : "static-azure-blobstore", - "blobs": [ - { - "container": "telemetry-data-store", - "path": "/druid-content-snapshot/snapshot.txt" - } - ], +{% if cloud_storage.provider == "azure" %} + "type": "static-azure-blobstore", + "blobs": [ + { + "container": "{{cloud_storage.container}}", + "path": "/{{cloud_storage.object_key}}" + } + ], +{% else %} + "type": "static-s3", + "uris": [ "s3://{{cloud_storage.container}}/{{cloud_storage.object_key}}"], +{% endif %} "fetchTimeout": 300000 } }, diff --git a/etl-jobs/ingestion-spec/vdn_content_index_batch.json b/etl-jobs/ingestion-spec/vdn_content_index_batch.json index f03b29a4c..cb17c5e88 100644 --- a/etl-jobs/ingestion-spec/vdn_content_index_batch.json +++ b/etl-jobs/ingestion-spec/vdn_content_index_batch.json @@ -847,13 +847,18 @@ "ioConfig":{ "type":"index", "firehose" : { - "type" : "static-azure-blobstore", - "blobs": [ - { - "container": "telemetry-data-store", - "path": "/druid-content-snapshot/vdn-snapshot.txt" - } - ], +{% if cloud_storage.provider == "azure" %} + "type": "static-azure-blobstore", + "blobs": [ + { + "container": "{{cloud_storage.container}}", + "path": "/{{cloud_storage.object_key}}" + } + ], +{% else %} + "type": "static-s3", + "uris": [ "s3://{{cloud_storage.container}}/{{cloud_storage.object_key}}"], +{% endif %} "fetchTimeout": 3000000 } }, diff --git a/etl-jobs/pom.xml b/etl-jobs/pom.xml index 6633d8803..9816db29c 100644 --- a/etl-jobs/pom.xml +++ b/etl-jobs/pom.xml @@ -24,7 +24,7 @@ jets3t - net.java.dev.jets3t + org.jets3t org.apache.xbean @@ -153,7 +153,7 @@ org.sunbird cloud-store-sdk_${scala.maj.version} - 1.3.0 + 1.4.6 com.microsoft.azure diff --git a/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala b/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala index 17e6a2bd8..37f27a228 100644 --- a/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala +++ b/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala @@ -46,7 +46,7 @@ object ESCloudUploader { .saveAsTextFile(outputFilePath) // backup the output file to cloud - val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey"))) + val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey"),Option(config.getString("cloudStorage.accountEndpoint")),Option(""))) storageService.upload(config.getString("cloudStorage.container"), outputFilePath + "/part-00000", config.getString("cloudStorage.objectKey"), isDirectory = Option(false)) println("successfully backed up file to cloud!") System.exit(0)