Merge pull request #167 from sowmya-dixit/release-5.1.3-s3n

WFS execution issue fixes
Sunbird-Obsrv · Mar 4, 2024 · 49fd59b · 49fd59b
2 parents 8bca99e + 4e237ff
commit 49fd59b
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/analytics-core/pom.xml b/analytics-core/pom.xml
@@ -16,6 +16,9 @@
 		<maven.compiler.target>11</maven.compiler.target>
 		<encoding>UTF-8</encoding>
 		<scoverage.plugin.version>1.1.1</scoverage.plugin.version>
+		<CLOUD_STORE_GROUP_ID>org.sunbird</CLOUD_STORE_GROUP_ID>
+		<CLOUD_STORE_ARTIFACT_ID>cloud-store-sdk_2.12</CLOUD_STORE_ARTIFACT_ID>
+		<CLOUD_STORE_VERSION>1.4.6</CLOUD_STORE_VERSION>
 	</properties>
 
 	<repositories>

diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CSPUtils.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CSPUtils.scala
@@ -65,5 +65,9 @@ object OCIProvider extends ICloudStorageProvider {
     JobLogger.log("Configuring OCI Access Key & Secret Key to SparkContext")
     sc.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", key);
     sc.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", secret);
+    val storageEndpoint = AppConf.getConfig("cloud_storage_endpoint_with_protocol")
+    if (storageEndpoint.nonEmpty) {
+      sc.hadoopConfiguration.set("fs.s3n.endpoint", storageEndpoint)
+    }
   }
 }
diff --git a/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala b/analytics-core/src/main/scala/org/ekstep/analytics/framework/util/CommonUtil.scala
@@ -16,11 +16,9 @@ import org.ekstep.analytics.framework.Level._
 import org.ekstep.analytics.framework.Period._
 import org.ekstep.analytics.framework.util.CloudStorageProviders.setSparkCSPConfigurations
 import org.ekstep.analytics.framework.{DtRange, Event, JobConfig, _}
-
 import scala.collection.mutable.ListBuffer
 //import org.ekstep.analytics.framework.conf.AppConf
 import java.util.zip.{ZipEntry, ZipOutputStream}
-
 import org.apache.commons.lang3.StringUtils
 import org.joda.time.format.{DateTimeFormat, DateTimeFormatter}
 import org.joda.time.{DateTime, DateTimeZone, Days, LocalDate, Weeks, Years}
@@ -91,7 +89,9 @@ object CommonUtil {
     }
 
     val sc = new SparkContext(conf)
-    setSparkCSPConfigurations(sc, AppConf.getConfig("cloud_storage_type"), None, None)
+    val key = AppConf.getConfig("storage.key.config")
+    val secret = AppConf.getConfig("storage.secret.config")
+    setSparkCSPConfigurations(sc, AppConf.getConfig("cloud_storage_type"), Option(key), Option(secret))
     JobLogger.log("Spark Context initialized")
     sc
   }