Merge pull request #786 from Sunbird-Knowlg/revert-781-KN-976

Revert "Issue #KN-976 fix: Updated Elasticsearch version to 7.17.13"

.circleci/config.yml

@@ -2,8 +2,8 @@ version: 2.0
 
 jobs:
   unit-tests:
-    machine:
-      image: ubuntu-2004:202201-02
+    docker:
+      - image: circleci/openjdk:14-jdk-buster-node-browsers-legacy
     resource_class: medium
     working_directory: ~/kp
     steps:

jobs-core/pom.xml

@@ -157,7 +157,7 @@
         <dependency>
             <groupId>org.elasticsearch.client</groupId>
             <artifactId>elasticsearch-rest-high-level-client</artifactId>
-            <version>7.17.13</version>
+            <version>6.8.22</version>
         </dependency>
         <dependency>
             <groupId>com.twitter</groupId>

jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala

@@ -8,19 +8,20 @@ import org.apache.commons.lang3.StringUtils
 import org.apache.http.HttpHost
 import org.apache.http.client.config.RequestConfig
 import org.elasticsearch.action.admin.indices.alias.Alias
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest
 import org.elasticsearch.action.bulk.BulkRequest
 import org.elasticsearch.action.delete.DeleteRequest
 import org.elasticsearch.action.get.GetRequest
 import org.elasticsearch.action.index.IndexRequest
 import org.elasticsearch.action.update.UpdateRequest
-import org.elasticsearch.client.indices.CreateIndexRequest
-import org.elasticsearch.client.{Request, RequestOptions, Response, RestClient, RestClientBuilder, RestHighLevelClient}
+import org.elasticsearch.client.{Response, RestClient, RestClientBuilder, RestHighLevelClient}
 import org.elasticsearch.common.settings.Settings
-import org.elasticsearch.xcontent.XContentType
+import org.elasticsearch.common.xcontent.XContentType
 import org.slf4j.LoggerFactory
+
 import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
 
-class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: Int = 1000) extends Serializable {
+class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: String, batchSize: Int = 1000) extends Serializable {
 
   private val resultLimit = 100
   private val esClient: RestHighLevelClient = createClient(connectionInfo)
@@ -48,7 +49,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
 
   def isIndexExists(): Boolean = {
     try {
-      val response = esClient.getLowLevelClient.performRequest(new Request("HEAD", "/" + indexName))
+      val response = esClient.getLowLevelClient.performRequest("HEAD", "/" + indexName)
       response.getStatusLine.getStatusCode == 200
     } catch {
       case e: IOException => {
@@ -58,27 +59,27 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
     }
   }
 
-    def addIndex(settings: String, mappings: String, alias: String = ""): Boolean = {
-      var response = false
-      val client = esClient
-      if (!isIndexExists()) {
-        val createRequest = new CreateIndexRequest(indexName)
-        if (StringUtils.isNotBlank(alias)) createRequest.alias(new Alias(alias))
-        if (StringUtils.isNotBlank(settings)) createRequest.settings(Settings.builder.loadFromSource(settings, XContentType.JSON))
-        if (StringUtils.isNotBlank(mappings)) createRequest.mapping(mappings, XContentType.JSON)
-        val createIndexResponse = client.indices.create(createRequest, RequestOptions.DEFAULT)
-        response = createIndexResponse.isAcknowledged
-      }
-      response
+  def addIndex(settings: String, mappings: String, alias: String = ""): Boolean = {
+    var response = false
+    val client = esClient
+    if (!isIndexExists()) {
+      val createRequest = new CreateIndexRequest(indexName)
+      if (StringUtils.isNotBlank(alias)) createRequest.alias(new Alias(alias))
+      if (StringUtils.isNotBlank(settings)) createRequest.settings(Settings.builder.loadFromSource(settings, XContentType.JSON))
+      if (StringUtils.isNotBlank(indexType) && StringUtils.isNotBlank(mappings)) createRequest.mapping(indexType, mappings, XContentType.JSON)
+      val createIndexResponse = client.indices.create(createRequest)
+      response = createIndexResponse.isAcknowledged
     }
+    response
+  }
 
   def addDocument(identifier: String, document: String): Unit = {
     try {
       // TODO
       // Replace mapper with JSONUtil once the JSONUtil is fixed
       val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {})
       val updatedDoc = checkDocStringLength(doc)
-      val response = esClient.index(new IndexRequest(indexName).id(identifier).source(updatedDoc), RequestOptions.DEFAULT)
+      val response = esClient.index(new IndexRequest(indexName, indexType, identifier).source(updatedDoc))
       logger.info(s"Added ${response.getId} to index ${response.getIndex}")
     } catch {
       case e: IOException =>
@@ -93,12 +94,14 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
       // Replace mapper with JSONUtil once the JSONUtil is fixed
       val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {})
       val updatedDoc = checkDocStringLength(doc)
-      val indexRequest = if(identifier == null) new IndexRequest(indexName) else new IndexRequest(indexName).id(identifier)
-      val response = esClient.index(indexRequest.source(updatedDoc), RequestOptions.DEFAULT)
+      val indexRequest = if(identifier == null) new IndexRequest(indexName, indexType) else new IndexRequest(indexName, indexType, identifier)
+      val response = esClient.index(indexRequest.source(updatedDoc))
       logger.info(s"Added ${response.getId} to index ${response.getIndex}")
     } catch {
       case e: IOException =>
         logger.error(s"ElasticSearchUtil:: Error while adding document to index : $indexName : " + e.getMessage)
+        e.printStackTrace()
+        throw e
     }
   }
 
@@ -108,9 +111,9 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
       // Replace mapper with JSONUtil once the JSONUtil is fixed
       val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {})
       val updatedDoc = checkDocStringLength(doc)
-      val indexRequest = new IndexRequest(indexName).id(identifier).source(updatedDoc)
-      val request = new UpdateRequest().index(indexName).id(identifier).doc(updatedDoc).upsert(indexRequest)
-      val response = esClient.update(request, RequestOptions.DEFAULT)
+      val indexRequest = new IndexRequest(indexName, indexType, identifier).source(updatedDoc)
+      val request = new UpdateRequest().index(indexName).`type`(indexType).id(identifier).doc(updatedDoc).upsert(indexRequest)
+      val response = esClient.update(request)
       logger.info(s"Updated ${response.getId} to index ${response.getIndex}")
     } catch {
       case e: IOException =>
@@ -119,12 +122,12 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
   }
 
   def deleteDocument(identifier: String): Unit = {
-    val response = esClient.delete(new DeleteRequest(indexName).id(identifier), RequestOptions.DEFAULT)
+    val response = esClient.delete(new DeleteRequest(indexName, indexType, identifier))
     logger.info(s"Deleted ${response.getId} to index ${response.getIndex}")
   }
 
   def getDocumentAsString(identifier: String): String = {
-    val response = esClient.get(new GetRequest(indexName).id(identifier), RequestOptions.DEFAULT)
+    val response = esClient.get(new GetRequest(indexName, indexType, identifier))
     response.getSourceAsString
   }
 
@@ -149,9 +152,9 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
           val doc: util.Map[String, AnyRef] = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {})
           val updatedDoc = checkDocStringLength(doc)
           logger.debug("ElasticSearchUtil:: bulkIndexWithIndexId:: doc: " + updatedDoc)
-          request.add(new IndexRequest(indexName).id(key).source(updatedDoc))
+          request.add(new IndexRequest(indexName, documentType, key).source(updatedDoc))
           if (count % batchSize == 0 || (count % batchSize < batchSize && count == jsonObjects.size)) {
-            val bulkResponse = esClient.bulk(request, RequestOptions.DEFAULT)
+            val bulkResponse = esClient.bulk(request)
             if (bulkResponse.hasFailures) logger.info("ElasticSearchUtil:: bulkIndexWithIndexId:: Failures in Elasticsearch bulkIndex : " + bulkResponse.buildFailureMessage)
           }
         }
@@ -162,7 +165,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In
 
   def isIndexExists(indexName: String): Boolean = {
     try {
-      val response: Response = esClient.getLowLevelClient.performRequest(new Request("HEAD", "/" + indexName))
+      val response: Response = esClient.getLowLevelClient.performRequest("HEAD", "/" + indexName)
       response.getStatusLine.getStatusCode == 200
     } catch {
       case e: IOException =>  false

jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala

@@ -9,7 +9,7 @@ import org.sunbird.job.util.ElasticSearchUtil
 class ElasticSearchUtilSpec extends FlatSpec with Matchers {
 
   val config: Config = ConfigFactory.load("base-test.conf")
-  val esUtil = new ElasticSearchUtil(config.getString("es.basePath"), "compositesearch")
+  val esUtil = new ElasticSearchUtil(config.getString("es.basePath"), "compositesearch", "cs")
 
 //  "isIndexExists" should "return true if index exists" in {
 //    val indexExists = esUtil.isIndexExists("compositesearch")

publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala

@@ -47,7 +47,7 @@ class CollectionPublishFunction(config: ContentPublishConfig, httpUtil: HttpUtil
     super.open(parameters)
     cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config)
     neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config)
-    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName)
+    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType)
     cloudStorageUtil = new CloudStorageUtil(config)
     ec = ExecutionContexts.global
     definitionCache = new DefinitionCache()

publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala

@@ -47,7 +47,7 @@ class LiveCollectionPublishFunction(config: LiveNodePublisherConfig, httpUtil: H
     super.open(parameters)
     cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config)
     neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config)
-    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName)
+    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType)
     cloudStorageUtil = new CloudStorageUtil(config)
     ec = ExecutionContexts.global
     definitionCache = new DefinitionCache()

publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala

@@ -35,7 +35,7 @@ class EcarGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers {
   "Object Ecar Generator generateEcar" should "return a Map containing Packaging Type and its url after uploading it to cloud" in {
 
     val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")))
-    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
+    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
     val objData = new ObjectData("do_123", metadata, None, Some(hierarchy))
     val obj = new TestEcarGenerator()
     val result = obj.generateEcar(objData,List("SPINE"))
@@ -50,6 +50,6 @@ class TestEcarGenerator extends EcarGenerator {
     "src" -> "somepath/sunbird_1551961194254.jpeg",
     "baseUrl" -> "some_base_url"
   )
-  val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))
+  val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))
   override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = Some(testObj)
 }

publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala

@@ -37,7 +37,7 @@ class ObjectEnrichmentSpec extends FlatSpec with BeforeAndAfterAll with Matchers
     when(mockNeo4JUtil.getNodesName(List("NCERT"))).thenReturn(Map("NCERT"-> "NCERT"))
 
     val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2")))
-    val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
+    val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
     val objData = new ObjectData("do_123", metadata, None, Some(hierarchy))
 
     val objectEnrichment = new TestObjectEnrichment()

publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala

@@ -25,7 +25,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche
   "Object Thumbnail Generator generateThumbnail" should "add the thumbnail to ObjectData" in {
 
     val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2")))
-    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
+    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
     val objData = new ObjectData("do_123", metadata, None, Some(hierarchy))
 
     val thumbnailGenerator = new TestThumbnailGenerator()
@@ -35,7 +35,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche
     resultMetadata.isEmpty should be(false)
     resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false)
     resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false)
-    resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png"
+    resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png"
     resultMetadata.getOrElse("appIcon", "").asInstanceOf[String] shouldBe "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_123/artifact/avatar_anonymous.thumb.png"
   }
 

qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala

@@ -27,7 +27,7 @@ class QRCodeImageGeneratorFunction(config: QRCodeImageGeneratorConfig,
   override def open(parameters: Configuration): Unit = {
     cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config)
     cloudStorageUtil = new CloudStorageUtil(config)
-    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex)
+    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType)
     qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil)
     super.open(parameters)
   }

qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala

@@ -24,7 +24,7 @@ class QRCodeIndexImageUrlFunction(config: QRCodeImageGeneratorConfig,
 
   override def open(parameters: Configuration): Unit = {
     cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config)
-    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex)
+    esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType)
     qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil)
     super.open(parameters)
   }

search-indexer/pom.xml

@@ -73,9 +73,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>elasticsearch</artifactId>
-            <version>1.19.0</version>
+            <groupId>pl.allegro.tech</groupId>
+            <artifactId>embedded-elasticsearch</artifactId>
+            <version>2.7.0</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala

@@ -10,7 +10,7 @@ trait DIALCodeIndexerHelper {
   private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeIndexerHelper])
 
   def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = {
-    val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}"""
+    val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["standard","lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}"""
     val mappings: String = """{"dynamic_templates":[{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"all_fields":{"type":"text","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}"""
     esUtil.addIndex(settings, mappings)
   }