From 3119dbbd95cf8dfe42df530034c3d419d16dc8f6 Mon Sep 17 00:00:00 2001 From: Kartheek Palla Date: Thu, 14 Mar 2024 10:25:24 +0530 Subject: [PATCH] Revert "Issue #KN-976 fix: Updated Elasticsearch version to 7.17.13" --- .circleci/config.yml | 4 +- jobs-core/pom.xml | 2 +- .../sunbird/job/util/ElasticSearchUtil.scala | 59 +++++++------- .../sunbird/spec/ElasticSearchUtilSpec.scala | 2 +- .../function/CollectionPublishFunction.scala | 2 +- .../LiveCollectionPublishFunction.scala | 2 +- .../job/publish/spec/EcarGeneratorSpec.scala | 4 +- .../publish/spec/ObjectEnrichmentSpec.scala | 2 +- .../publish/spec/ThumbnailGeneratorSpec.scala | 4 +- .../QRCodeImageGeneratorFunction.scala | 2 +- .../QRCodeIndexImageUrlFunction.scala | 2 +- search-indexer/pom.xml | 6 +- .../CompositeSearchIndexerHelper.scala | 4 +- .../helpers/DIALCodeIndexerHelper.scala | 2 +- .../DIALCodeMetricsIndexerHelper.scala | 2 +- .../CompositeSearchIndexerFunction.scala | 2 +- .../functions/DIALCodeIndexerFunction.scala | 2 +- .../DIALCodeMetricsIndexerFunction.scala | 2 +- .../task/SearchIndexerConfig.scala | 9 +-- search-indexer/src/test/resources/test.conf | 14 +--- .../job/spec/SearchIndexerTaskTestSpec.scala | 78 +++++++------------ .../functions/AuditHistoryIndexer.scala | 2 +- ...ransactionEventProcessorTaskTestSpec.scala | 12 +-- 23 files changed, 89 insertions(+), 131 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91028a4ec..378d1a698 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,8 +2,8 @@ version: 2.0 jobs: unit-tests: - machine: - image: ubuntu-2004:202201-02 + docker: + - image: circleci/openjdk:14-jdk-buster-node-browsers-legacy resource_class: medium working_directory: ~/kp steps: diff --git a/jobs-core/pom.xml b/jobs-core/pom.xml index d2c5c3366..e2f3d7c3b 100644 --- a/jobs-core/pom.xml +++ b/jobs-core/pom.xml @@ -157,7 +157,7 @@ org.elasticsearch.client elasticsearch-rest-high-level-client - 7.17.13 + 6.8.22 com.twitter diff --git a/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala b/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala index 0bd328619..16127e8eb 100644 --- a/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala +++ b/jobs-core/src/main/scala/org/sunbird/job/util/ElasticSearchUtil.scala @@ -8,19 +8,20 @@ import org.apache.commons.lang3.StringUtils import org.apache.http.HttpHost import org.apache.http.client.config.RequestConfig import org.elasticsearch.action.admin.indices.alias.Alias +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.delete.DeleteRequest import org.elasticsearch.action.get.GetRequest import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.update.UpdateRequest -import org.elasticsearch.client.indices.CreateIndexRequest -import org.elasticsearch.client.{Request, RequestOptions, Response, RestClient, RestClientBuilder, RestHighLevelClient} +import org.elasticsearch.client.{Response, RestClient, RestClientBuilder, RestHighLevelClient} import org.elasticsearch.common.settings.Settings -import org.elasticsearch.xcontent.XContentType +import org.elasticsearch.common.xcontent.XContentType import org.slf4j.LoggerFactory + import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` -class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: Int = 1000) extends Serializable { +class ElasticSearchUtil(connectionInfo: String, indexName: String, indexType: String, batchSize: Int = 1000) extends Serializable { private val resultLimit = 100 private val esClient: RestHighLevelClient = createClient(connectionInfo) @@ -48,7 +49,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In def isIndexExists(): Boolean = { try { - val response = esClient.getLowLevelClient.performRequest(new Request("HEAD", "/" + indexName)) + val response = esClient.getLowLevelClient.performRequest("HEAD", "/" + indexName) response.getStatusLine.getStatusCode == 200 } catch { case e: IOException => { @@ -58,19 +59,19 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In } } - def addIndex(settings: String, mappings: String, alias: String = ""): Boolean = { - var response = false - val client = esClient - if (!isIndexExists()) { - val createRequest = new CreateIndexRequest(indexName) - if (StringUtils.isNotBlank(alias)) createRequest.alias(new Alias(alias)) - if (StringUtils.isNotBlank(settings)) createRequest.settings(Settings.builder.loadFromSource(settings, XContentType.JSON)) - if (StringUtils.isNotBlank(mappings)) createRequest.mapping(mappings, XContentType.JSON) - val createIndexResponse = client.indices.create(createRequest, RequestOptions.DEFAULT) - response = createIndexResponse.isAcknowledged - } - response + def addIndex(settings: String, mappings: String, alias: String = ""): Boolean = { + var response = false + val client = esClient + if (!isIndexExists()) { + val createRequest = new CreateIndexRequest(indexName) + if (StringUtils.isNotBlank(alias)) createRequest.alias(new Alias(alias)) + if (StringUtils.isNotBlank(settings)) createRequest.settings(Settings.builder.loadFromSource(settings, XContentType.JSON)) + if (StringUtils.isNotBlank(indexType) && StringUtils.isNotBlank(mappings)) createRequest.mapping(indexType, mappings, XContentType.JSON) + val createIndexResponse = client.indices.create(createRequest) + response = createIndexResponse.isAcknowledged } + response + } def addDocument(identifier: String, document: String): Unit = { try { @@ -78,7 +79,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In // Replace mapper with JSONUtil once the JSONUtil is fixed val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) - val response = esClient.index(new IndexRequest(indexName).id(identifier).source(updatedDoc), RequestOptions.DEFAULT) + val response = esClient.index(new IndexRequest(indexName, indexType, identifier).source(updatedDoc)) logger.info(s"Added ${response.getId} to index ${response.getIndex}") } catch { case e: IOException => @@ -93,12 +94,14 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In // Replace mapper with JSONUtil once the JSONUtil is fixed val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) - val indexRequest = if(identifier == null) new IndexRequest(indexName) else new IndexRequest(indexName).id(identifier) - val response = esClient.index(indexRequest.source(updatedDoc), RequestOptions.DEFAULT) + val indexRequest = if(identifier == null) new IndexRequest(indexName, indexType) else new IndexRequest(indexName, indexType, identifier) + val response = esClient.index(indexRequest.source(updatedDoc)) logger.info(s"Added ${response.getId} to index ${response.getIndex}") } catch { case e: IOException => logger.error(s"ElasticSearchUtil:: Error while adding document to index : $indexName : " + e.getMessage) + e.printStackTrace() + throw e } } @@ -108,9 +111,9 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In // Replace mapper with JSONUtil once the JSONUtil is fixed val doc = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) - val indexRequest = new IndexRequest(indexName).id(identifier).source(updatedDoc) - val request = new UpdateRequest().index(indexName).id(identifier).doc(updatedDoc).upsert(indexRequest) - val response = esClient.update(request, RequestOptions.DEFAULT) + val indexRequest = new IndexRequest(indexName, indexType, identifier).source(updatedDoc) + val request = new UpdateRequest().index(indexName).`type`(indexType).id(identifier).doc(updatedDoc).upsert(indexRequest) + val response = esClient.update(request) logger.info(s"Updated ${response.getId} to index ${response.getIndex}") } catch { case e: IOException => @@ -119,12 +122,12 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In } def deleteDocument(identifier: String): Unit = { - val response = esClient.delete(new DeleteRequest(indexName).id(identifier), RequestOptions.DEFAULT) + val response = esClient.delete(new DeleteRequest(indexName, indexType, identifier)) logger.info(s"Deleted ${response.getId} to index ${response.getIndex}") } def getDocumentAsString(identifier: String): String = { - val response = esClient.get(new GetRequest(indexName).id(identifier), RequestOptions.DEFAULT) + val response = esClient.get(new GetRequest(indexName, indexType, identifier)) response.getSourceAsString } @@ -149,9 +152,9 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In val doc: util.Map[String, AnyRef] = mapper.readValue(document, new TypeReference[util.Map[String, AnyRef]]() {}) val updatedDoc = checkDocStringLength(doc) logger.debug("ElasticSearchUtil:: bulkIndexWithIndexId:: doc: " + updatedDoc) - request.add(new IndexRequest(indexName).id(key).source(updatedDoc)) + request.add(new IndexRequest(indexName, documentType, key).source(updatedDoc)) if (count % batchSize == 0 || (count % batchSize < batchSize && count == jsonObjects.size)) { - val bulkResponse = esClient.bulk(request, RequestOptions.DEFAULT) + val bulkResponse = esClient.bulk(request) if (bulkResponse.hasFailures) logger.info("ElasticSearchUtil:: bulkIndexWithIndexId:: Failures in Elasticsearch bulkIndex : " + bulkResponse.buildFailureMessage) } } @@ -162,7 +165,7 @@ class ElasticSearchUtil(connectionInfo: String, indexName: String, batchSize: In def isIndexExists(indexName: String): Boolean = { try { - val response: Response = esClient.getLowLevelClient.performRequest(new Request("HEAD", "/" + indexName)) + val response: Response = esClient.getLowLevelClient.performRequest("HEAD", "/" + indexName) response.getStatusLine.getStatusCode == 200 } catch { case e: IOException => false diff --git a/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala b/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala index cf5330502..838e33ac2 100644 --- a/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala +++ b/jobs-core/src/test/scala/org/sunbird/spec/ElasticSearchUtilSpec.scala @@ -9,7 +9,7 @@ import org.sunbird.job.util.ElasticSearchUtil class ElasticSearchUtilSpec extends FlatSpec with Matchers { val config: Config = ConfigFactory.load("base-test.conf") - val esUtil = new ElasticSearchUtil(config.getString("es.basePath"), "compositesearch") + val esUtil = new ElasticSearchUtil(config.getString("es.basePath"), "compositesearch", "cs") // "isIndexExists" should "return true if index exists" in { // val indexExists = esUtil.isIndexExists("compositesearch") diff --git a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala index 48deff45c..5dd7e0f63 100644 --- a/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala +++ b/publish-pipeline/content-publish/src/main/scala/org/sunbird/job/content/function/CollectionPublishFunction.scala @@ -47,7 +47,7 @@ class CollectionPublishFunction(config: ContentPublishConfig, httpUtil: HttpUtil super.open(parameters) cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType) cloudStorageUtil = new CloudStorageUtil(config) ec = ExecutionContexts.global definitionCache = new DefinitionCache() diff --git a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala index 1ad6e255f..b776e4206 100644 --- a/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala +++ b/publish-pipeline/live-node-publisher/src/main/scala/org/sunbird/job/livenodepublisher/function/LiveCollectionPublishFunction.scala @@ -47,7 +47,7 @@ class LiveCollectionPublishFunction(config: LiveNodePublisherConfig, httpUtil: H super.open(parameters) cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) neo4JUtil = new Neo4JUtil(config.graphRoutePath, config.graphName, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndexName, config.compositeSearchIndexType) cloudStorageUtil = new CloudStorageUtil(config) ec = ExecutionContexts.global definitionCache = new DefinitionCache() diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala index c71006152..e62da72e5 100644 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala +++ b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala @@ -35,7 +35,7 @@ class EcarGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers { "Object Ecar Generator generateEcar" should "return a Map containing Packaging Type and its url after uploading it to cloud" in { val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") + val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) val obj = new TestEcarGenerator() val result = obj.generateEcar(objData,List("SPINE")) @@ -50,6 +50,6 @@ class TestEcarGenerator extends EcarGenerator { "src" -> "somepath/sunbird_1551961194254.jpeg", "baseUrl" -> "some_base_url" ) - val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")) + val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")) override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = Some(testObj) } diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala index f43a3e159..d66422709 100644 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala +++ b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala @@ -37,7 +37,7 @@ class ObjectEnrichmentSpec extends FlatSpec with BeforeAndAfterAll with Matchers when(mockNeo4JUtil.getNodesName(List("NCERT"))).thenReturn(Map("NCERT"-> "NCERT")) val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") + val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) val objectEnrichment = new TestObjectEnrichment() diff --git a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala index 35a049da8..a5767806f 100644 --- a/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala +++ b/publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala @@ -25,7 +25,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche "Object Thumbnail Generator generateThumbnail" should "add the thumbnail to ObjectData" in { val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2"))) - val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") + val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live") val objData = new ObjectData("do_123", metadata, None, Some(hierarchy)) val thumbnailGenerator = new TestThumbnailGenerator() @@ -35,7 +35,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche resultMetadata.isEmpty should be(false) resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false) resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false) - resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png" + resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png" resultMetadata.getOrElse("appIcon", "").asInstanceOf[String] shouldBe "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_123/artifact/avatar_anonymous.thumb.png" } diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala index b781fa068..348ab95f2 100644 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala +++ b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeImageGeneratorFunction.scala @@ -27,7 +27,7 @@ class QRCodeImageGeneratorFunction(config: QRCodeImageGeneratorConfig, override def open(parameters: Configuration): Unit = { cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) cloudStorageUtil = new CloudStorageUtil(config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil) super.open(parameters) } diff --git a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala index 776461c06..c5d2a4001 100644 --- a/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala +++ b/qrcode-image-generator/src/main/scala/org/sunbird/job/qrimagegenerator/functions/QRCodeIndexImageUrlFunction.scala @@ -24,7 +24,7 @@ class QRCodeIndexImageUrlFunction(config: QRCodeImageGeneratorConfig, override def open(parameters: Configuration): Unit = { cassandraUtil = new CassandraUtil(config.cassandraHost, config.cassandraPort, config) - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) qRCodeImageGeneratorUtil = new QRCodeImageGeneratorUtil(config, cassandraUtil, cloudStorageUtil, esUtil) super.open(parameters) } diff --git a/search-indexer/pom.xml b/search-indexer/pom.xml index 32e6a3dda..a865f9b0b 100644 --- a/search-indexer/pom.xml +++ b/search-indexer/pom.xml @@ -73,9 +73,9 @@ test - org.testcontainers - elasticsearch - 1.19.0 + pl.allegro.tech + embedded-elasticsearch + 2.7.0 test diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala index 14fa62a1d..0915868e3 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/CompositeSearchIndexerHelper.scala @@ -13,8 +13,8 @@ trait CompositeSearchIndexerHelper { private[this] val logger = LoggerFactory.getLogger(classOf[CompositeSearchIndexerHelper]) def createCompositeSearchIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1500"}},"analysis":{"filter":{"mynGram":{"token_chars":["letter","digit","whitespace","punctuation","symbol"],"min_gram":"1","type":"nGram","max_gram":"30"}},"analyzer":{"cs_index_analyzer":{"filter":["lowercase","mynGram"],"type":"custom","tokenizer":"standard"},"keylower":{"filter":"lowercase","tokenizer":"keyword"},"cs_search_analyzer":{"filter":["lowercase"],"type":"custom","tokenizer":"standard"}}}}""" - val mappings = """{"dynamic_templates":[{"nested":{"match_mapping_type":"object","mapping":{"fields":{"type":"nested"},"type":"nested"}}},{"longs":{"match_mapping_type":"long","mapping":{"fields":{"raw":{"type":"long"}},"type":"long"}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"fields":{"raw":{"type":"boolean"}},"type":"boolean"}}},{"doubles":{"match_mapping_type":"double","mapping":{"fields":{"raw":{"type":"double"}},"type":"double"}}},{"dates":{"match_mapping_type":"date","mapping":{"fields":{"raw":{"type":"date"}},"type":"date"}}},{"strings":{"match_mapping_type":"string","mapping":{"analyzer":"cs_index_analyzer","copy_to":"all_fields","fields":{"raw":{"fielddata":true,"analyzer":"keylower","type":"text"}},"search_analyzer":"cs_search_analyzer","type":"text"}}}],"properties":{"IL_FUNC_OBJECT_TYPE":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"IL_SYS_NODE_TYPE":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"IL_UNIQUE_ID":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"SYS_INTERNAL_LAST_UPDATED_ON":{"type":"date","fields":{"raw":{"type":"date"}}},"additionalCategories":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"all_fields":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"appIcon":{"type":"text","index":false},"appId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"artifactUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"assets":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"associations":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"associationswith":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"audience":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"author":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"autoCreateBatch":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"batches":{"type":"nested","properties":{"batchId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"createdFor":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"enrollmentType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"startDate":{"type":"date","fields":{"raw":{"type":"date"}}},"status":{"type":"long","fields":{"raw":{"type":"long"}}}}},"board":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"body":{"type":"text","index":false},"categories":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"category":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"categoryId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"channel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"channels":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"childNodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"cloudStorageKey":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"code":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"compatibilityLevel":{"type":"long","fields":{"raw":{"type":"long"}}},"consumerId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentDisposition":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentEncoding":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"contentTypesCount":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"copyright":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"copyrightYear":{"type":"long","fields":{"raw":{"type":"long"}}},"createdBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"createdFor":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"createdOn":{"type":"date","fields":{"raw":{"type":"date"}}},"creator":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"credentials":{"type":"nested","properties":{"enabled":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"defaultCourseFramework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"defaultFramework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"defaultLicense":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"depth":{"type":"long","fields":{"raw":{"type":"long"}}},"description":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"dialcodeRequired":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"dialcodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"discussionForum":{"type":"nested","properties":{"enabled":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"downloadUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"editorState":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"framework":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"frameworks":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"generateDIALCodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"gradeLevel":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"graph_id":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"idealScreenDensity":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"idealScreenSize":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"identifier":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"index":{"type":"long","fields":{"raw":{"type":"long"}}},"interceptionPoints":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"language":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"languageCode":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastPublishedBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastPublishedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"lastStatusChangedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"lastSubmittedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"lastUpdatedBy":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lastUpdatedOn":{"type":"date","fields":{"raw":{"type":"date"}}},"leafNodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"leafNodesCount":{"type":"long","fields":{"raw":{"type":"long"}}},"license":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"lockKey":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mediaType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"medium":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mimeType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"mimeTypesCount":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"name":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"nodeType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"node_id":{"type":"long","fields":{"raw":{"type":"long"}}},"objectType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"orgIdFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"organisation":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"os":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"osId":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"ownershipType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"parent":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"pkgVersion":{"type":"long","fields":{"raw":{"type":"long"}}},"plugins":{"type":"nested","properties":{"identifier":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"semanticVersion":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"pragma":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"prevState":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"prevStatus":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"previewUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"primaryCategory":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"publishError":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"publish_type":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"reservedDialcodes":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"resourceType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"s3Key":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"sYS_INTERNAL_LAST_UPDATED_ON":{"type":"date","fields":{"raw":{"type":"date"}}},"screenshots":{"type":"text","index":false},"se_FWIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_boardIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_boards":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_gradeLevelIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_gradeLevels":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_mediumIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_mediums":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_subjectIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"se_subjects":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"searchIdFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"searchLabelFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"size":{"type":"double","fields":{"raw":{"type":"double"}}},"status":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"streamingUrl":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"subject":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"subjectIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"systemDefault":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetBoardIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetFWIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetGradeLevelIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetIdFieldName":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetMediumIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetObjectType":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"targetSubjectIds":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"terms":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"toc_url":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"totalCompressedSize":{"type":"long","fields":{"raw":{"type":"long"}}},"totalQuestions":{"type":"long","fields":{"raw":{"type":"long"}}},"totalScore":{"type":"long","fields":{"raw":{"type":"long"}}},"trackable":{"type":"nested","properties":{"autoBatch":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"enabled":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}},"type":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"url":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"userConsent":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"variants":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"version":{"type":"long","fields":{"raw":{"type":"long"}}},"versionKey":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"},"visibility":{"type":"text","fields":{"raw":{"type":"text","analyzer":"keylower","fielddata":true}},"copy_to":["all_fields"],"analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer"}}}""" + val settings = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1500"}},"analysis":{"filter":{"mynGram":{"token_chars":["letter","digit","whitespace","punctuation","symbol"],"min_gram":"1","type":"nGram","max_gram":"30"}},"analyzer":{"cs_index_analyzer":{"filter":["lowercase","mynGram"],"type":"custom","tokenizer":"standard"},"keylower":{"filter":"lowercase","tokenizer":"keyword"},"cs_search_analyzer":{"filter":["standard","lowercase"],"type":"custom","tokenizer":"standard"}}}}""" + val mappings = """{"dynamic_templates":[{"nested":{"match_mapping_type":"object","mapping":{"type":"nested","fields":{"type":"nested"}}}},{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"screenshots":{"type":"text","index":false},"body":{"type":"text","index":false},"appIcon":{"type":"text","index":false},"all_fields":{"type":"text","analyzer":"cs_index_analyzer","search_analyzer":"cs_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}""" esUtil.addIndex(settings, mappings) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala index 4343b1eaf..5b5050ede 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeIndexerHelper.scala @@ -10,7 +10,7 @@ trait DIALCodeIndexerHelper { private[this] val logger = LoggerFactory.getLogger(classOf[DIALCodeIndexerHelper]) def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = { - val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}""" + val settings: String = """{"max_ngram_diff":"29","mapping":{"total_fields":{"limit":"1050"}},"analysis":{"analyzer":{"dc_index_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","mynGram"]},"dc_search_analyzer":{"type":"custom","tokenizer":"standard","filter":["standard","lowercase"]},"keylower":{"tokenizer":"keyword","filter":"lowercase"}},"filter":{"mynGram":{"type":"nGram","min_gram":1,"max_gram":30,"token_chars":["letter","digit","whitespace","punctuation","symbol"]}}}}""" val mappings: String = """{"dynamic_templates":[{"longs":{"match_mapping_type":"long","mapping":{"type":"long","fields":{"raw":{"type":"long"}}}}},{"booleans":{"match_mapping_type":"boolean","mapping":{"type":"boolean","fields":{"raw":{"type":"boolean"}}}}},{"doubles":{"match_mapping_type":"double","mapping":{"type":"double","fields":{"raw":{"type":"double"}}}}},{"dates":{"match_mapping_type":"date","mapping":{"type":"date","fields":{"raw":{"type":"date"}}}}},{"strings":{"match_mapping_type":"string","mapping":{"type":"text","copy_to":"all_fields","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}],"properties":{"all_fields":{"type":"text","analyzer":"dc_index_analyzer","search_analyzer":"dc_search_analyzer","fields":{"raw":{"type":"text","fielddata":true,"analyzer":"keylower"}}}}}""" esUtil.addIndex(settings, mappings) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala index bae69abcd..3c0a403af 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/compositesearch/helpers/DIALCodeMetricsIndexerHelper.scala @@ -11,7 +11,7 @@ trait DIALCodeMetricsIndexerHelper { def createDialCodeIndex()(esUtil: ElasticSearchUtil): Boolean = { val settings: String = """{"number_of_shards":5}""" - val mappings: String = """{"dynamic":false,"properties":{"dial_code":{"type":"keyword"},"total_dial_scans_local":{"type":"double"},"total_dial_scans_global":{"type":"double"},"average_scans_per_day":{"type":"double"},"last_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"first_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"}}}""" + val mappings: String = """{"dcm":{"dynamic":false,"properties":{"dial_code":{"type":"keyword"},"total_dial_scans_local":{"type":"double"},"total_dial_scans_global":{"type":"double"},"average_scans_per_day":{"type":"double"},"last_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"first_scan":{"type":"date","format":"strict_date_optional_time||epoch_millis"}}}}""" esUtil.addIndex(settings, mappings) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala index a1845db62..998dce0f3 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/CompositeSearchIndexerFunction.scala @@ -23,7 +23,7 @@ class CompositeSearchIndexerFunction(config: SearchIndexerConfig, override def open(parameters: Configuration): Unit = { super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndex) + elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.compositeSearchIndex, config.compositeSearchIndexType) createCompositeSearchIndex()(elasticUtil) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala index fa7160157..41f4e1cfd 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeIndexerFunction.scala @@ -21,7 +21,7 @@ class DIALCodeIndexerFunction(config: SearchIndexerConfig, override def open(parameters: Configuration): Unit = { super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex) + elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeExternalIndex, config.dialcodeExternalIndexType) createDialCodeIndex()(elasticUtil) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala index 6db5ff9fe..eeb6684d2 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/functions/DIALCodeMetricsIndexerFunction.scala @@ -21,7 +21,7 @@ class DIALCodeMetricsIndexerFunction(config: SearchIndexerConfig, override def open(parameters: Configuration): Unit = { super.open(parameters) - elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeMetricIndex) + elasticUtil = new ElasticSearchUtil(config.esConnectionInfo, config.dialcodeMetricIndex, config.dialcodeMetricIndexType) createDialCodeIndex()(elasticUtil) } diff --git a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala index 77bb323eb..9adad0e00 100644 --- a/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala +++ b/search-indexer/src/main/scala/org/sunbird/job/searchindexer/task/SearchIndexerConfig.scala @@ -72,11 +72,4 @@ class SearchIndexerConfig(override val config: Config) extends BaseJobConfig(con val ignoredFields: List[String] = if (config.hasPath("ignored.fields")) config.getStringList("ignored.fields").asScala.toList else List("responseDeclaration", "body") val isrRelativePathEnabled: Boolean = if (config.hasPath("cloudstorage.metadata.replace_absolute_path")) config.getBoolean("cloudstorage.metadata.replace_absolute_path") else false - val esImage: String = if (config.hasPath("es.image")) config.getString("es.image") else "docker.elastic.co/elasticsearch/elasticsearch" - val esImageTag: String = if (config.hasPath("es.imageTag")) config.getString("es.imageTag") else "7.17.13" - val esPorts : util.List[String] = if (config.hasPath("es.ports")) config.getStringList("es.ports") else List("9200:9200").asJava - val esJavaOpts: String = if (config.hasPath("es.javaOpts")) config.getString("es.javaOpts") else "-Xms128m -Xmx512m" - val esJavaOptsKey: String = if (config.hasPath("es.javaOptsKey")) config.getString("es.javaOptsKey") else "ES_JAVA_OPTS" - val xpackSecurityEnabled: String = if (config.hasPath("es.xpackSecurityEnabled")) config.getString("es.xpackSecurityEnabled") else "false" - val xpackSecurityKey: String = if (config.hasPath("es.xpackSecurityKey")) config.getString("es.xpackSecurityKey") else "xpack.security.enabled" -} \ No newline at end of file +} diff --git a/search-indexer/src/test/resources/test.conf b/search-indexer/src/test/resources/test.conf index edae4e796..1faacbda4 100644 --- a/search-indexer/src/test/resources/test.conf +++ b/search-indexer/src/test/resources/test.conf @@ -32,16 +32,4 @@ schema { itemset = "2.0" } } -schema.definition_cache.expiry = 14400 - -es { - image = "docker.elastic.co/elasticsearch/elasticsearch" - imageTag = "7.17.13" - ports = ["9200:9200"] - javaOptsKey = "ES_JAVA_OPTS" - javaOpts = "-Xms128m -Xmx512m" - xpackSecurityKey = "xpack.security.enabled" - xpackSecurityEnabled = "false" - -} - +schema.definition_cache.expiry = 14400 \ No newline at end of file diff --git a/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala b/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala index 4e93135e7..d8bc1ee6b 100644 --- a/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala +++ b/search-indexer/src/test/scala/org/sunbird/job/spec/SearchIndexerTaskTestSpec.scala @@ -8,8 +8,6 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext import org.apache.flink.test.util.MiniClusterWithClientResource -import org.apache.http.HttpHost -import org.elasticsearch.client.{Request, RestClient, RestClientBuilder} import org.mockito.ArgumentMatchers.anyString import org.mockito.Mockito import org.mockito.Mockito.{doNothing, times, verify, when} @@ -21,11 +19,8 @@ import org.sunbird.job.searchindexer.functions.{CompositeSearchIndexerFunction, import org.sunbird.job.searchindexer.task.{SearchIndexerConfig, SearchIndexerStreamTask} import org.sunbird.job.util.{ElasticSearchUtil, ScalaJsonUtil} import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} -import org.apache.http.client.config.RequestConfig -import org.testcontainers.containers.wait.strategy.Wait -import org.testcontainers.elasticsearch.ElasticsearchContainer -import org.testcontainers.utility.DockerImageName -import java.time.Duration +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic + import java.util import scala.collection.JavaConverters._ @@ -44,40 +39,23 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { val config: Config = ConfigFactory.load("test.conf") val jobConfig = new SearchIndexerConfig(config) val mockElasticUtil = mock[ElasticSearchUtil](Mockito.withSettings().serializable()) - var elasticContainer = new ElasticsearchContainer(DockerImageName.parse(jobConfig.esImage).withTag(jobConfig.esImageTag)) - var restClient: RestClient = _ + var embeddedElastic: EmbeddedElastic = _ val defCache = new DefinitionCache() - def setupElasticContainer(): Unit = { - elasticContainer.setHostAccessible(true) - elasticContainer.setPortBindings(jobConfig.esPorts) - elasticContainer.withEnv(jobConfig.esJavaOptsKey, jobConfig.esJavaOpts) - elasticContainer.withEnv(jobConfig.xpackSecurityKey, jobConfig.xpackSecurityEnabled) - elasticContainer.withStartupTimeout(Duration.ofSeconds(60)) - elasticContainer.waitingFor(Wait.forListeningPort()) - elasticContainer.start() - Thread.sleep(20000) - - restClient = RestClient - .builder(HttpHost.create(elasticContainer.getHttpHostAddress)) - .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { - override def customizeRequestConfig(requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = { - requestConfigBuilder.setConnectionRequestTimeout(-1) - } - }).build() - } - override protected def beforeAll(): Unit = { super.beforeAll() - setupElasticContainer() + + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion("6.8.22") + .withEsJavaOpts("-Xms128m -Xmx512m") + .build() + .start() flinkCluster.before() } override protected def afterAll(): Unit = { super.afterAll() - if(elasticContainer != null){ - elasticContainer.stop() - } + embeddedElastic.stop() flinkCluster.after() } @@ -363,7 +341,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data.isEmpty should be(false) data.contains("do_1132247274257203201191") should be(true) @@ -378,7 +356,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data.isEmpty should be(false) data.contains("do_1132247274257203201191") should be(true) @@ -390,12 +368,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " create and delete the Data Node " in { - restClient.performRequest(new Request("DELETE", "/_all")) + embeddedElastic.deleteIndices() when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_CREATE, EventFixture.DATA_NODE_DELETE))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data should be(null) BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) @@ -405,12 +383,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " do nothing for the Data Node due to UNKNOWN Operation " in { - restClient.performRequest(new Request("DELETE", "/_all")) + embeddedElastic.deleteIndices() when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_UNKNOWN))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.compositeSearchIndex, jobConfig.compositeSearchIndexType) val data = elasticUtil.getDocumentAsString("do_1132247274257203201191") data should be(null) } @@ -420,7 +398,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) val data = elasticUtil.getDocumentAsString("X8R3W4") data.isEmpty should be(false) data.contains("X8R3W4") should be(true) @@ -435,7 +413,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) val data = elasticUtil.getDocumentAsString("X8R3W4") data.isEmpty should be(false) data.contains("X8R3W4") should be(true) @@ -447,12 +425,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " create and delete the External Dialcode Data " in { - restClient.performRequest(new Request("DELETE", "/_all")) + embeddedElastic.deleteIndices() when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_CREATE, EventFixture.DIALCODE_EXTERNAL_DELETE))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) val data = elasticUtil.getDocumentAsString("X8R3W4") data should be(null) BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) @@ -462,12 +440,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " do nothing for the External Dialcode Data due to UNKNOWN Operation " in { - restClient.performRequest(new Request("DELETE", "/_all")) + embeddedElastic.deleteIndices() when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_EXTERNAL_UNKNOWN))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeExternalIndex, jobConfig.dialcodeExternalIndexType) val data = elasticUtil.getDocumentAsString("X8R3W4") data should be(null) } @@ -477,7 +455,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) val data = elasticUtil.getDocumentAsString("QR1234") data.isEmpty should be(false) data.contains("QR1234") should be(true) @@ -492,7 +470,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) val data = elasticUtil.getDocumentAsString("QR1234") data.isEmpty should be(false) data.contains("QR1234") should be(true) @@ -504,12 +482,12 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " create and delete the Dialcode Metrics Data " in { - restClient.performRequest(new Request("DELETE", "/_all")) + embeddedElastic.deleteIndices() when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_CREATE, EventFixture.DIALCODE_METRIC_DELETE))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) val data = elasticUtil.getDocumentAsString("QR1234") data should be(null) BaseMetricsReporter.gaugeMetrics(s"${jobConfig.jobName}.${jobConfig.totalEventsCount}").getValue() should be(2) @@ -522,7 +500,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DIALCODE_METRIC_UNKNOWN))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) new SearchIndexerStreamTask(jobConfig, mockKafkaUtil).process() - val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex) + val elasticUtil = new ElasticSearchUtil(jobConfig.esConnectionInfo, jobConfig.dialcodeMetricIndex, jobConfig.dialcodeMetricIndexType) val data = elasticUtil.getDocumentAsString("QR1234") data should be(null) } @@ -544,7 +522,7 @@ class SearchIndexerTaskTestSpec extends BaseTestSpec { } "Composite Search Indexer" should " give error for the External Dialcode Data due to UNKNOWN objectType " in { - restClient.performRequest(new Request("DELETE", "/_all")) + embeddedElastic.deleteIndices() when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new CompositeSearchEventSource(List[String](EventFixture.DATA_NODE_FAILED))) when(mockKafkaUtil.kafkaStringSink(jobConfig.kafkaErrorTopic)).thenReturn(new CompositeSearchFailedEventSink) intercept[Exception] { diff --git a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala index 6c7a216e9..56378bd22 100644 --- a/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala +++ b/transaction-event-processor/src/main/scala/org/sunbird/job/transaction/functions/AuditHistoryIndexer.scala @@ -25,7 +25,7 @@ class AuditHistoryIndexer(config: TransactionEventProcessorConfig, var esUtil: E override def open(parameters: Configuration): Unit = { super.open(parameters) if (esUtil == null) { - esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.auditHistoryIndex) + esUtil = new ElasticSearchUtil(config.esConnectionInfo, config.auditHistoryIndex, config.auditHistoryIndexType) } } diff --git a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala b/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala index f481d49cd..4e3a98231 100644 --- a/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala +++ b/transaction-event-processor/src/test/scala/org/sunbird/job/spec/TransactionEventProcessorTaskTestSpec.scala @@ -129,11 +129,9 @@ class TransactionEventProcessorTaskTestSpec extends BaseTestSpec { "TransactionEventProcessorStreamTask" should "not generate audit history indexer event" in { server.start(9200) - server.enqueue(new MockResponse().setHeader("X-Elastic-Product", "Elasticsearch").setHeader( + server.enqueue(new MockResponse().setHeader( "Content-Type", "application/json" - ).setBody( - """{"name": "MacBook-Air.local","cluster_name": "elasticsearch","cluster_uuid": "9ra4wTGZSamseO3I99w","version": {"number": "7.17.13","build_flavor": "default","build_type": "tar","build_hash": "2b211dbb8bfd7f5b44d356bdfe54b1050c13","build_date": "2023-08-31T17:33:19.958690787Z","build_snapshot": false,"lucene_version": "8.11.1","minimum_wire_compatibility_version": "6.8.0","minimum_index_compatibility_version": "6.0.0-beta1"},"tagline": "You Know, for Search"} - |,{"_index":"kp_audit_log_2018_7","_type":"_doc","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":0,"failed":1},"_seq_no":1,"_primary_term":1}""".stripMargin)) + ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) if (jobConfig.auditHistoryIndexer) { @@ -147,11 +145,9 @@ class TransactionEventProcessorTaskTestSpec extends BaseTestSpec { } "TransactionEventProcessorStreamTask" should "generate audit history indexer event" in { - server.enqueue(new MockResponse().setHeader("X-Elastic-Product", "Elasticsearch").setHeader( + server.enqueue(new MockResponse().setHeader( "Content-Type", "application/json" - ).setBody( - """{"name": "MacBook-Air.local","cluster_name": "elasticsearch","cluster_uuid": "9ra4wTGZEFPeO3I99w","version": {"number": "7.17.13","build_flavor": "default","build_type": "tar","build_hash": "2b211dbb8bf7f5b44d356bdfe54b1050c13","build_date": "2023-08-31T17:33:19.958690787Z","build_snapshot": false,"lucene_version": "8.11.1","minimum_wire_compatibility_version": "6.8.0","minimum_index_compatibility_version": "6.0.0-beta1"},"tagline": "You Know, for Search"} - |,{"_index":"kp_audit_log_2018_7","_type":"_doc","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""".stripMargin)) + ).setBody("""{"_index":"kp_audit_log_2018_7","_type":"ah","_id":"HLZ-1ngBtZ15DPx6ENjU","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}""")) when(mockKafkaUtil.kafkaJobRequestSource[Event](jobConfig.kafkaInputTopic)).thenReturn(new AuditHistoryMapSource) val setBoolean = config.withValue("job.audit-history-indexer", ConfigValueFactory.fromAnyRef(true))