From 7aaf654f0193b619a504d1cb68cc2faee2bf5ec0 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Thu, 10 Aug 2023 16:47:37 -0700 Subject: [PATCH] Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch (#12782) * Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch * Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch * Fix #12779: Add support for SearchIndexes for ElasticSearch and OpenSearch * Rebase fixes with main * Add Sample Data * lint fix * remove unused import * Fix service count test --------- Co-authored-by: ulixius9 --- .../v014__create_db_connection_info.sql | 24 - .../v015__create_db_connection_info.sql | 51 ++ .../v014__create_db_connection_info.sql | 26 +- .../v015__create_db_connection_info.sql | 51 ++ .../searchIndexes/searchIndexes.json | 76 +++ .../sample_data/searchIndexes/service.json | 12 + .../src/metadata/ingestion/api/parser.py | 10 + .../src/metadata/ingestion/ometa/ometa_api.py | 15 +- .../src/metadata/ingestion/ometa/utils.py | 3 + .../ingestion/source/database/sample_data.py | 57 +++ .../java/org/openmetadata/service/Entity.java | 2 + .../service/jdbi3/CollectionDAO.java | 49 +- .../service/jdbi3/SearchIndexRepository.java | 459 +++++++++++++++++ .../jdbi3/SearchServiceRepository.java | 19 + .../searchindex/SearchIndexResource.java | 480 ++++++++++++++++++ .../searchIndexes/SearchServiceResource.java | 436 ++++++++++++++++ .../service/security/mask/PIIMasker.java | 22 + .../openmetadata/service/util/EntityUtil.java | 14 + .../service/resources/EntityResourceTest.java | 4 + .../searchindex/SearchIndexResourceTest.java | 426 ++++++++++++++++ .../services/SearchServiceResourceTest.java | 208 ++++++++ .../resources/system/SystemResourceTest.java | 13 +- .../openmetadata/service/util/TestUtils.java | 14 + .../schema/api/data/createSearchIndex.json | 70 +++ .../api/services/createSearchService.json | 48 ++ .../json/schema/entity/data/searchIndex.json | 229 +++++++++ .../search/customSearchConnection.json | 36 ++ .../search/elasticSearchConnection.json | 80 +++ .../search/openSearchConnection.json | 79 +++ .../connections/serviceConnection.json | 3 + .../schema/entity/services/searchService.json | 142 ++++++ .../schema/entity/services/serviceType.json | 3 +- .../searchServiceMetadataPipeline.json | 27 + .../schema/metadataIngestion/workflow.json | 3 + 34 files changed, 3129 insertions(+), 62 deletions(-) create mode 100644 ingestion/examples/sample_data/searchIndexes/searchIndexes.json create mode 100644 ingestion/examples/sample_data/searchIndexes/service.json create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchServiceRepository.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/services/SearchServiceResourceTest.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/data/createSearchIndex.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/services/createSearchService.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/data/searchIndex.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/customSearchConnection.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/openSearchConnection.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql index 6da982a11d14..470e2c618ad9 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v014__create_db_connection_info.sql @@ -1,27 +1,3 @@ --- create domain entity table -CREATE TABLE IF NOT EXISTS domain_entity ( - id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, - name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, - fqnHash VARCHAR(256) NOT NULL, - json JSON NOT NULL, - updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, - updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, - PRIMARY KEY (id), - UNIQUE (fqnHash) -); - --- create data product entity table -CREATE TABLE IF NOT EXISTS data_product_entity ( - id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, - name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, - fqnHash VARCHAR(256) NOT NULL, - json JSON NOT NULL, - updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, - updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, - PRIMARY KEY (id), - UNIQUE (fqnHash) -); - -- Rename includeTempTables with includeTransTables UPDATE dbservice_entity SET json = JSON_REMOVE( diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v015__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v015__create_db_connection_info.sql index ca2ec2ac907b..665ccd764c8b 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v015__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v015__create_db_connection_info.sql @@ -1,3 +1,54 @@ -- column deleted not needed for entities that don't support soft delete ALTER TABLE query_entity DROP COLUMN deleted; ALTER TABLE event_subscription_entity DROP COLUMN deleted; + +-- create domain entity table +CREATE TABLE IF NOT EXISTS domain_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) + ); + +-- create data product entity table +CREATE TABLE IF NOT EXISTS data_product_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) + ); + +-- create search service entity +CREATE TABLE IF NOT EXISTS search_service_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.serviceType') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), + PRIMARY KEY (id), + UNIQUE (nameHash) + ); + +-- create search index entity +CREATE TABLE IF NOT EXISTS search_index_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'), + PRIMARY KEY (id), + UNIQUE (fqnHash) + ); \ No newline at end of file diff --git a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql index 553a5fa1baf0..e544791a33a8 100644 --- a/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v014__create_db_connection_info.sql @@ -1,28 +1,4 @@ --- create domain entity table -CREATE TABLE IF NOT EXISTS domain_entity ( - id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, - name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, - fqnHash VARCHAR(256) NOT NULL, - json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, - updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, - PRIMARY KEY (id), - UNIQUE (fqnHash) -); - --- create data product entity table -CREATE TABLE IF NOT EXISTS data_product_entity ( - id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, - name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, - fqnHash VARCHAR(256) NOT NULL, - json JSONB NOT NULL, - updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, - updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, - PRIMARY KEY (id), - UNIQUE (fqnHash) -); - --- Rename includeTempTables in snowflake to includeTransientTables +-- Rename includeTempTables in snowflake to includeTransientTables UPDATE dbservice_entity SET json = jsonb_set(json::jsonb #- '{connection,config,includeTempTables}', '{connection,config,includeTransientTables}', diff --git a/bootstrap/sql/org.postgresql.Driver/v015__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v015__create_db_connection_info.sql index ca2ec2ac907b..8d8c24f4ceec 100644 --- a/bootstrap/sql/org.postgresql.Driver/v015__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v015__create_db_connection_info.sql @@ -1,3 +1,54 @@ -- column deleted not needed for entities that don't support soft delete ALTER TABLE query_entity DROP COLUMN deleted; ALTER TABLE event_subscription_entity DROP COLUMN deleted; + +-- create domain entity table +CREATE TABLE IF NOT EXISTS domain_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) + ); + +-- create data product entity table +CREATE TABLE IF NOT EXISTS data_product_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) + ); + +-- create search service entity +CREATE TABLE IF NOT EXISTS search_service_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + serviceType VARCHAR(256) GENERATED ALWAYS AS (json ->> 'serviceType') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, + PRIMARY KEY (id), + UNIQUE (nameHash) + ); + +-- create search index entity +CREATE TABLE IF NOT EXISTS search_index_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED, + PRIMARY KEY (id), + UNIQUE (fqnHash) + ); \ No newline at end of file diff --git a/ingestion/examples/sample_data/searchIndexes/searchIndexes.json b/ingestion/examples/sample_data/searchIndexes/searchIndexes.json new file mode 100644 index 000000000000..9c496d38ed6f --- /dev/null +++ b/ingestion/examples/sample_data/searchIndexes/searchIndexes.json @@ -0,0 +1,76 @@ +{ + "searchIndexes": [ + { + "id": "e093dd27-390e-4360-8efd-e4d63ec167a9", + "name": "table_search_index", + "displayName": "TableSearchIndex", + "fullyQualifiedName": "elasticsearch_sample.table_search_index", + "description": "Table Search Index", + "version": 0.1, + "updatedAt": 1638354087591, + "serviceType": "ElasticSearch", + "fields": [ + { + "name": "name", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Table Entity Name.", + "tags": [] + }, + { + "name": "displayName", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Table Entity DisplayName.", + "tags": [] + }, + { + "name": "description", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Table Entity Description.", + "tags": [] + }, + { + "name": "columns", + "dataType": "NESTED", + "dataTypeDisplay": "nested", + "description": "Table Columns.", + "tags": [], + "children": [ + { + "name": "name", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Column Name.", + "tags": [] + }, + { + "name": "displayName", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Column DisplayName.", + "tags": [] + }, + { + "name": "description", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Column Description.", + "tags": [] + } + ] + }, + { + "name": "databaseSchema", + "dataType": "TEXT", + "dataTypeDisplay": "text", + "description": "Database Schema that this table belongs to.", + "tags": [] + } + ], + "tags": [], + "followers": [] + } + ] +} \ No newline at end of file diff --git a/ingestion/examples/sample_data/searchIndexes/service.json b/ingestion/examples/sample_data/searchIndexes/service.json new file mode 100644 index 000000000000..82e9b21cc80b --- /dev/null +++ b/ingestion/examples/sample_data/searchIndexes/service.json @@ -0,0 +1,12 @@ +{ + "type": "elasticsearch", + "serviceName": "elasticsearch_sample", + "serviceConnection": { + "config": { + "type": "ElasticSearch", + "hostPort": "localhost:9200" + } + }, + "sourceConfig": { + } +} \ No newline at end of file diff --git a/ingestion/src/metadata/ingestion/api/parser.py b/ingestion/src/metadata/ingestion/api/parser.py index 3743a331257c..dfbc302c7f80 100644 --- a/ingestion/src/metadata/ingestion/api/parser.py +++ b/ingestion/src/metadata/ingestion/api/parser.py @@ -42,6 +42,10 @@ PipelineConnection, PipelineServiceType, ) +from metadata.generated.schema.entity.services.searchService import ( + SearchConnection, + SearchServiceType, +) from metadata.generated.schema.entity.services.storageService import ( StorageConnection, StorageServiceType, @@ -74,6 +78,10 @@ PipelineMetadataConfigType, PipelineServiceMetadataPipeline, ) +from metadata.generated.schema.metadataIngestion.searchServiceMetadataPipeline import ( + SearchMetadataConfigType, + SearchServiceMetadataPipeline, +) from metadata.generated.schema.metadataIngestion.storageServiceMetadataPipeline import ( StorageMetadataConfigType, StorageServiceMetadataPipeline, @@ -102,6 +110,7 @@ **{service: PipelineConnection for service in PipelineServiceType.__members__}, **{service: MlModelConnection for service in MlModelServiceType.__members__}, **{service: StorageConnection for service in StorageServiceType.__members__}, + **{service: SearchConnection for service in SearchServiceType.__members__}, } SOURCE_CONFIG_CLASS_MAP = { @@ -113,6 +122,7 @@ MlModelMetadataConfigType.MlModelMetadata.value: MlModelServiceMetadataPipeline, DatabaseMetadataConfigType.DatabaseMetadata.value: DatabaseServiceMetadataPipeline, StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline, + SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline, } diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 47a02733f1cb..c2207f26d3a2 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -49,6 +49,7 @@ from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.query import Query from metadata.generated.schema.entity.data.report import Report +from metadata.generated.schema.entity.data.searchIndex import SearchIndex from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.policies.policy import Policy @@ -67,6 +68,7 @@ from metadata.generated.schema.entity.services.metadataService import MetadataService from metadata.generated.schema.entity.services.mlmodelService import MlModelService from metadata.generated.schema.entity.services.pipelineService import PipelineService +from metadata.generated.schema.entity.services.searchService import SearchService from metadata.generated.schema.entity.services.storageService import StorageService from metadata.generated.schema.entity.teams.role import Role from metadata.generated.schema.entity.teams.team import Team @@ -358,6 +360,12 @@ def get_suffix(self, entity: Type[T]) -> str: # pylint: disable=R0911,R0912 ): return "/containers" + if issubclass( + entity, + get_args(Union[SearchIndex, self.get_create_entity_type(SearchIndex)]), + ): + return "/searchIndexes" + if issubclass( entity, get_args(Union[Workflow, self.get_create_entity_type(Workflow)]) ): @@ -422,11 +430,9 @@ def get_suffix(self, entity: Type[T]) -> str: # pylint: disable=R0911,R0912 if issubclass( entity, - get_args( - Union[StorageService, self.get_create_entity_type(StorageService)] - ), + get_args(Union[SearchService, self.get_create_entity_type(SearchService)]), ): - return "/services/storageServices" + return "/services/searchServices" if issubclass( entity, @@ -533,6 +539,7 @@ def get_entity_from_create(self, create: Type[C]) -> Type[T]: .replace("testsuite", "testSuite") .replace("testdefinition", "testDefinition") .replace("testcase", "testCase") + .replace("searchindex", "searchIndex") ) class_path = ".".join( diff --git a/ingestion/src/metadata/ingestion/ometa/utils.py b/ingestion/src/metadata/ingestion/ometa/utils.py index c5b1db7f52ae..43a8ee6085db 100644 --- a/ingestion/src/metadata/ingestion/ometa/utils.py +++ b/ingestion/src/metadata/ingestion/ometa/utils.py @@ -31,6 +31,7 @@ def format_name(name: str) -> str: return re.sub(r"[" + subs + "]", "_", name) +# pylint: disable=too-many-return-statements def get_entity_type( entity: Union[Type[T], str], ) -> str: @@ -54,6 +55,8 @@ def get_entity_type( return class_name.replace("testsuite", "testSuite") if "databaseschema" in class_name: return class_name.replace("databaseschema", "databaseSchema") + if "searchindex" in class_name: + return class_name.replace("searchindex", "searchIndex") return class_name diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 07cd406e4e3b..479289e9c156 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -34,6 +34,9 @@ ) from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest +from metadata.generated.schema.api.data.createSearchIndex import ( + CreateSearchIndexRequest, +) from metadata.generated.schema.api.data.createTable import CreateTableRequest from metadata.generated.schema.api.data.createTableProfile import ( CreateTableProfileRequest, @@ -77,6 +80,7 @@ from metadata.generated.schema.entity.services.messagingService import MessagingService from metadata.generated.schema.entity.services.mlmodelService import MlModelService from metadata.generated.schema.entity.services.pipelineService import PipelineService +from metadata.generated.schema.entity.services.searchService import SearchService from metadata.generated.schema.entity.services.storageService import StorageService from metadata.generated.schema.entity.teams.team import Team from metadata.generated.schema.entity.teams.user import User @@ -458,6 +462,34 @@ def __init__(self, config: WorkflowSource, metadata_config: OpenMetadataConnecti ) ) + self.storage_service_json = json.load( + open( # pylint: disable=consider-using-with + sample_data_folder + "/storage/service.json", + "r", + encoding=UTF_8, + ) + ) + + self.search_service_json = json.load( + open( # pylint: disable=consider-using-with + sample_data_folder + "/searchIndexes/service.json", + "r", + encoding=UTF_8, + ) + ) + self.search_service = self.metadata.get_service_or_create( + entity=SearchService, + config=WorkflowSource(**self.search_service_json), + ) + + self.search_indexes = json.load( + open( # pylint: disable=consider-using-with + sample_data_folder + "/searchIndexes/searchIndexes.json", + "r", + encoding=UTF_8, + ) + ) + @classmethod def create(cls, config_dict, metadata_config: OpenMetadataConnection): """Create class instance""" @@ -487,6 +519,7 @@ def next_record(self) -> Iterable[Entity]: yield from self.ingest_pipeline_status() yield from self.ingest_mlmodels() yield from self.ingest_containers() + yield from self.ingest_search_indexes() yield from self.ingest_profiles() yield from self.ingest_test_suite() yield from self.ingest_test_case() @@ -707,6 +740,30 @@ def ingest_topics(self) -> Iterable[CreateTopicRequest]: sample_data=TopicSampleData(messages=topic["sampleData"]), ) + def ingest_search_indexes(self) -> Iterable[CreateSearchIndexRequest]: + """ + Ingest Sample SearchIndexes + """ + for search_index in self.search_indexes["searchIndexes"]: + search_index["service"] = EntityReference( + id=self.search_service.id, type="searchService" + ) + create_search_index = CreateSearchIndexRequest( + name=search_index["name"], + description=search_index["description"], + displayName=search_index["displayName"], + tags=search_index["tags"], + fields=search_index["fields"], + service=self.search_service.fullyQualifiedName, + ) + + self.status.scanned( + f"SearchIndex Scanned: {create_search_index.name.__root__}" + ) + yield create_search_index + + # TODO: Add search index sample data + def ingest_looker(self) -> Iterable[Entity]: """ Looker sample data diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index c5773c73e402..3183ebc12904 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -86,6 +86,7 @@ public final class Entity { public static final String STORAGE_SERVICE = "storageService"; public static final String MLMODEL_SERVICE = "mlmodelService"; public static final String METADATA_SERVICE = "metadataService"; + public static final String SEARCH_SERVICE = "searchService"; // // Data asset entities // @@ -99,6 +100,7 @@ public final class Entity { public static final String CHART = "chart"; public static final String REPORT = "report"; public static final String TOPIC = "topic"; + public static final String SEARCH_INDEX = "searchIndex"; public static final String MLMODEL = "mlmodel"; public static final String CONTAINER = "container"; public static final String QUERY = "query"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 8a8ec5b32dd8..d4d9edb98c53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -79,6 +79,7 @@ import org.openmetadata.schema.entity.data.Pipeline; import org.openmetadata.schema.entity.data.Query; import org.openmetadata.schema.entity.data.Report; +import org.openmetadata.schema.entity.data.SearchIndex; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.domains.DataProduct; @@ -91,6 +92,7 @@ import org.openmetadata.schema.entity.services.MetadataService; import org.openmetadata.schema.entity.services.MlModelService; import org.openmetadata.schema.entity.services.PipelineService; +import org.openmetadata.schema.entity.services.SearchService; import org.openmetadata.schema.entity.services.StorageService; import org.openmetadata.schema.entity.services.connections.TestConnectionDefinition; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; @@ -188,6 +190,9 @@ public interface CollectionDAO { @CreateSqlObject MlModelDAO mlModelDAO(); + @CreateSqlObject + SearchIndexDAO searchIndexDAO(); + @CreateSqlObject GlossaryDAO glossaryDAO(); @@ -233,6 +238,9 @@ public interface CollectionDAO { @CreateSqlObject StorageServiceDAO storageServiceDAO(); + @CreateSqlObject + SearchServiceDAO searchServiceDAO(); + @CreateSqlObject ContainerDAO containerDAO(); @@ -531,6 +539,40 @@ int listCount( @Define("sqlCondition") String mysqlCond); } + interface SearchServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "search_service_entity"; + } + + @Override + default Class getEntityClass() { + return SearchService.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } + + interface SearchIndexDAO extends EntityDAO { + @Override + default String getTableName() { + return "search_index_entity"; + } + + @Override + default Class getEntityClass() { + return SearchIndex.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + interface EntityExtensionDAO { @ConnectionAwareSqlUpdate( value = @@ -3397,6 +3439,7 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM pipeline_entity ) as pipelineCount, " + "(SELECT COUNT(*) FROM ml_model_entity ) as mlmodelCount, " + "(SELECT COUNT(*) FROM storage_container_entity ) as storageContainerCount, " + + "(SELECT COUNT(*) FROM search_index_entity ) as searchIndexCount, " + "(SELECT COUNT(*) FROM glossary_entity ) as glossaryCount, " + "(SELECT COUNT(*) FROM glossary_term_entity ) as glossaryTermCount, " + "(SELECT (SELECT COUNT(*) FROM metadata_service_entity ) + " @@ -3405,6 +3448,7 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM dashboard_service_entity )+ " + "(SELECT COUNT(*) FROM pipeline_service_entity )+ " + "(SELECT COUNT(*) FROM mlmodel_service_entity )+ " + + "(SELECT COUNT(*) FROM search_service_entity )+ " + "(SELECT COUNT(*) FROM storage_service_entity )) as servicesCount, " + "(SELECT COUNT(*) FROM user_entity AND (JSON_EXTRACT(json, '$.isBot') IS NULL OR JSON_EXTRACT(json, '$.isBot') = FALSE)) as userCount, " + "(SELECT COUNT(*) FROM team_entity ) as teamCount, " @@ -3418,6 +3462,7 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM pipeline_entity ) as pipelineCount, " + "(SELECT COUNT(*) FROM ml_model_entity ) as mlmodelCount, " + "(SELECT COUNT(*) FROM storage_container_entity ) as storageContainerCount, " + + "(SELECT COUNT(*) FROM search_index_entity ) as searchIndexCount, " + "(SELECT COUNT(*) FROM glossary_entity ) as glossaryCount, " + "(SELECT COUNT(*) FROM glossary_term_entity ) as glossaryTermCount, " + "(SELECT (SELECT COUNT(*) FROM metadata_service_entity ) + " @@ -3426,6 +3471,7 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM dashboard_service_entity )+ " + "(SELECT COUNT(*) FROM pipeline_service_entity )+ " + "(SELECT COUNT(*) FROM mlmodel_service_entity )+ " + + "(SELECT COUNT(*) FROM search_service_entity )+ " + "(SELECT COUNT(*) FROM storage_service_entity )) as servicesCount, " + "(SELECT COUNT(*) FROM user_entity AND (json#>'{isBot}' IS NULL OR ((json#>'{isBot}')::boolean) = FALSE)) as userCount, " + "(SELECT COUNT(*) FROM team_entity ) as teamCount, " @@ -3440,7 +3486,8 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM dashboard_service_entity ) as dashboardServiceCount, " + "(SELECT COUNT(*) FROM pipeline_service_entity ) as pipelineServiceCount, " + "(SELECT COUNT(*) FROM mlmodel_service_entity ) as mlModelServiceCount, " - + "(SELECT COUNT(*) FROM storage_service_entity ) as storageServiceCount") + + "(SELECT COUNT(*) FROM storage_service_entity ) as storageServiceCount, " + + "(SELECT COUNT(*) FROM search_service_entity ) as searchServiceCount") @RegisterRowMapper(ServicesCountRowMapper.class) ServicesCount getAggregatedServicesCount(@Define("cond") String cond) throws StatementException; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java new file mode 100644 index 000000000000..ad56a704802a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java @@ -0,0 +1,459 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; +import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; +import static org.openmetadata.service.Entity.FIELD_DOMAIN; +import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; +import static org.openmetadata.service.Entity.FIELD_TAGS; +import static org.openmetadata.service.Entity.SEARCH_SERVICE; +import static org.openmetadata.service.util.EntityUtil.getSearchIndexField; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.json.JsonPatch; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.entity.data.SearchIndex; +import org.openmetadata.schema.entity.services.SearchService; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.SearchIndexField; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.TaskDetails; +import org.openmetadata.schema.type.searchindex.SearchIndexSampleData; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.resources.searchindex.SearchIndexResource; +import org.openmetadata.service.security.mask.PIIMasker; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.JsonUtils; + +public class SearchIndexRepository extends EntityRepository { + @Override + public void setFullyQualifiedName(SearchIndex searchIndex) { + searchIndex.setFullyQualifiedName( + FullyQualifiedName.add(searchIndex.getService().getFullyQualifiedName(), searchIndex.getName())); + if (searchIndex.getFields() != null) { + setFieldFQN(searchIndex.getFullyQualifiedName(), searchIndex.getFields()); + } + } + + public SearchIndexRepository(CollectionDAO dao) { + super( + SearchIndexResource.COLLECTION_PATH, Entity.SEARCH_INDEX, SearchIndex.class, dao.searchIndexDAO(), dao, "", ""); + } + + @Override + public void prepare(SearchIndex searchIndex) { + SearchService searchService = Entity.getEntity(searchIndex.getService(), "", ALL); + searchIndex.setService(searchService.getEntityReference()); + searchIndex.setServiceType(searchService.getServiceType()); + // Validate field tags + if (searchIndex.getFields() != null) { + addDerivedFieldTags(searchIndex.getFields()); + validateSchemaFieldTags(searchIndex.getFields()); + } + } + + @Override + public void storeEntity(SearchIndex searchIndex, boolean update) { + // Relationships and fields such as service are derived and not stored as part of json + EntityReference service = searchIndex.getService(); + searchIndex.withService(null); + + // Don't store fields tags as JSON but build it on the fly based on relationships + List fieldsWithTags = null; + if (searchIndex.getFields() != null) { + fieldsWithTags = searchIndex.getFields(); + searchIndex.setFields(cloneWithoutTags(fieldsWithTags)); + searchIndex.getFields().forEach(field -> field.setTags(null)); + } + + store(searchIndex, update); + + // Restore the relationships + if (fieldsWithTags != null) { + searchIndex.setFields(fieldsWithTags); + } + searchIndex.withService(service); + } + + @Override + public void storeRelationships(SearchIndex searchIndex) { + setService(searchIndex, searchIndex.getService()); + } + + @Override + public SearchIndex setInheritedFields(SearchIndex searchIndex, Fields fields) { + // If searchIndex does not have domain, then inherit it from parent messaging service + if (fields.contains(FIELD_DOMAIN) && searchIndex.getDomain() == null) { + SearchService service = Entity.getEntity(SEARCH_SERVICE, searchIndex.getService().getId(), "domain", ALL); + searchIndex.withDomain(service.getDomain()); + } + return searchIndex; + } + + @Override + public SearchIndex setFields(SearchIndex searchIndex, Fields fields) { + searchIndex.setService(getContainer(searchIndex.getId())); + searchIndex.setFollowers(fields.contains(FIELD_FOLLOWERS) ? getFollowers(searchIndex) : null); + if (searchIndex.getFields() != null) { + getFieldTags(fields.contains(FIELD_TAGS), searchIndex.getFields()); + } + return searchIndex; + } + + @Override + public SearchIndex clearFields(SearchIndex searchIndex, Fields fields) { + return searchIndex; + } + + @Override + public SearchIndexUpdater getUpdater(SearchIndex original, SearchIndex updated, Operation operation) { + return new SearchIndexUpdater(original, updated, operation); + } + + public void setService(SearchIndex searchIndex, EntityReference service) { + if (service != null && searchIndex != null) { + addRelationship( + service.getId(), searchIndex.getId(), service.getType(), Entity.SEARCH_INDEX, Relationship.CONTAINS); + searchIndex.setService(service); + } + } + + public SearchIndex getSampleData(UUID searchIndexId, boolean authorizePII) { + // Validate the request content + SearchIndex searchIndex = dao.findEntityById(searchIndexId); + + SearchIndexSampleData sampleData = + JsonUtils.readValue( + daoCollection.entityExtensionDAO().getExtension(searchIndex.getId().toString(), "searchIndex.sampleData"), + SearchIndexSampleData.class); + searchIndex.setSampleData(sampleData); + setFieldsInternal(searchIndex, Fields.EMPTY_FIELDS); + + // Set the fields tags. Will be used to mask the sample data + if (!authorizePII) { + getFieldTags(true, searchIndex.getFields()); + searchIndex.setTags(getTags(searchIndex.getFullyQualifiedName())); + return PIIMasker.getSampleData(searchIndex); + } + + return searchIndex; + } + + @Transaction + public SearchIndex addSampleData(UUID searchIndexId, SearchIndexSampleData sampleData) { + // Validate the request content + SearchIndex searchIndex = daoCollection.searchIndexDAO().findEntityById(searchIndexId); + + daoCollection + .entityExtensionDAO() + .insert( + searchIndexId.toString(), + "searchIndex.sampleData", + "searchIndexSampleData", + JsonUtils.pojoToJson(sampleData)); + setFieldsInternal(searchIndex, Fields.EMPTY_FIELDS); + return searchIndex.withSampleData(sampleData); + } + + private void setFieldFQN(String parentFQN, List fields) { + fields.forEach( + c -> { + String fieldFqn = FullyQualifiedName.add(parentFQN, c.getName()); + c.setFullyQualifiedName(fieldFqn); + if (c.getChildren() != null) { + setFieldFQN(fieldFqn, c.getChildren()); + } + }); + } + + private void getFieldTags(boolean setTags, List fields) { + for (SearchIndexField f : listOrEmpty(fields)) { + f.setTags(setTags ? getTags(f.getFullyQualifiedName()) : null); + getFieldTags(setTags, f.getChildren()); + } + } + + private void addDerivedFieldTags(List fields) { + if (nullOrEmpty(fields)) { + return; + } + + for (SearchIndexField field : fields) { + field.setTags(addDerivedTags(field.getTags())); + if (field.getChildren() != null) { + addDerivedFieldTags(field.getChildren()); + } + } + } + + List cloneWithoutTags(List fields) { + if (nullOrEmpty(fields)) { + return fields; + } + List copy = new ArrayList<>(); + fields.forEach(f -> copy.add(cloneWithoutTags(f))); + return copy; + } + + private SearchIndexField cloneWithoutTags(SearchIndexField field) { + List children = cloneWithoutTags(field.getChildren()); + return new SearchIndexField() + .withDescription(field.getDescription()) + .withName(field.getName()) + .withDisplayName(field.getDisplayName()) + .withFullyQualifiedName(field.getFullyQualifiedName()) + .withDataType(field.getDataType()) + .withDataTypeDisplay(field.getDataTypeDisplay()) + .withChildren(children); + } + + private void validateSchemaFieldTags(List fields) { + // Add field level tags by adding tag to field relationship + for (SearchIndexField field : fields) { + checkMutuallyExclusive(field.getTags()); + if (field.getChildren() != null) { + validateSchemaFieldTags(field.getChildren()); + } + } + } + + private void applyTags(List fields) { + // Add field level tags by adding tag to field relationship + for (SearchIndexField field : fields) { + applyTags(field.getTags(), field.getFullyQualifiedName()); + if (field.getChildren() != null) { + applyTags(field.getChildren()); + } + } + } + + @Override + public void applyTags(SearchIndex searchIndex) { + // Add table level tags by adding tag to table relationship + super.applyTags(searchIndex); + if (searchIndex.getFields() != null) { + applyTags(searchIndex.getFields()); + } + } + + @Override + public List getAllTags(EntityInterface entity) { + List allTags = new ArrayList<>(); + SearchIndex searchIndex = (SearchIndex) entity; + EntityUtil.mergeTags(allTags, searchIndex.getTags()); + List schemaFields = searchIndex.getFields() != null ? searchIndex.getFields() : null; + for (SearchIndexField schemaField : listOrEmpty(schemaFields)) { + EntityUtil.mergeTags(allTags, schemaField.getTags()); + } + return allTags; + } + + @Override + public void update(TaskDetails task, MessageParser.EntityLink entityLink, String newValue, String user) { + if (entityLink.getFieldName().equals("fields")) { + String schemaName = entityLink.getArrayFieldName(); + String childrenSchemaName = ""; + if (entityLink.getArrayFieldName().contains(".")) { + String fieldNameWithoutQuotes = + entityLink.getArrayFieldName().substring(1, entityLink.getArrayFieldName().length() - 1); + schemaName = fieldNameWithoutQuotes.substring(0, fieldNameWithoutQuotes.indexOf(".")); + childrenSchemaName = fieldNameWithoutQuotes.substring(fieldNameWithoutQuotes.lastIndexOf(".") + 1); + } + SearchIndex searchIndex = getByName(null, entityLink.getEntityFQN(), getFields("tags"), ALL, false); + SearchIndexField schemaField = null; + for (SearchIndexField field : searchIndex.getFields()) { + if (field.getName().equals(schemaName)) { + schemaField = field; + break; + } + } + if (!"".equals(childrenSchemaName) && schemaField != null) { + schemaField = getChildrenSchemaField(schemaField.getChildren(), childrenSchemaName); + } + if (schemaField == null) { + throw new IllegalArgumentException( + CatalogExceptionMessage.invalidFieldName("schema", entityLink.getArrayFieldName())); + } + + String origJson = JsonUtils.pojoToJson(searchIndex); + if (EntityUtil.isDescriptionTask(task.getType())) { + schemaField.setDescription(newValue); + } else if (EntityUtil.isTagTask(task.getType())) { + List tags = JsonUtils.readObjects(newValue, TagLabel.class); + schemaField.setTags(tags); + } + String updatedEntityJson = JsonUtils.pojoToJson(searchIndex); + JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); + patch(null, searchIndex.getId(), user, patch); + return; + } + super.update(task, entityLink, newValue, user); + } + + private static SearchIndexField getChildrenSchemaField(List fields, String childrenSchemaName) { + SearchIndexField childrenSchemaField = null; + for (SearchIndexField field : fields) { + if (field.getName().equals(childrenSchemaName)) { + childrenSchemaField = field; + break; + } + } + if (childrenSchemaField == null) { + for (SearchIndexField field : fields) { + if (field.getChildren() != null) { + childrenSchemaField = getChildrenSchemaField(field.getChildren(), childrenSchemaName); + if (childrenSchemaField != null) { + break; + } + } + } + } + return childrenSchemaField; + } + + public static Set getAllFieldTags(SearchIndexField field) { + Set tags = new HashSet<>(); + if (!listOrEmpty(field.getTags()).isEmpty()) { + tags.addAll(field.getTags()); + } + for (SearchIndexField c : listOrEmpty(field.getChildren())) { + tags.addAll(getAllFieldTags(c)); + } + return tags; + } + + public class SearchIndexUpdater extends EntityUpdater { + public static final String FIELD_DATA_TYPE_DISPLAY = "dataTypeDisplay"; + + public SearchIndexUpdater(SearchIndex original, SearchIndex updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate() { + if (updated.getFields() != null) { + updateSearchIndexFields( + "fields", + original.getFields() == null ? null : original.getFields(), + updated.getFields(), + EntityUtil.searchIndexFieldMatch); + } + recordChange("searchIndexSettings", original.getSearchIndexSettings(), updated.getSearchIndexSettings()); + } + + private void updateSearchIndexFields( + String fieldName, + List origFields, + List updatedFields, + BiPredicate fieldMatch) { + List deletedFields = new ArrayList<>(); + List addedFields = new ArrayList<>(); + recordListChange(fieldName, origFields, updatedFields, addedFields, deletedFields, fieldMatch); + // carry forward tags and description if deletedFields matches added field + Map addedFieldMap = + addedFields.stream().collect(Collectors.toMap(SearchIndexField::getName, Function.identity())); + + for (SearchIndexField deleted : deletedFields) { + if (addedFieldMap.containsKey(deleted.getName())) { + SearchIndexField addedField = addedFieldMap.get(deleted.getName()); + if (nullOrEmpty(addedField.getDescription()) && nullOrEmpty(deleted.getDescription())) { + addedField.setDescription(deleted.getDescription()); + } + if (nullOrEmpty(addedField.getTags()) && nullOrEmpty(deleted.getTags())) { + addedField.setTags(deleted.getTags()); + } + } + } + + // Delete tags related to deleted fields + deletedFields.forEach(deleted -> daoCollection.tagUsageDAO().deleteTagsByTarget(deleted.getFullyQualifiedName())); + + // Add tags related to newly added fields + for (SearchIndexField added : addedFields) { + applyTags(added.getTags(), added.getFullyQualifiedName()); + } + + // Carry forward the user generated metadata from existing fields to new fields + for (SearchIndexField updated : updatedFields) { + // Find stored field matching name, data type and ordinal position + SearchIndexField stored = origFields.stream().filter(c -> fieldMatch.test(c, updated)).findAny().orElse(null); + if (stored == null) { // New field added + continue; + } + updateFieldDescription(stored, updated); + updateFieldDataTypeDisplay(stored, updated); + updateFieldDisplayName(stored, updated); + updateTags( + stored.getFullyQualifiedName(), + EntityUtil.getFieldName(fieldName, updated.getName(), FIELD_TAGS), + stored.getTags(), + updated.getTags()); + + if (updated.getChildren() != null && stored.getChildren() != null) { + String childrenFieldName = EntityUtil.getFieldName(fieldName, updated.getName()); + updateSearchIndexFields(childrenFieldName, stored.getChildren(), updated.getChildren(), fieldMatch); + } + } + majorVersionChange = majorVersionChange || !deletedFields.isEmpty(); + } + + private void updateFieldDescription(SearchIndexField origField, SearchIndexField updatedField) { + if (operation.isPut() && !nullOrEmpty(origField.getDescription()) && updatedByBot()) { + // Revert the non-empty field description if being updated by a bot + updatedField.setDescription(origField.getDescription()); + return; + } + String field = getSearchIndexField(original, origField, FIELD_DESCRIPTION); + recordChange(field, origField.getDescription(), updatedField.getDescription()); + } + + private void updateFieldDisplayName(SearchIndexField origField, SearchIndexField updatedField) { + if (operation.isPut() && !nullOrEmpty(origField.getDescription()) && updatedByBot()) { + // Revert the non-empty field description if being updated by a bot + updatedField.setDisplayName(origField.getDisplayName()); + return; + } + String field = getSearchIndexField(original, origField, FIELD_DISPLAY_NAME); + recordChange(field, origField.getDisplayName(), updatedField.getDisplayName()); + } + + private void updateFieldDataTypeDisplay(SearchIndexField origField, SearchIndexField updatedField) { + if (operation.isPut() && !nullOrEmpty(origField.getDataTypeDisplay()) && updatedByBot()) { + // Revert the non-empty field dataTypeDisplay if being updated by a bot + updatedField.setDataTypeDisplay(origField.getDataTypeDisplay()); + return; + } + String field = getSearchIndexField(original, origField, FIELD_DATA_TYPE_DISPLAY); + recordChange(field, origField.getDataTypeDisplay(), updatedField.getDataTypeDisplay()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchServiceRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchServiceRepository.java new file mode 100644 index 000000000000..2cbcba49127c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchServiceRepository.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.jdbi3; + +import org.openmetadata.schema.entity.services.SearchService; +import org.openmetadata.schema.entity.services.ServiceType; +import org.openmetadata.schema.type.SearchConnection; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.services.storage.StorageServiceResource; + +public class SearchServiceRepository extends ServiceEntityRepository { + public SearchServiceRepository(CollectionDAO dao) { + super( + StorageServiceResource.COLLECTION_PATH, + Entity.SEARCH_SERVICE, + dao, + dao.searchServiceDAO(), + SearchConnection.class, + ServiceType.SEARCH); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java new file mode 100644 index 000000000000..2f1329962855 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java @@ -0,0 +1,480 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.resources.searchindex; + +import static org.openmetadata.common.utils.CommonUtil.listOf; + +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.List; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.api.data.CreateSearchIndex; +import org.openmetadata.schema.api.data.RestoreEntity; +import org.openmetadata.schema.entity.data.SearchIndex; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.searchindex.SearchIndexSampleData; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.SearchIndexRepository; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.util.ResultList; + +@Path("/v1/searchIndexes") +@Tag( + name = "SearchIndex", + description = "A `SearchIndex` is a index mapping for indexing documents in a `Search Service`.") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "searchIndexes") +public class SearchIndexResource extends EntityResource { + public static final String COLLECTION_PATH = "v1/searchIndexes/"; + static final String FIELDS = "owner,followers,tags,extension,domain,dataProducts"; + + @Override + public SearchIndex addHref(UriInfo uriInfo, SearchIndex searchIndex) { + Entity.withHref(uriInfo, searchIndex.getOwner()); + Entity.withHref(uriInfo, searchIndex.getService()); + Entity.withHref(uriInfo, searchIndex.getFollowers()); + Entity.withHref(uriInfo, searchIndex.getDomain()); + return searchIndex; + } + + public SearchIndexResource(CollectionDAO dao, Authorizer authorizer) { + super(SearchIndex.class, new SearchIndexRepository(dao), authorizer); + } + + @Override + protected List getEntitySpecificOperations() { + addViewOperation("sampleData", MetadataOperation.VIEW_SAMPLE_DATA); + return listOf(MetadataOperation.VIEW_SAMPLE_DATA, MetadataOperation.EDIT_SAMPLE_DATA); + } + + public static class SearchIndexList extends ResultList { + /* Required for serde */ + } + + @GET + @Operation( + operationId = "listSearchIndexes", + summary = "List searchIndexes", + description = + "Get a list of SearchIndexes, optionally filtered by `service` it belongs to. Use `fields` " + + "parameter to get only necessary fields. Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of SearchIndexes", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndexList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Filter SearchIndexes by service name", + schema = @Schema(type = "string", example = "ElasticSearchWestCoast")) + @QueryParam("service") + String serviceParam, + @Parameter(description = "Limit the number SearchIndexes returned. (1 to 1000000, default = " + "10)") + @DefaultValue("10") + @QueryParam("limit") + @Min(0) + @Max(1000000) + int limitParam, + @Parameter(description = "Returns list of SearchIndexes before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of SearchIndexes after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + ListFilter filter = new ListFilter(include).addQueryParam("service", serviceParam); + return super.listInternal(uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listAllSearchIndexVersion", + summary = "List SearchIndex versions", + description = "Get a list of all the versions of a SearchIndex identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of SearchIndex versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { + return super.listVersionsInternal(securityContext, id); + } + + @GET + @Path("/{id}") + @Operation( + summary = "Get a SearchIndex by id", + description = "Get a SearchIndex by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The SearchIndex", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))), + @ApiResponse(responseCode = "404", description = "SearchIndex for instance {id} is not found") + }) + public SearchIndex get( + @Context UriInfo uriInfo, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @GET + @Path("/name/{fqn}") + @Operation( + operationId = "getSearchIndexByFQN", + summary = "Get a SearchIndex by fully qualified name", + description = "Get a SearchIndex by fully qualified name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The SearchIndex", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))), + @ApiResponse(responseCode = "404", description = "SearchIndex for instance {fqn} is not found") + }) + public SearchIndex getByName( + @Context UriInfo uriInfo, + @Parameter(description = "Fully qualified name of the SearchIndex", schema = @Schema(type = "string")) + @PathParam("fqn") + String fqn, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "getSpecificSearchIndexVersion", + summary = "Get a version of the SearchIndex", + description = "Get a version of the SearchIndex by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "SearchIndex", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))), + @ApiResponse( + responseCode = "404", + description = "SearchIndex for instance {id} and version {version} is " + "not found") + }) + public SearchIndex getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter( + description = "SearchIndex version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) { + return super.getVersionInternal(securityContext, id, version); + } + + @POST + @Operation( + operationId = "createSearchIndex", + summary = "Create a SearchIndex", + description = "Create a SearchIndex under an existing `service`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The SearchIndex", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchIndex create) { + SearchIndex searchIndex = getSearchIndex(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, searchIndex); + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchSearchIndex", + summary = "Update a SearchIndex", + description = "Update an existing SearchIndex using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateDescription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @PUT + @Operation( + operationId = "createOrUpdateSearchIndex", + summary = "Update SearchIndex", + description = "Create a SearchIndex, it it does not exist or update an existing SearchIndex.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The updated SearchIndex ", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))) + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchIndex create) { + SearchIndex searchIndex = getSearchIndex(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, searchIndex); + } + + @PUT + @Path("/{id}/sampleData") + @Operation( + operationId = "addSampleData", + summary = "Add sample data", + description = "Add sample data to the searchIndex.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The SearchIndex", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))), + }) + public SearchIndex addSampleData( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Valid SearchIndexSampleData sampleData) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.EDIT_SAMPLE_DATA); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + SearchIndex searchIndex = repository.addSampleData(id, sampleData); + return addHref(uriInfo, searchIndex); + } + + @GET + @Path("/{id}/sampleData") + @Operation( + operationId = "getSampleData", + summary = "Get sample data", + description = "Get sample data from the SearchIndex.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully obtained the SampleData for SearchIndex", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))) + }) + public SearchIndex getSampleData( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_SAMPLE_DATA); + ResourceContext resourceContext = getResourceContextById(id); + authorizer.authorize(securityContext, operationContext, resourceContext); + boolean authorizePII = authorizer.authorizePII(securityContext, resourceContext.getOwner()); + + SearchIndex searchIndex = repository.getSampleData(id, authorizePII); + return addHref(uriInfo, searchIndex); + } + + @PUT + @Path("/{id}/followers") + @Operation( + operationId = "addFollower", + summary = "Add a follower", + description = "Add a user identified by `userId` as followed of this SearchIndex", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))), + @ApiResponse(responseCode = "404", description = "SearchIndex for instance {id} is not found") + }) + public Response addFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter(description = "Id of the user to be added as follower", schema = @Schema(type = "UUID")) UUID userId) { + return repository.addFollower(securityContext.getUserPrincipal().getName(), id, userId).toResponse(); + } + + @DELETE + @Path("/{id}/followers/{userId}") + @Operation( + summary = "Remove a follower", + description = "Remove the user identified `userId` as a follower of the SearchIndex.", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))) + }) + public Response deleteFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter(description = "Id of the user being removed as follower", schema = @Schema(type = "string")) + @PathParam("userId") + String userId) { + return repository + .deleteFollower(securityContext.getUserPrincipal().getName(), id, UUID.fromString(userId)) + .toResponse(); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteSearchIndex", + summary = "Delete a SearchIndex by id", + description = "Delete a SearchIndex by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "SearchIndex for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Id of the SearchIndex", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { + return delete(uriInfo, securityContext, id, false, hardDelete); + } + + @DELETE + @Path("/name/{fqn}") + @Operation( + operationId = "deleteSearchIndexByFQN", + summary = "Delete a SearchIndex by fully qualified name", + description = "Delete a SearchIndex by `fullyQualifiedName`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "SearchIndex for instance {fqn} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Fully qualified name of the SearchIndex", schema = @Schema(type = "string")) + @PathParam("fqn") + String fqn) { + return deleteByName(uriInfo, securityContext, fqn, false, hardDelete); + } + + @PUT + @Path("/restore") + @Operation( + operationId = "restore", + summary = "Restore a soft deleted SearchIndex", + description = "Restore a soft deleted SearchIndex.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully restored the SearchIndex. ", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchIndex.class))) + }) + public Response restoreSearchIndex( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore) { + return restoreEntity(uriInfo, securityContext, restore.getId()); + } + + private SearchIndex getSearchIndex(CreateSearchIndex create, String user) { + return copy(new SearchIndex(), create, user) + .withService(getEntityReference(Entity.SEARCH_SERVICE, create.getService())) + .withFields(create.getFields()) + .withSearchIndexSettings(create.getSearchIndexSettings()) + .withTags(create.getTags()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java new file mode 100644 index 000000000000..5252d0e1706b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java @@ -0,0 +1,436 @@ +package org.openmetadata.service.resources.services.searchIndexes; + +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import javax.json.JsonPatch; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.PATCH; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.data.RestoreEntity; +import org.openmetadata.schema.api.services.CreateSearchService; +import org.openmetadata.schema.entity.services.SearchService; +import org.openmetadata.schema.entity.services.ServiceType; +import org.openmetadata.schema.entity.services.connections.TestConnectionResult; +import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.SearchConnection; +import org.openmetadata.schema.utils.EntityInterfaceUtil; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.SearchServiceRepository; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.services.ServiceEntityResource; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; + +@Slf4j +@Path("/v1/services/searchServices") +@Tag( + name = "Search Services", + description = "APIs related `Search Service` entities, such as ElasticSearch, OpenSearch.") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "searchServices") +public class SearchServiceResource + extends ServiceEntityResource { + public static final String COLLECTION_PATH = "v1/services/searchServices/"; + static final String FIELDS = "pipelines,owner,tags,domain"; + + @Override + public SearchService addHref(UriInfo uriInfo, SearchService service) { + service.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, service.getId())); + Entity.withHref(uriInfo, service.getOwner()); + Entity.withHref(uriInfo, service.getPipelines()); + return service; + } + + public SearchServiceResource(CollectionDAO dao, Authorizer authorizer) { + super(SearchService.class, new SearchServiceRepository(dao), authorizer, ServiceType.SEARCH); + } + + @Override + protected List getEntitySpecificOperations() { + addViewOperation("pipelines", MetadataOperation.VIEW_BASIC); + return null; + } + + public static class SearchServiceList extends ResultList { + /* Required for serde */ + } + + @GET + @Operation( + operationId = "listSearchServices", + summary = "List search services", + description = "Get a list of search services.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of search service instances", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SearchServiceResource.SearchServiceList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam, + @Parameter(description = "Returns list of search services before this cursor", schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter(description = "Returns list of search services after this cursor", schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + RestUtil.validateCursors(before, after); + EntityUtil.Fields fields = getFields(fieldsParam); + ResultList searchServices; + + ListFilter filter = new ListFilter(include); + if (before != null) { + searchServices = repository.listBefore(uriInfo, fields, filter, limitParam, before); + } else { + searchServices = repository.listAfter(uriInfo, fields, filter, limitParam, after); + } + return addHref(uriInfo, decryptOrNullify(securityContext, searchServices)); + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getSearchServiceByID", + summary = "Get an search service", + description = "Get an search service by `id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "search service instance", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))), + @ApiResponse(responseCode = "404", description = "search service for instance {id} is not found") + }) + public SearchService get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + SearchService searchService = getInternal(uriInfo, securityContext, id, fieldsParam, include); + return decryptOrNullify(securityContext, searchService); + } + + @GET + @Path("/name/{name}") + @Operation( + operationId = "getSearchServiceByFQN", + summary = "Get search service by name", + description = "Get a search service by the service `name`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "search service instance", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))), + @ApiResponse(responseCode = "404", description = "search service for instance {id} is not found") + }) + public SearchService getByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("name") String name, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + SearchService searchService = + getByNameInternal(uriInfo, securityContext, EntityInterfaceUtil.quoteName(name), fieldsParam, include); + return decryptOrNullify(securityContext, searchService); + } + + @PUT + @Path("/{id}/testConnectionResult") + @Operation( + operationId = "addTestConnectionResult", + summary = "Add test connection result", + description = "Add test connection result to the service.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully updated the service", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))) + }) + public SearchService addTestConnectionResult( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the service", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Valid TestConnectionResult testConnectionResult) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.CREATE); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + SearchService service = repository.addTestConnectionResult(id, testConnectionResult); + return decryptOrNullify(securityContext, service); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listAllSearchServiceVersion", + summary = "List search service versions", + description = "Get a list of all the versions of an search service identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of search service versions", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "search service Id", schema = @Schema(type = "string")) @PathParam("id") UUID id) { + EntityHistory entityHistory = super.listVersionsInternal(securityContext, id); + + List versions = + entityHistory.getVersions().stream() + .map( + json -> { + try { + SearchService searchService = JsonUtils.readValue((String) json, SearchService.class); + return JsonUtils.pojoToJson(decryptOrNullify(securityContext, searchService)); + } catch (Exception e) { + return json; + } + }) + .collect(Collectors.toList()); + entityHistory.setVersions(versions); + return entityHistory; + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "getSpecificSearchServiceVersion", + summary = "Get a version of the search service", + description = "Get a version of the search service by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "search service", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))), + @ApiResponse( + responseCode = "404", + description = "Object store service for instance {id} and version " + "{version} is not found") + }) + public SearchService getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "search service Id", schema = @Schema(type = "string")) @PathParam("id") UUID id, + @Parameter( + description = "search service version number in the form `major`" + ".`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) { + SearchService searchService = super.getVersionInternal(securityContext, id, version); + return decryptOrNullify(securityContext, searchService); + } + + @POST + @Operation( + operationId = "createSearchService", + summary = "Create search service", + description = "Create a new search service.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Search service instance", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchService create) { + SearchService service = getService(create, securityContext.getUserPrincipal().getName()); + Response response = create(uriInfo, securityContext, service); + decryptOrNullify(securityContext, (SearchService) response.getEntity()); + return response; + } + + @PUT + @Operation( + operationId = "createOrUpdateSearchService", + summary = "Update search service", + description = "Update an existing or create a new search service.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Object store service instance", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchService update) { + SearchService service = getService(update, securityContext.getUserPrincipal().getName()); + Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); + decryptOrNullify(securityContext, (SearchService) response.getEntity()); + return response; + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchSearchService", + summary = "Update an search service", + description = "Update an existing search service using JsonPatch.", + externalDocs = @ExternalDocumentation(description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response patch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject("[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteSearchService", + summary = "Delete an search service", + description = "Delete an search services. If containers belong the service, it can't be " + "deleted.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "SearchService service for instance {id} " + "is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Recursively delete this entity and it's children. (Default `false`)") + @DefaultValue("false") + @QueryParam("recursive") + boolean recursive, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Id of the search service", schema = @Schema(type = "string")) @PathParam("id") + UUID id) { + return delete(uriInfo, securityContext, id, recursive, hardDelete); + } + + @DELETE + @Path("/name/{fqn}") + @Operation( + operationId = "deleteSearchServiceByFQN", + summary = "Delete an SearchService by fully qualified name", + description = "Delete an SearchService by `fullyQualifiedName`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "SearchService for instance {fqn} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Name of the SearchService", schema = @Schema(type = "string")) @PathParam("fqn") + String fqn) { + return deleteByName(uriInfo, securityContext, EntityInterfaceUtil.quoteName(fqn), false, hardDelete); + } + + @PUT + @Path("/restore") + @Operation( + operationId = "restore", + summary = "Restore a soft deleted SearchService.", + description = "Restore a soft deleted SearchService.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully restored the SearchService.", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = SearchService.class))) + }) + public Response restoreSearchService( + @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid RestoreEntity restore) { + return restoreEntity(uriInfo, securityContext, restore.getId()); + } + + private SearchService getService(CreateSearchService create, String user) { + return copy(new SearchService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } + + @Override + protected SearchService nullifyConnection(SearchService service) { + return service.withConnection(null); + } + + @Override + protected String extractServiceType(SearchService service) { + return service.getServiceType().value(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java index 59f15d7e34be..b2a9c9a70049 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java @@ -13,6 +13,7 @@ import java.util.stream.IntStream; import javax.ws.rs.core.SecurityContext; import org.openmetadata.schema.entity.data.Query; +import org.openmetadata.schema.entity.data.SearchIndex; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.tests.TestCase; @@ -21,6 +22,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TableData; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.searchindex.SearchIndexSampleData; import org.openmetadata.schema.type.topic.TopicSampleData; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.ColumnUtil; @@ -99,6 +101,22 @@ public static Topic getSampleData(Topic topic) { return topic; } + public static SearchIndex getSampleData(SearchIndex searchIndex) { + SearchIndexSampleData sampleData = searchIndex.getSampleData(); + + // If we don't have sample data, there's nothing to do + if (sampleData == null) { + return searchIndex; + } + + if (hasPiiSensitiveTag(searchIndex)) { + sampleData.setMessages(List.of(MASKED_VALUE)); + searchIndex.setSampleData(sampleData); + } + + return searchIndex; + } + public static Table getTableProfile(Table table) { for (Column column : table.getColumns()) { if (hasPiiSensitiveTag(column)) { @@ -190,6 +208,10 @@ private static boolean hasPiiSensitiveTag(Table table) { return table.getTags().stream().map(TagLabel::getTagFQN).anyMatch(SENSITIVE_PII_TAG::equals); } + private static boolean hasPiiSensitiveTag(SearchIndex searchIndex) { + return searchIndex.getTags().stream().map(TagLabel::getTagFQN).anyMatch(SENSITIVE_PII_TAG::equals); + } + /* Check if the Topic is flagged as PII or any of its fields */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java index a4ebae07a8e7..a679f34daa4a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java @@ -42,6 +42,7 @@ import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.entity.classification.Tag; import org.openmetadata.schema.entity.data.GlossaryTerm; +import org.openmetadata.schema.entity.data.SearchIndex; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.policies.accessControl.Rule; @@ -129,6 +130,10 @@ public final class EntityUtil { (field1, field2) -> field1.getName().equalsIgnoreCase(field2.getName()) && field1.getDataType() == field2.getDataType(); + public static final BiPredicate searchIndexFieldMatch = + (field1, field2) -> + field1.getName().equalsIgnoreCase(field2.getName()) && field1.getDataType() == field2.getDataType(); + private EntityUtil() {} /** Validate that JSON payload can be turned into POJO object */ @@ -371,6 +376,15 @@ public static String getSchemaField(Topic topic, Field field, String fieldName) ? FullyQualifiedName.build("schemaFields", localFieldName) : FullyQualifiedName.build("schemaFields", localFieldName, fieldName); } + /** Return searchIndex field name of format "fields".fieldName.fieldName */ + public static String getSearchIndexField(SearchIndex searchIndex, SearchIndexField field, String fieldName) { + // Remove topic FQN from schemaField FQN to get the local name + String localFieldName = + EntityUtil.getLocalColumnName(searchIndex.getFullyQualifiedName(), field.getFullyQualifiedName()); + return fieldName == null + ? FullyQualifiedName.build("fields", localFieldName) + : FullyQualifiedName.build("fields", localFieldName, fieldName); + } /** Return rule field name of format "rules".ruleName.ruleFieldName */ public static String getRuleField(Rule rule, String ruleField) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 1e2129fd5037..7d35b7af719e 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -197,6 +197,7 @@ import org.openmetadata.service.resources.services.MetadataServiceResourceTest; import org.openmetadata.service.resources.services.MlModelServiceResourceTest; import org.openmetadata.service.resources.services.PipelineServiceResourceTest; +import org.openmetadata.service.resources.services.SearchServiceResourceTest; import org.openmetadata.service.resources.services.StorageServiceResourceTest; import org.openmetadata.service.resources.tags.TagResourceTest; import org.openmetadata.service.resources.teams.RoleResourceTest; @@ -292,6 +293,8 @@ public abstract class EntityResourceTest { + public static final List SEARCH_INDEX_FIELDS = + Arrays.asList( + getField("id", SearchIndexDataType.KEYWORD, null), + getField("name", SearchIndexDataType.KEYWORD, null), + getField("address", SearchIndexDataType.TEXT, null)); + + public SearchIndexResourceTest() { + super( + Entity.SEARCH_INDEX, + SearchIndex.class, + SearchIndexResource.SearchIndexList.class, + "searchIndexes", + SearchIndexResource.FIELDS); + supportsSearchIndex = true; + } + + @Test + void post_searchIndexWithoutRequiredFields_4xx(TestInfo test) { + // Service is required field + assertResponse( + () -> createEntity(createRequest(test).withService(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "[service must not be null]"); + + // Partitions is required field + assertResponse( + () -> createEntity(createRequest(test).withFields(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "[fields must not be null]"); + } + + @Test + void post_searchIndexWithDifferentService_200_ok(TestInfo test) throws IOException { + String[] differentServices = { + ELASTICSEARCH_SEARCH_SERVICE_REFERENCE.getName(), OPENSEARCH_SEARCH_SERVICE_REFERENCE.getName() + }; + + // Create searchIndex for each service and test APIs + for (String service : differentServices) { + createAndCheckEntity(createRequest(test).withService(service), ADMIN_AUTH_HEADERS); + + // List searchIndexes by filtering on service name and ensure right searchIndexes in the response + Map queryParams = new HashMap<>(); + queryParams.put("service", service); + + ResultList list = listEntities(queryParams, ADMIN_AUTH_HEADERS); + for (SearchIndex searchIndex : list.getData()) { + assertEquals(service, searchIndex.getService().getName()); + } + } + } + + @Test + void put_searchIndexAttributes_200_ok(TestInfo test) throws IOException { + ArrayList fields = + new ArrayList<>( + Arrays.asList( + new SearchIndexField().withName("name").withDataType(SearchIndexDataType.TEXT), + new SearchIndexField().withName("displayName").withDataType(SearchIndexDataType.KEYWORD))); + List searchIndexFields = + Arrays.asList( + new SearchIndexField() + .withName("tableSearchIndex") + .withDataType(SearchIndexDataType.NESTED) + .withChildren(fields)); + CreateSearchIndex createSearchIndex = createRequest(test).withOwner(USER1_REF).withFields(searchIndexFields); + + SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS); + ChangeDescription change = getChangeDescription(searchIndex.getVersion()); + + // Patch and update the searchIndex + fields.add(new SearchIndexField().withName("updatedBy").withDataType(SearchIndexDataType.KEYWORD)); + List updatedSearchIndexFields = + List.of( + new SearchIndexField() + .withName("tableSearchIndex") + .withChildren(fields) + .withDataType(SearchIndexDataType.NESTED)); + createSearchIndex.withOwner(TEAM11_REF).withDescription("searchIndex").withFields(updatedSearchIndexFields); + SearchIndexField addedField = fields.get(2); + addedField.setFullyQualifiedName( + searchIndex.getFields().get(0).getFullyQualifiedName() + "." + addedField.getName()); + fieldUpdated(change, FIELD_OWNER, USER1_REF, TEAM11_REF); + fieldUpdated(change, "description", "", "searchIndex"); + fieldAdded(change, "fields.tableSearchIndex", JsonUtils.pojoToJson(List.of(addedField))); + updateAndCheckEntity(createSearchIndex, Status.OK, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change); + } + + @Test + void put_searchIndexFields_200_ok(TestInfo test) throws IOException { + List fields = + Arrays.asList( + getField("id", SearchIndexDataType.KEYWORD, null), + getField("first_name", SearchIndexDataType.KEYWORD, null), + getField("last_name", SearchIndexDataType.TEXT, null), + getField("email", SearchIndexDataType.KEYWORD, null), + getField("address_line_1", SearchIndexDataType.ARRAY, null), + getField("address_line_2", SearchIndexDataType.TEXT, null), + getField("post_code", SearchIndexDataType.TEXT, null), + getField("county", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL)); + + CreateSearchIndex createSearchIndex = createRequest(test).withOwner(USER1_REF).withFields(fields); + + // update the searchIndex + SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS); + searchIndex = getEntity(searchIndex.getId(), ADMIN_AUTH_HEADERS); + assertFields(fields, searchIndex.getFields()); + } + + @Test + void patch_searchIndexAttributes_200_ok(TestInfo test) throws IOException { + List fields = + Arrays.asList( + getField("id", SearchIndexDataType.KEYWORD, null), + getField("first_name", SearchIndexDataType.KEYWORD, null), + getField("last_name", SearchIndexDataType.TEXT, null), + getField("email", SearchIndexDataType.KEYWORD, null), + getField("address_line_1", SearchIndexDataType.ARRAY, null), + getField("address_line_2", SearchIndexDataType.TEXT, null), + getField("post_code", SearchIndexDataType.TEXT, null), + getField("county", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL)); + CreateSearchIndex createSearchIndex = createRequest(test).withOwner(USER1_REF).withFields(fields); + + SearchIndex searchIndex = createEntity(createSearchIndex, ADMIN_AUTH_HEADERS); + String origJson = JsonUtils.pojoToJson(searchIndex); + + List updatedFields = + Arrays.asList( + getField("id", SearchIndexDataType.KEYWORD, null), + getField("first_name", SearchIndexDataType.KEYWORD, null), + getField("last_name", SearchIndexDataType.TEXT, null), + getField("email", SearchIndexDataType.KEYWORD, null), + getField("address_line_1", SearchIndexDataType.ARRAY, null), + getField("address_line_2", SearchIndexDataType.TEXT, null), + getField("post_code", SearchIndexDataType.TEXT, null), + getField("county", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL), + getField("phone", SearchIndexDataType.TEXT, PERSONAL_DATA_TAG_LABEL)); + + searchIndex.withOwner(TEAM11_REF).withFields(updatedFields); + + SearchIndexField addedField = updatedFields.get(updatedFields.size() - 1); + addedField.setFullyQualifiedName(searchIndex.getFullyQualifiedName() + "." + addedField.getName()); + + ChangeDescription change = getChangeDescription(searchIndex.getVersion()); + fieldUpdated(change, FIELD_OWNER, USER1_REF, TEAM11_REF); + fieldAdded(change, "fields", JsonUtils.pojoToJson(List.of(addedField))); + patchEntityAndCheck(searchIndex, origJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change); + } + + @Test + void test_mutuallyExclusiveTags(TestInfo testInfo) { + // Apply mutually exclusive tags to a table + List fields = + Arrays.asList( + getField("id", SearchIndexDataType.KEYWORD, null), + getField("first_name", SearchIndexDataType.KEYWORD, null), + getField("last_name", SearchIndexDataType.TEXT, null), + getField("email", SearchIndexDataType.KEYWORD, null)); + + CreateSearchIndex create = + createRequest(testInfo) + .withTags(List.of(TIER1_TAG_LABEL, TIER2_TAG_LABEL)) + .withOwner(USER1_REF) + .withFields(fields); + + // Apply mutually exclusive tags to a searchIndex + assertResponse( + () -> createEntity(create, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL)); + + // Apply mutually exclusive tags to a searchIndex field + CreateSearchIndex create1 = createRequest(testInfo, 1).withOwner(USER1_REF); + SearchIndexField field = + getField("first_name", SearchIndexDataType.TEXT, null).withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL)); + create1.withFields(List.of(field)); + assertResponse( + () -> createEntity(create1, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL)); + + // Apply mutually exclusive tags to a searchIndexes's nested field + CreateSearchIndex create2 = createRequest(testInfo, 1).withOwner(USER1_REF); + SearchIndexField nestedField = + getField("testNested", SearchIndexDataType.TEXT, null).withTags(listOf(TIER1_TAG_LABEL, TIER2_TAG_LABEL)); + SearchIndexField field1 = getField("test", SearchIndexDataType.NESTED, null).withChildren(List.of(nestedField)); + create2.withFields(List.of(field1)); + assertResponse( + () -> createEntity(create2, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + CatalogExceptionMessage.mutuallyExclusiveLabels(TIER2_TAG_LABEL, TIER1_TAG_LABEL)); + } + + @Test + void put_searchIndexSampleData_200(TestInfo test) throws IOException { + List fields = + Arrays.asList( + getField("email", SearchIndexDataType.KEYWORD, null), + getField("firstName", SearchIndexDataType.KEYWORD, null), + getField("lastName", SearchIndexDataType.TEXT, null)); + SearchIndex searchIndex = createAndCheckEntity(createRequest(test).withFields(fields), ADMIN_AUTH_HEADERS); + List messages = + Arrays.asList( + "{\"email\": \"email1@email.com\", \"firstName\": \"Bob\", \"lastName\": \"Jones\"}", + "{\"email\": \"email2@email.com\", \"firstName\": \"Test\", \"lastName\": \"Jones\"}", + "{\"email\": \"email3@email.com\", \"firstName\": \"Bob\", \"lastName\": \"Jones\"}"); + SearchIndexSampleData searchIndexSampleData = new SearchIndexSampleData().withMessages(messages); + SearchIndex searchIndex1 = putSampleData(searchIndex.getId(), searchIndexSampleData, ADMIN_AUTH_HEADERS); + assertEquals(searchIndexSampleData, searchIndex1.getSampleData()); + + SearchIndex searchIndex2 = getSampleData(searchIndex.getId(), ADMIN_AUTH_HEADERS); + assertEquals(searchIndex2.getSampleData(), searchIndex1.getSampleData()); + messages = + Arrays.asList( + "{\"email\": \"email1@email.com\", \"firstName\": \"Bob\", \"lastName\": \"Jones\"}", + "{\"email\": \"email2@email.com\", \"firstName\": \"Test\", \"lastName\": \"Jones\"}"); + searchIndexSampleData.withMessages(messages); + SearchIndex putResponse = putSampleData(searchIndex2.getId(), searchIndexSampleData, ADMIN_AUTH_HEADERS); + assertEquals(searchIndexSampleData, putResponse.getSampleData()); + searchIndex2 = getSampleData(searchIndex.getId(), ADMIN_AUTH_HEADERS); + assertEquals(searchIndexSampleData, searchIndex2.getSampleData()); + } + + @Test + void test_inheritDomain(TestInfo test) throws IOException { + // When domain is not set for a searchIndex, carry it forward from the search service + SearchServiceResourceTest serviceTest = new SearchServiceResourceTest(); + CreateSearchService createService = serviceTest.createRequest(test).withDomain(DOMAIN.getFullyQualifiedName()); + SearchService service = serviceTest.createEntity(createService, ADMIN_AUTH_HEADERS); + + // Create a searchIndex without domain and ensure it inherits domain from the parent + CreateSearchIndex create = createRequest("user").withService(service.getFullyQualifiedName()); + assertDomainInheritance(create, DOMAIN.getEntityReference()); + } + + @Override + public SearchIndex validateGetWithDifferentFields(SearchIndex searchIndex, boolean byName) + throws HttpResponseException { + // .../searchIndex?fields=owner + String fields = ""; + searchIndex = + byName + ? getSearchIndexByName(searchIndex.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getSearchIndex(searchIndex.getId(), fields, ADMIN_AUTH_HEADERS); + assertListNull(searchIndex.getOwner(), searchIndex.getFollowers(), searchIndex.getFollowers()); + + fields = "owner, followers, tags"; + searchIndex = + byName + ? getSearchIndexByName(searchIndex.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getSearchIndex(searchIndex.getId(), fields, ADMIN_AUTH_HEADERS); + assertListNotNull(searchIndex.getService(), searchIndex.getServiceType()); + // Checks for other owner, tags, and followers is done in the base class + return searchIndex; + } + + public SearchIndex getSearchIndex(UUID id, String fields, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource(id); + target = fields != null ? target.queryParam("fields", fields) : target; + return TestUtils.get(target, SearchIndex.class, authHeaders); + } + + public SearchIndex getSearchIndexByName(String fqn, String fields, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResourceByName(fqn); + target = fields != null ? target.queryParam("fields", fields) : target; + return TestUtils.get(target, SearchIndex.class, authHeaders); + } + + @Override + public CreateSearchIndex createRequest(String name) { + return new CreateSearchIndex() + .withName(name) + .withService(getContainer().getFullyQualifiedName()) + .withFields(SEARCH_INDEX_FIELDS); + } + + @Override + public EntityReference getContainer() { + return ELASTICSEARCH_SEARCH_SERVICE_REFERENCE; + } + + @Override + public EntityReference getContainer(SearchIndex entity) { + return entity.getService(); + } + + @Override + public void validateCreatedEntity( + SearchIndex searchIndex, CreateSearchIndex createRequest, Map authHeaders) + throws HttpResponseException { + assertReference(createRequest.getService(), searchIndex.getService()); + // TODO add other fields + TestUtils.validateTags(createRequest.getTags(), searchIndex.getTags()); + } + + @Override + public void compareEntities(SearchIndex expected, SearchIndex updated, Map authHeaders) + throws HttpResponseException { + assertReference(expected.getService(), expected.getService()); + // TODO add other fields + TestUtils.validateTags(expected.getTags(), updated.getTags()); + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (expected == actual) { + return; + } + assertCommonFieldChange(fieldName, expected, actual); + } + + public SearchIndex putSampleData(UUID searchIndexId, SearchIndexSampleData data, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource(searchIndexId).path("/sampleData"); + return TestUtils.put(target, data, SearchIndex.class, OK, authHeaders); + } + + public SearchIndex getSampleData(UUID searchIndexId, Map authHeaders) throws HttpResponseException { + WebTarget target = getResource(searchIndexId).path("/sampleData"); + return TestUtils.get(target, SearchIndex.class, authHeaders); + } + + private static SearchIndexField getField(String name, SearchIndexDataType fieldDataType, TagLabel tag) { + List tags = tag == null ? new ArrayList<>() : singletonList(tag); + return new SearchIndexField().withName(name).withDataType(fieldDataType).withDescription(name).withTags(tags); + } + + private static void assertFields(List expectedFields, List actualFields) + throws HttpResponseException { + if (expectedFields == actualFields) { + return; + } + // Sort columns by name + assertEquals(expectedFields.size(), actualFields.size()); + + // Make a copy before sorting in case the lists are immutable + List expected = new ArrayList<>(expectedFields); + List actual = new ArrayList<>(actualFields); + expected.sort(Comparator.comparing(SearchIndexField::getName)); + actual.sort(Comparator.comparing(SearchIndexField::getName)); + for (int i = 0; i < expected.size(); i++) { + assertField(expected.get(i), actual.get(i)); + } + } + + private static void assertField(SearchIndexField expectedField, SearchIndexField actualField) + throws HttpResponseException { + assertNotNull(actualField.getFullyQualifiedName()); + assertTrue( + expectedField.getName().equals(actualField.getName()) + || expectedField.getName().equals(actualField.getDisplayName())); + assertEquals(expectedField.getDescription(), actualField.getDescription()); + assertEquals(expectedField.getDataType(), actualField.getDataType()); + TestUtils.validateTags(expectedField.getTags(), actualField.getTags()); + + // Check the nested columns + assertFields(expectedField.getChildren(), actualField.getChildren()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/SearchServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/SearchServiceResourceTest.java new file mode 100644 index 000000000000..fed491009f88 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/SearchServiceResourceTest.java @@ -0,0 +1,208 @@ +package org.openmetadata.service.resources.services; + +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.OK; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openmetadata.service.util.EntityUtil.fieldAdded; +import static org.openmetadata.service.util.EntityUtil.fieldUpdated; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.assertResponse; + +import java.io.IOException; +import java.util.Map; +import java.util.UUID; +import javax.ws.rs.client.WebTarget; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openmetadata.schema.api.services.CreateSearchService; +import org.openmetadata.schema.entity.services.SearchService; +import org.openmetadata.schema.entity.services.connections.TestConnectionResult; +import org.openmetadata.schema.entity.services.connections.TestConnectionResultStatus; +import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; +import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.SearchConnection; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.EntityResourceTest; +import org.openmetadata.service.resources.services.searchIndexes.SearchServiceResource; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.TestUtils; + +public class SearchServiceResourceTest extends EntityResourceTest { + public SearchServiceResourceTest() { + super( + Entity.SEARCH_SERVICE, + SearchService.class, + SearchServiceResource.SearchServiceList.class, + "services/searchServices", + "owner"); + this.supportsPatch = false; + } + + public void setupSearchService(TestInfo test) throws HttpResponseException { + SearchServiceResourceTest esSearchServiceResourceTest = new SearchServiceResourceTest(); + CreateSearchService createSearchService = + esSearchServiceResourceTest + .createRequest(test, 1) + .withName("elasticSearch") + .withServiceType(CreateSearchService.SearchServiceType.ElasticSearch) + .withConnection(TestUtils.ELASTIC_SEARCH_CONNECTION); + + SearchService esSearchService = + new SearchServiceResourceTest().createEntity(createSearchService, ADMIN_AUTH_HEADERS); + ELASTICSEARCH_SEARCH_SERVICE_REFERENCE = esSearchService.getEntityReference(); + SearchServiceResourceTest osSearchServiceResourceTest = new SearchServiceResourceTest(); + createSearchService = + osSearchServiceResourceTest + .createRequest(test, 1) + .withName("opensearch") + .withServiceType(CreateSearchService.SearchServiceType.OpenSearch) + .withConnection(TestUtils.OPEN_SEARCH_CONNECTION); + SearchService osSearchService = + new SearchServiceResourceTest().createEntity(createSearchService, ADMIN_AUTH_HEADERS); + OPENSEARCH_SEARCH_SERVICE_REFERENCE = osSearchService.getEntityReference(); + } + + @Test + void post_withoutRequiredFields_400_badRequest(TestInfo test) { + // Create StorageService with mandatory serviceType field empty + assertResponse( + () -> createEntity(createRequest(test).withServiceType(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "[serviceType must not be null]"); + + // Create StorageService with mandatory connection field empty + assertResponse( + () -> createEntity(createRequest(test).withConnection(null), ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "[connection must not be null]"); + } + + @Test + void post_validService_as_admin_200_ok(TestInfo test) throws IOException { + // Create Storage service with different optional fields + Map authHeaders = ADMIN_AUTH_HEADERS; + createAndCheckEntity(createRequest(test, 1).withDescription(null), authHeaders); + createAndCheckEntity(createRequest(test, 2).withDescription("description"), authHeaders); + createAndCheckEntity(createRequest(test, 3).withConnection(TestUtils.ELASTIC_SEARCH_CONNECTION), authHeaders); + } + + @Test + void put_updateService_as_admin_2xx(TestInfo test) throws IOException { + SearchConnection connection1 = + new SearchConnection().withConfig(new ElasticSearchConnection().withHostPort("http://localhost:9300")); + SearchService service = + createAndCheckEntity(createRequest(test).withDescription(null).withConnection(connection1), ADMIN_AUTH_HEADERS); + + ElasticSearchConnection credentials2 = new ElasticSearchConnection().withHostPort("https://localhost:9400"); + SearchConnection connection2 = new SearchConnection().withConfig(credentials2); + + // Update SearchService description and connection + + CreateSearchService update = createRequest(test).withDescription("description1").withConnection(connection2); + + ChangeDescription change = getChangeDescription(service.getVersion()); + fieldAdded(change, "description", "description1"); + fieldUpdated(change, "connection", connection1, connection2); + updateAndCheckEntity(update, OK, ADMIN_AUTH_HEADERS, TestUtils.UpdateType.MINOR_UPDATE, change); + } + + @Test + void put_testConnectionResult_200(TestInfo test) throws IOException { + SearchService service = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS); + // By default, we have no result logged in + assertNull(service.getTestConnectionResult()); + SearchService updatedService = putTestConnectionResult(service.getId(), TEST_CONNECTION_RESULT, ADMIN_AUTH_HEADERS); + // Validate that the data got properly stored + assertNotNull(updatedService.getTestConnectionResult()); + assertEquals(TestConnectionResultStatus.SUCCESSFUL, updatedService.getTestConnectionResult().getStatus()); + assertEquals(updatedService.getConnection(), service.getConnection()); + // Check that the stored data is also correct + SearchService stored = getEntity(service.getId(), ADMIN_AUTH_HEADERS); + assertNotNull(stored.getTestConnectionResult()); + assertEquals(TestConnectionResultStatus.SUCCESSFUL, stored.getTestConnectionResult().getStatus()); + assertEquals(stored.getConnection(), service.getConnection()); + } + + public SearchService putTestConnectionResult( + UUID serviceId, TestConnectionResult testConnectionResult, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource(serviceId).path("/testConnectionResult"); + return TestUtils.put(target, testConnectionResult, SearchService.class, OK, authHeaders); + } + + @Override + public CreateSearchService createRequest(String name) { + return new CreateSearchService() + .withName(name) + .withServiceType(CreateSearchService.SearchServiceType.ElasticSearch) + .withConnection( + new SearchConnection().withConfig(new ElasticSearchConnection().withHostPort("http://localhost:9200"))); + } + + @Override + public void validateCreatedEntity( + SearchService service, CreateSearchService createRequest, Map authHeaders) + throws HttpResponseException { + assertEquals(createRequest.getName(), service.getName()); + SearchConnection expectedConnection = createRequest.getConnection(); + SearchConnection actualConnection = service.getConnection(); + validateConnection(expectedConnection, actualConnection, service.getServiceType()); + } + + @Override + public void compareEntities(SearchService expected, SearchService updated, Map authHeaders) + throws HttpResponseException { + // PATCH operation is not supported by this entity + + } + + @Override + public SearchService validateGetWithDifferentFields(SearchService service, boolean byName) + throws HttpResponseException { + String fields = ""; + service = + byName + ? getEntityByName(service.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(service.getId(), fields, ADMIN_AUTH_HEADERS); + TestUtils.assertListNull(service.getOwner()); + + fields = "owner,tags"; + service = + byName + ? getEntityByName(service.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS) + : getEntity(service.getId(), fields, ADMIN_AUTH_HEADERS); + // Checks for other owner, tags, and followers is done in the base class + return service; + } + + @Override + public void assertFieldChange(String fieldName, Object expected, Object actual) throws IOException { + if (fieldName.equals("connection")) { + assertTrue(((String) actual).contains("-encrypted-value")); + } else { + super.assertCommonFieldChange(fieldName, expected, actual); + } + } + + private void validateConnection( + SearchConnection expectedConnection, + SearchConnection actualConnection, + CreateSearchService.SearchServiceType serviceType) { + if (expectedConnection != null && actualConnection != null) { + if (serviceType == CreateSearchService.SearchServiceType.ElasticSearch) { + ElasticSearchConnection expectedESConnection = (ElasticSearchConnection) expectedConnection.getConfig(); + ElasticSearchConnection actualESConnection; + if (actualConnection.getConfig() instanceof ElasticSearchConnection) { + actualESConnection = (ElasticSearchConnection) actualConnection.getConfig(); + } else { + actualESConnection = JsonUtils.convertValue(actualConnection.getConfig(), ElasticSearchConnection.class); + } + assertEquals(expectedESConnection.getHostPort(), actualESConnection.getHostPort()); + } + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index 6b2b337f7b07..dcc8ef7d66ec 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -23,13 +23,7 @@ import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.api.configuration.LogoConfiguration; -import org.openmetadata.schema.api.data.CreateContainer; -import org.openmetadata.schema.api.data.CreateDashboard; -import org.openmetadata.schema.api.data.CreateGlossary; -import org.openmetadata.schema.api.data.CreateGlossaryTerm; -import org.openmetadata.schema.api.data.CreatePipeline; -import org.openmetadata.schema.api.data.CreateTable; -import org.openmetadata.schema.api.data.CreateTopic; +import org.openmetadata.schema.api.data.*; import org.openmetadata.schema.api.services.CreateDashboardService; import org.openmetadata.schema.api.services.CreateDatabaseService; import org.openmetadata.schema.api.services.CreateMessagingService; @@ -57,6 +51,7 @@ import org.openmetadata.service.resources.glossary.GlossaryResourceTest; import org.openmetadata.service.resources.glossary.GlossaryTermResourceTest; import org.openmetadata.service.resources.pipelines.PipelineResourceTest; +import org.openmetadata.service.resources.searchindex.SearchIndexResourceTest; import org.openmetadata.service.resources.services.DashboardServiceResourceTest; import org.openmetadata.service.resources.services.DatabaseServiceResourceTest; import org.openmetadata.service.resources.services.MessagingServiceResourceTest; @@ -141,6 +136,10 @@ void entitiesCount(TestInfo test) throws HttpResponseException { CreateContainer createContainer = containerResourceTest.createRequest(test); containerResourceTest.createEntity(createContainer, ADMIN_AUTH_HEADERS); + SearchIndexResourceTest SearchIndexResourceTest = new SearchIndexResourceTest(); + CreateSearchIndex createSearchIndex = SearchIndexResourceTest.createRequest(test); + SearchIndexResourceTest.createEntity(createSearchIndex, ADMIN_AUTH_HEADERS); + GlossaryResourceTest glossaryResourceTest = new GlossaryResourceTest(); CreateGlossary createGlossary = glossaryResourceTest.createRequest(test); glossaryResourceTest.createEntity(createGlossary, ADMIN_AUTH_HEADERS); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index 09d4f6c8d62e..1197c0eadbbb 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -69,12 +69,15 @@ import org.openmetadata.schema.services.connections.mlmodel.MlflowConnection; import org.openmetadata.schema.services.connections.pipeline.AirflowConnection; import org.openmetadata.schema.services.connections.pipeline.GluePipelineConnection; +import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; +import org.openmetadata.schema.services.connections.search.OpenSearchConnection; import org.openmetadata.schema.services.connections.storage.S3Connection; import org.openmetadata.schema.type.DashboardConnection; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.MessagingConnection; import org.openmetadata.schema.type.MlModelConnection; import org.openmetadata.schema.type.PipelineConnection; +import org.openmetadata.schema.type.SearchConnection; import org.openmetadata.schema.type.StorageConnection; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; @@ -111,6 +114,10 @@ public final class TestUtils { public static final MlModelConnection MLFLOW_CONNECTION; public static final StorageConnection S3_STORAGE_CONNECTION; + + public static final SearchConnection ELASTIC_SEARCH_CONNECTION; + public static final SearchConnection OPEN_SEARCH_CONNECTION; + public static MetadataConnection AMUNDSEN_CONNECTION; public static MetadataConnection ATLAS_CONNECTION; @@ -218,6 +225,13 @@ public enum UpdateType { S3_STORAGE_CONNECTION = new StorageConnection().withConfig(new S3Connection().withAwsConfig(AWS_CREDENTIALS)); } + static { + ELASTIC_SEARCH_CONNECTION = + new SearchConnection().withConfig(new ElasticSearchConnection().withHostPort("http://localhost:9200")); + OPEN_SEARCH_CONNECTION = + new SearchConnection().withConfig(new OpenSearchConnection().withHostPort("http://localhost:9200")); + } + static { try { PIPELINE_URL = new URI("http://localhost:8080"); diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createSearchIndex.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createSearchIndex.json new file mode 100644 index 000000000000..e70b2b63252e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createSearchIndex.json @@ -0,0 +1,70 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createSearchIndex.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateSearchIndexRequest", + "description": "Create a SearchIndex entity request", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.CreateSearchIndex", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + + "properties": { + "name": { + "description": "Name that identifies this SearchIndex instance uniquely.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this SearchIndex.", + "type": "string" + }, + "description": { + "description": "Description of the SearchIndex instance. What it has and how to use it.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "service": { + "description": "Fully qualified name of the search service where this searchIndex is hosted in", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "fields": { + "description": "Fields in this SearchIndex.", + "type": "array", + "items": { + "$ref": "../../entity/data/searchIndex.json#/definitions/searchIndexField" + }, + "default": null + }, + "searchIndexSettings": { + "description": "Contains key/value pair of searchIndex settings.", + "$ref": "../../entity/data/searchIndex.json#/definitions/searchIndexSettings" + }, + "owner": { + "description": "Owner of this SearchIndex", + "$ref": "../../type/entityReference.json" + }, + "tags": { + "description": "Tags for this SearchIndex", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "extension": { + "description": "Entity extension data with custom attributes added to the entity.", + "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Fully qualified name of the domain the SearchIndex belongs to.", + "type": "string", + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "dataProducts" : { + "description": "List of fully qualified names of data products this entity is part of.", + "type": "array", + "items" : { + "$ref" : "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } + } + }, + "required": ["name", "service", "fields"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createSearchService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createSearchService.json new file mode 100644 index 000000000000..da5c731374de --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createSearchService.json @@ -0,0 +1,48 @@ +{ + "$id": "https://open-metadata.org/schema/api/services/createSearchService.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateSearchServiceRequest", + "description": "Create Search Service entity request", + "type": "object", + "javaType": "org.openmetadata.schema.api.services.CreateSearchService", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + + "properties": { + "name": { + "description": "Name that identifies the this entity instance uniquely", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this search service. It could be title or label from the source services.", + "type": "string" + }, + "description": { + "description": "Description of search service entity.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "serviceType": { + "$ref": "../../entity/services/searchService.json#/definitions/searchServiceType" + }, + "connection": { + "$ref": "../../entity/services/searchService.json#/definitions/searchConnection" + }, + "tags": { + "description": "Tags for this Search Service.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "owner": { + "description": "Owner of this search service.", + "$ref": "../../type/entityReference.json" + }, + "domain" : { + "description": "Fully qualified name of the domain the Search Service belongs to.", + "type": "string" + } + }, + "required": ["name", "serviceType", "connection"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/searchIndex.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/searchIndex.json new file mode 100644 index 000000000000..18aea312099d --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/searchIndex.json @@ -0,0 +1,229 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/SearchIndex.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SearchIndex", + "$comment": "@om-entity-type", + "description": "A `SearchIndex` is a index mapping definition in ElasticSearch or OpenSearch", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.SearchIndex", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "definitions": { + "searchIndexSettings": { + "javaType": "org.openmetadata.schema.type.searchindex.SearchIndexSettings", + "description": "Contains key/value pair of SearchIndex Settings.", + "type": "object" + }, + "searchIndexSampleData": { + "type": "object", + "javaType": "org.openmetadata.schema.type.searchindex.SearchIndexSampleData", + "description": "This schema defines the type to capture sample data for a SearchIndex.", + "properties": { + "messages": { + "description": "List of local sample messages for a SearchIndex.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "dataType": { + "javaType": "org.openmetadata.schema.type.SearchIndexDataType", + "description": "This enum defines the type of data stored in a searchIndex.", + "type": "string", + "enum": [ + "NUMBER", + "TEXT", + "BINARY", + "TIMESTAMP", + "TIMESTAMPZ", + "TIME", + "DATE", + "DATETIME", + "KEYWORD", + "ARRAY", + "OBJECT", + "FLATTENED", + "NESTED", + "JOIN", + "RANGE", + "IP", + "VERSION", + "MURMUR3", + "AGGREGATE_METRIC_DOUBLE", + "HISTOGRAM", + "ANNOTATED-TEXT", + "COMPLETION", + "SEARCH_AS_YOU_TYPE", + "DENSE_VECTOR", + "RANK_FEATURE", + "RANK_FEATURES", + "GEO_POINT", + "GEO_SHAPE", + "POINT", + "SHAPE", + "PERCOLATOR" + ] + }, + "searchIndexFieldName": { + "description": "Local name (not fully qualified name) of the field. ", + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^((?!::).)*$" + }, + "searchIndexField": { + "type": "object", + "javaType": "org.openmetadata.schema.type.SearchIndexField", + "description": "This schema defines the type for a field in a searchIndex.", + "properties": { + "name": { + "$ref": "#/definitions/searchIndexFieldName" + }, + "displayName": { + "description": "Display Name that identifies this searchIndexField name.", + "type": "string" + }, + "dataType": { + "description": "Data type of the searchIndex (int, date etc.).", + "$ref": "#/definitions/dataType" + }, + "dataTypeDisplay": { + "description": "Display name used for dataType. ", + "type": "string" + }, + "description": { + "description": "Description of the field.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "fullyQualifiedName": { + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "tags": { + "description": "Tags associated with the column.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "children": { + "description": "Child columns if dataType has properties.", + "type": "array", + "items": { + "$ref": "#/definitions/searchIndexField" + }, + "default": null + } + }, + "required": [ + "name", + "dataType" + ], + "additionalProperties": false + } + }, + "properties": { + "id": { + "description": "Unique identifier that identifies this SearchIndex instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies the SearchIndex.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "Name that uniquely identifies a SearchIndex in the format 'searchServiceName.searchIndexName'.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Display Name that identifies this SearchIndex. It could be title or label from the source services.", + "type": "string" + }, + "description": { + "description": "Description of the SearchIndex instance.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "service": { + "description": "Link to the search cluster/service where this SearchIndex is hosted in.", + "$ref": "../../type/entityReference.json" + }, + "serviceType": { + "description": "Service type where this SearchIndex is hosted in.", + "$ref": "../services/searchService.json#/definitions/searchServiceType" + }, + "fields": { + "description": "Fields in this SearchIndex.", + "type": "array", + "items": { + "$ref": "#/definitions/searchIndexField" + }, + "default": null + }, + "searchIndexSettings": { + "description": "Contains key/value pair of searchIndex settings.", + "$ref": "#/definitions/searchIndexSettings" + }, + "sampleData": { + "description": "Sample data for a searchIndex.", + "$ref": "#/definitions/searchIndexSampleData", + "default": null + }, + "owner": { + "description": "Owner of this searchIndex.", + "$ref": "../../type/entityReference.json" + }, + "followers": { + "description": "Followers of this searchIndex.", + "$ref": "../../type/entityReferenceList.json" + }, + "tags": { + "description": "Tags for this searchIndex.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + }, + "extension": { + "description": "Entity extension data with custom attributes added to the entity.", + "$ref": "../../type/basic.json#/definitions/entityExtension" + }, + "domain" : { + "description": "Domain the SearchIndex belongs to. When not set, the SearchIndex inherits the domain from the messaging service it belongs to.", + "$ref": "../../type/entityReference.json" + }, + "dataProducts" : { + "description": "List of of data products this entity is part of.", + "$ref" : "../../type/entityReferenceList.json" + } + }, + "required": ["id", "name", "service", "fields"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/customSearchConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/customSearchConnection.json new file mode 100644 index 000000000000..20d6208f3f06 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/customSearchConnection.json @@ -0,0 +1,36 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/search/customSearchConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CustomSearchConnection", + "description": "Custom Search Service connection to build a source that is not supported by OpenMetadata yet.", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.search.CustomSearchConnection", + "definitions": { + "customSearchType": { + "title": "Service Type", + "description": "Custom search service type", + "type": "string", + "enum": ["CustomSearch"], + "default": "CustomSearch" + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Custom search service type", + "$ref": "#/definitions/customSearchType", + "default": "CustomSearch" + }, + "sourcePythonClass": { + "title": "Source Python Class Name", + "description": "Source Python Class Name to instantiated by the ingestion workflow", + "type": "string" + }, + "connectionOptions": { + "title": "Connection Options", + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" + } + }, + "additionalProperties": false, + "required": ["type"] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json new file mode 100644 index 000000000000..244c696c2d66 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json @@ -0,0 +1,80 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/search/elasticSearchConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ElasticSearch Connection", + "description": "ElasticSearch Connection.", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.search.ElasticSearchConnection", + "definitions": { + "elasticSearchType": { + "description": "ElasticSearch service type", + "type": "string", + "enum": ["ElasticSearch"], + "default": "ElasticSearch" + } + }, + "properties": { + "type": { + "title": "ElasticSearch Type", + "description": "ElasticSearch Type", + "$ref": "#/definitions/elasticSearchType", + "default": "ElasticSearch" + }, + "hostPort": { + "title": "Host and Port", + "description": "Host and port of the ElasticSearch service.", + "type": "string" + }, + "scheme": { + "description": "Http/Https connection scheme", + "type": "string", + "default": "http" + }, + "username": { + "description": "Elastic Search Username for Login", + "type": "string" + }, + "password": { + "description": "Elastic Search Password for Login", + "type": "string" + }, + "truststorePath": { + "description": "Truststore Path", + "type": "string" + }, + "truststorePassword": { + "description": "Truststore Password", + "type": "string" + }, + "connectionTimeoutSecs": { + "description": "Connection Timeout in Seconds", + "type": "integer", + "default": 5 + }, + "socketTimeoutSecs": { + "description": "Socket Timeout in Seconds", + "type": "integer", + "default": 60 + }, + "keepAliveTimeoutSecs": { + "description": "Keep Alive Timeout in Seconds", + "type": "integer" + }, + "connectionOptions": { + "title": "Connection Options", + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" + }, + "connectionArguments": { + "title": "Connection Arguments", + "$ref": "../connectionBasicType.json#/definitions/connectionArguments" + }, + "supportsMetadataExtraction": { + "title": "Supports Metadata Extraction", + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + } + }, + "additionalProperties": false, + "required": [ + "hostPort" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/openSearchConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/openSearchConnection.json new file mode 100644 index 000000000000..b2679e3b051a --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/openSearchConnection.json @@ -0,0 +1,79 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/search/openSearchConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "OpenSearch Connection", + "description": "OpenSearch Connection.", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.search.OpenSearchConnection", + "definitions": { + "openSearchType": { + "description": "OpenSearch service type", + "type": "string", + "enum": ["OpenSearch"], + "default": "OpenSearch" + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/openSearchType", + "default": "OpenSearch" + }, + "hostPort": { + "title": "Host and Port", + "description": "Host and port of the OpenSearch service.", + "type": "string" + }, + "scheme": { + "description": "Http/Https connection scheme", + "type": "string" + }, + "username": { + "description": "OpenSearch Username for Login", + "type": "string" + }, + "password": { + "description": "OpenSearch Password for Login", + "type": "string" + }, + "truststorePath": { + "description": "Truststore Path", + "type": "string" + }, + "truststorePassword": { + "description": "Truststore Password", + "type": "string" + }, + "connectionTimeoutSecs": { + "description": "Connection Timeout in Seconds", + "type": "integer", + "default": 5 + }, + "socketTimeoutSecs": { + "description": "Socket Timeout in Seconds", + "type": "integer", + "default": 60 + }, + "keepAliveTimeoutSecs": { + "description": "Keep Alive Timeout in Seconds", + "type": "integer" + }, + "connectionOptions": { + "title": "Connection Options", + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" + }, + "connectionArguments": { + "title": "Connection Arguments", + "$ref": "../connectionBasicType.json#/definitions/connectionArguments" + }, + "supportsMetadataExtraction": { + "title": "Supports Metadata Extraction", + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + } + }, + "additionalProperties": false, + "required": [ + "hostPort" + ] +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json index 570b901ac0b1..ae2c5d26bfb3 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json @@ -28,6 +28,9 @@ }, { "$ref": "../storageService.json#/definitions/storageConnection" + }, + { + "$ref": "../searchService.json#/definitions/searchConnection" } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json new file mode 100644 index 000000000000..040209c6e73d --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/searchService.json @@ -0,0 +1,142 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/searchService.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Search Service", + "description": "This schema defines the Search Service entity, such as ElasticSearch, OpenSearch.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.services.SearchService", + "javaInterfaces": [ + "org.openmetadata.schema.EntityInterface", + "org.openmetadata.schema.ServiceEntityInterface" + ], + "definitions": { + "searchServiceType": { + "description": "Type of search service such as ElasticSearch or OpenSearch.", + "javaInterfaces": [ + "org.openmetadata.schema.EnumInterface" + ], + "type": "string", + "enum": [ + "ElasticSearch", + "OpenSearch", + "CustomSearch" + ], + "javaEnums": [ + { + "name": "ElasticSearch" + }, + { + "name": "OpenSearch" + }, + { + "name": "CustomSearch" + } + ] + }, + "searchConnection": { + "type": "object", + "javaType": "org.openmetadata.schema.type.SearchConnection", + "description": "search Connection.", + "javaInterfaces": [ + "org.openmetadata.schema.ServiceConnectionEntityInterface" + ], + "properties": { + "config": { + "mask": true, + "oneOf": [ + { + "$ref": "connections/search/elasticSearchConnection.json" + }, + { + "$ref": "connections/search/openSearchConnection.json" + }, + { + "$ref": "connections/search/customSearchConnection.json" + } + ] + } + }, + "additionalProperties": false + } + }, + "properties": { + "id": { + "description": "Unique identifier of this search service instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name that identifies this search service.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "FullyQualifiedName same as `name`.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Display Name that identifies this search service.", + "type": "string" + }, + "serviceType": { + "description": "Type of search service such as S3, GCS, AZURE...", + "$ref": "#/definitions/searchServiceType" + }, + "description": { + "description": "Description of a search service instance.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "connection": { + "$ref": "#/definitions/searchConnection" + }, + "pipelines": { + "description": "References to pipelines deployed for this search service to extract metadata etc..", + "$ref": "../../type/entityReferenceList.json" + }, + "testConnectionResult": { + "description": "Last test connection results for this service", + "$ref": "connections/testConnectionResult.json" + }, + "tags": { + "description": "Tags for this search Service.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this search service.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "owner": { + "description": "Owner of this search service.", + "$ref": "../../type/entityReference.json" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When `true` indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + }, + "domain" : { + "description": "Domain the search service belongs to.", + "$ref": "../../type/entityReference.json" + } + }, + "required": ["id", "name", "serviceType"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json index 05110e189cf4..14b6faa5af78 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json @@ -12,7 +12,8 @@ "Metadata", "MlModel", "Pipeline", - "Storage" + "Storage", + "Search" ], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json new file mode 100644 index 000000000000..a52cc3c648ca --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json @@ -0,0 +1,27 @@ +{ + "$id": "https://open-metadata.org/schema/metadataIngestion/searchServiceMetadataPipeline.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SearchServiceMetadataPipeline", + "description": "SearchService Metadata Pipeline Configuration.", + "type": "object", + "definitions": { + "searchMetadataConfigType": { + "description": "Search Source Config Metadata Pipeline type", + "type": "string", + "enum": ["SearchMetadata"], + "default": "SearchMetadata" + } + }, + "properties": { + "type": { + "description": "Pipeline type", + "$ref": "#/definitions/searchMetadataConfigType", + "default": "SearchMetadata" + }, + "searchIndexFilterPattern": { + "description": "Regex to only fetch search indexes that matches the pattern.", + "$ref": "../type/filterPattern.json#/definitions/filterPattern" + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json index db0152b8eaf8..9c6b9bb9725e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json @@ -39,6 +39,9 @@ { "$ref": "storageServiceMetadataPipeline.json" }, + { + "$ref": "searchServiceMetadataPipeline.json" + }, { "$ref": "testSuitePipeline.json" },