diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql index 50d6aff17f8a..81409cb6d751 100644 --- a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql @@ -169,5 +169,17 @@ DELETE FROM event_subscription_entity; DELETE FROM change_event_consumers; DELETE FROM consumers_dlq; -UPDATE ingestion_pipeline_entity SET json = JSON_SET(json, '$.provider', 'user') +CREATE TABLE IF NOT EXISTS suggestions ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL COLLATE ascii_bin, + entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, + suggestionType VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json ->> '$.type')) NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + status VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json -> '$.status')) NOT NULL, + PRIMARY KEY (id) +); + +UPDATE ingestion_pipeline_entity SET json = JSON_SET(json, '$.provider', 'user') WHERE JSON_EXTRACT(json, '$.name') = 'OpenMetadata_dataInsight'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index 1089b255a5b0..8e8c6bee922e 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -181,5 +181,17 @@ DELETE FROM event_subscription_entity; DELETE FROM change_event_consumers; DELETE FROM consumers_dlq; +CREATE TABLE IF NOT EXISTS suggestions ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL, + entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'entityLink') STORED NOT NULL, + suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'type') STORED NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED NOT NULL, + PRIMARY KEY (id) +); + UPDATE ingestion_pipeline_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) WHERE json->>'name' = 'OpenMetadata_dataInsight'; \ No newline at end of file diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 793cbc41a79f..5e7704e38cf8 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -369,4 +369,3 @@ web: permission-policy: enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false} option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""} - diff --git a/ingestion/src/metadata/applications/auto_tagger.py b/ingestion/src/metadata/applications/auto_tagger.py index 3583fd373652..6719cde48e35 100644 --- a/ingestion/src/metadata/applications/auto_tagger.py +++ b/ingestion/src/metadata/applications/auto_tagger.py @@ -21,6 +21,9 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) +from metadata.generated.schema.metadataIngestion.application import ( + OpenMetadataApplicationConfig, +) from metadata.generated.schema.type.tagLabel import ( LabelType, State, @@ -60,16 +63,18 @@ class AutoTaggerApp(AppRunner): jwtToken: "..." """ - def __init__(self, config: AutoTaggerAppConfig, metadata: OpenMetadata): + def __init__(self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata): super().__init__(config, metadata) - if not isinstance(config, AutoTaggerAppConfig): + if not isinstance(self.app_config, AutoTaggerAppConfig): raise InvalidAppConfiguration( f"AutoTagger Runner expects an AutoTaggerAppConfig, we got [{config}]" ) self._ner_scanner = None - self.confidence_threshold = config.confidenceLevel or DEFAULT_CONFIDENCE + self.confidence_threshold = ( + self.app_config.confidenceLevel or DEFAULT_CONFIDENCE + ) @property def name(self) -> str: diff --git a/ingestion/src/metadata/data_quality/processor/test_case_runner.py b/ingestion/src/metadata/data_quality/processor/test_case_runner.py index 0c78af92e881..12205827472e 100644 --- a/ingestion/src/metadata/data_quality/processor/test_case_runner.py +++ b/ingestion/src/metadata/data_quality/processor/test_case_runner.py @@ -28,6 +28,7 @@ test_suite_source_factory, ) from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest +from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -200,8 +201,9 @@ def compare_and_create_test_cases( ), entityLink=EntityLink( __root__=entity_link.get_entity_link( - table_fqn, - test_case_to_create.columnName, + Table, + fqn=table_fqn, + column_name=test_case_to_create.columnName, ) ), testSuite=test_suite_fqn, @@ -252,8 +254,9 @@ def _update_test_cases( updated_test_case = self.metadata.patch_test_case_definition( source=test_case, entity_link=entity_link.get_entity_link( - table_fqn, - test_case_definition.columnName, + Table, + fqn=table_fqn, + column_name=test_case_definition.columnName, ), test_case_parameter_values=test_case_definition.parameterValues, ) diff --git a/ingestion/src/metadata/great_expectations/action.py b/ingestion/src/metadata/great_expectations/action.py index c34b685bf422..b127f224dc5e 100644 --- a/ingestion/src/metadata/great_expectations/action.py +++ b/ingestion/src/metadata/great_expectations/action.py @@ -420,8 +420,9 @@ def _handle_test_case( test_case = self.ometa_conn.get_or_create_test_case( test_case_fqn, entity_link=get_entity_link( - table_entity.fullyQualifiedName.__root__, - fqn.split_test_case_fqn(test_case_fqn).column, + Table, + fqn=table_entity.fullyQualifiedName.__root__, + column_name=fqn.split_test_case_fqn(test_case_fqn).column, ), test_suite_fqn=test_suite.fullyQualifiedName.__root__, test_definition_fqn=test_definition.fullyQualifiedName.__root__, diff --git a/ingestion/src/metadata/ingestion/api/status.py b/ingestion/src/metadata/ingestion/api/status.py index 49068e79aca6..8e5089d30601 100644 --- a/ingestion/src/metadata/ingestion/api/status.py +++ b/ingestion/src/metadata/ingestion/api/status.py @@ -56,6 +56,10 @@ def scanned(self, record: Any) -> None: else: self.records.append(log_name) + def updated(self, record: Any) -> None: + if log_name := get_log_name(record): + self.updated_records.append(log_name) + def warning(self, key: str, reason: str) -> None: self.warnings.append({key: reason}) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py new file mode 100644 index 000000000000..c2fc27cb1d10 --- /dev/null +++ b/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py @@ -0,0 +1,41 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Mixin class containing Suggestions specific methods + +To be used by OpenMetadata class +""" +from metadata.generated.schema.entity.feed.suggestion import Suggestion +from metadata.ingestion.ometa.client import REST +from metadata.utils.logger import ometa_logger + +logger = ometa_logger() + + +class OMetaSuggestionsMixin: + """ + OpenMetadata API methods related to the Suggestion Entity + + To be inherited by OpenMetadata + """ + + client: REST + + def update_suggestion(self, suggestion: Suggestion) -> Suggestion: + """ + Update an existing Suggestion with new fields + """ + resp = self.client.put( + f"{self.get_suffix(Suggestion)}/{str(suggestion.id.__root__)}", + data=suggestion.json(), + ) + + return Suggestion(**resp) diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 881390b8ec9d..b791581bc935 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -50,6 +50,7 @@ from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin +from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin @@ -108,6 +109,7 @@ class OpenMetadata( OMetaRolePolicyMixin, OMetaSearchIndexMixin, OMetaCustomPropertyMixin, + OMetaSuggestionsMixin, Generic[T, C], ): """ @@ -246,11 +248,9 @@ def get_entity_from_create(self, create: Type[C]) -> Type[T]: ) return entity_class - def create_or_update(self, data: C) -> T: + def _create(self, data: C, method: str) -> T: """ - We allow CreateEntity for PUT, so we expect a type C. - - We PUT to the endpoint and return the Entity generated result + Internal logic to run POST vs. PUT """ entity = data.__class__ is_create = "create" in data.__class__.__name__.lower() @@ -262,15 +262,23 @@ def create_or_update(self, data: C) -> T: raise InvalidEntityException( f"PUT operations need a CreateEntity, not {entity}" ) - resp = self.client.put( - self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder) - ) + + fn = getattr(self.client, method) + resp = fn(self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder)) if not resp: raise EmptyPayloadException( f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.json()}" ) return entity_class(**resp) + def create_or_update(self, data: C) -> T: + """Run a PUT requesting via create request C""" + return self._create(data=data, method="put") + + def create(self, data: C) -> T: + """Run a POST requesting via create request C""" + return self._create(data=data, method="post") + def get_by_name( self, entity: Type[T], diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index 6cf70529ad07..f625459a179f 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -50,6 +50,7 @@ CreateDataProductRequest, ) from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest +from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest from metadata.generated.schema.api.services.createDashboardService import ( @@ -90,6 +91,7 @@ from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart from metadata.generated.schema.dataInsight.kpi.kpi import Kpi from metadata.generated.schema.entity.automations.workflow import Workflow +from metadata.generated.schema.entity.bot import Bot from metadata.generated.schema.entity.classification.classification import ( Classification, ) @@ -113,6 +115,7 @@ from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.domains.dataProduct import DataProduct from metadata.generated.schema.entity.domains.domain import Domain +from metadata.generated.schema.entity.feed.suggestion import Suggestion from metadata.generated.schema.entity.policies.policy import Policy from metadata.generated.schema.entity.services.connections.testConnectionDefinition import ( TestConnectionDefinition, @@ -181,6 +184,7 @@ User.__name__: "/users", CreateUserRequest.__name__: "/users", AuthenticationMechanism.__name__: "/users/auth-mechanism", + Bot.__name__: "/bots", # We won't allow bot creation from the client # Roles Role.__name__: "/roles", CreateRoleRequest.__name__: "/roles", @@ -225,4 +229,7 @@ CreateDomainRequest.__name__: "/domains", DataProduct.__name__: "/dataProducts", CreateDataProductRequest.__name__: "/dataProducts", + # Suggestions + Suggestion.__name__: "/suggestions", + CreateSuggestionRequest.__name__: "/suggestions", } diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py index f5f7bf407e16..70bfcabe1b13 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py @@ -14,6 +14,7 @@ import traceback from typing import Optional, Union +from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.tests.testSuite import TestSuite from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -81,7 +82,8 @@ def generate_entity_link(dbt_test): manifest_node = dbt_test.get(DbtCommonEnum.MANIFEST_NODE.value) entity_link_list = [ entity_link.get_entity_link( - table_fqn=table_fqn, + Table, + fqn=table_fqn, column_name=manifest_node.column_name if hasattr(manifest_node, "column_name") else None, diff --git a/ingestion/src/metadata/utils/entity_link.py b/ingestion/src/metadata/utils/entity_link.py index 6ad0c2f0826b..0fbac0f0e0d8 100644 --- a/ingestion/src/metadata/utils/entity_link.py +++ b/ingestion/src/metadata/utils/entity_link.py @@ -13,17 +13,23 @@ Filter information has been taken from the ES indexes definitions """ -from typing import List, Optional +from typing import Any, List, Optional, TypeVar from antlr4.CommonTokenStream import CommonTokenStream from antlr4.error.ErrorStrategy import BailErrorStrategy from antlr4.InputStream import InputStream from antlr4.tree.Tree import ParseTreeWalker +from pydantic import BaseModel from requests.compat import unquote_plus from metadata.antlr.split_listener import EntityLinkSplitListener from metadata.generated.antlr.EntityLinkLexer import EntityLinkLexer from metadata.generated.antlr.EntityLinkParser import EntityLinkParser +from metadata.generated.schema.entity.data.table import Table +from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP +from metadata.utils.dispatch import class_register + +T = TypeVar("T", bound=BaseModel) class EntityLinkBuildingException(Exception): @@ -86,16 +92,30 @@ def get_table_or_column_fqn(entity_link: str) -> str: ) -def get_entity_link(table_fqn: str, column_name: Optional[str]) -> str: +get_entity_link_registry = class_register() + + +def get_entity_link(entity_type: Any, fqn: str, **kwargs) -> str: """From table fqn and column name get the entity_link Args: - table_fqn: table fqn - column_name: Optional param to generate entity link with column name + entity_type: Entity being built + fqn: Entity fqn """ + func = get_entity_link_registry.registry.get(entity_type.__name__) + if not func: + return f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}>" + + return func(fqn, **kwargs) + + +@get_entity_link_registry.add(Table) +def _(fqn: str, column_name: Optional[str] = None) -> str: + """From table fqn and column name get the entity_link""" + if column_name: - entity_link = f"<#E::table::" f"{table_fqn}" f"::columns::" f"{column_name}>" + entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[Table.__name__]}::{fqn}::columns::{column_name}>" else: - entity_link = f"<#E::table::" f"{table_fqn}>" + entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[Table.__name__]}::{fqn}>" return entity_link diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index f76c979934b8..94692ed1475b 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -31,7 +31,9 @@ from metadata.generated.schema.entity.data.chart import ChartType from metadata.generated.schema.entity.data.table import Column, Table +from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.type.basic import EntityLink from metadata.generated.schema.type.tagLabel import TagLabel from metadata.utils.constants import DEFAULT_DATABASE from metadata.utils.logger import utils_logger @@ -227,12 +229,38 @@ def find_in_iter(element: Any, container: Iterable[Any]) -> Optional[Any]: return next((elem for elem in container if elem == element), None) -def find_column_in_table(column_name: str, table: Table) -> Optional[Column]: +def find_column_in_table( + column_name: str, table: Table, case_sensitive: bool = True +) -> Optional[Column]: """ If the column exists in the table, return it """ + + def equals(first: str, second: str) -> bool: + if case_sensitive: + return first == second + return first.lower() == second.lower() + + return next( + (col for col in table.columns if equals(col.name.__root__, column_name)), None + ) + + +def find_suggestion( + suggestions: List[Suggestion], + suggestion_type: SuggestionType, + entity_link: EntityLink, +) -> Optional[Suggestion]: + """Given a list of suggestions, a suggestion type and an entity link, find + one suggestion in the list that matches the criteria + """ return next( - (col for col in table.columns if col.name.__root__ == column_name), None + ( + sugg + for sugg in suggestions + if sugg.type == suggestion_type and sugg.entityLink == entity_link + ), + None, ) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 09ebd2461a5d..07ce0688bb5f 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -15,9 +15,6 @@ from typing import List, Optional from metadata.config.common import WorkflowExecutionError -from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( - AppConfig, -) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) @@ -50,9 +47,14 @@ class AppRunner(Step, ABC): """Class that knows how to execute the Application logic.""" def __init__( - self, config: AppConfig.__fields__["__root__"].type_, metadata: OpenMetadata + self, + config: OpenMetadataApplicationConfig, + metadata: OpenMetadata, ): - self.config = config + self.app_config = config.appConfig.__root__ if config.appConfig else None + self.private_config = ( + config.appPrivateConfig.__root__ if config.appPrivateConfig else None + ) self.metadata = metadata super().__init__() @@ -67,7 +69,8 @@ def run(self) -> None: @classmethod def create(cls, config_dict: dict, metadata: OpenMetadata) -> "Step": - return cls(config=config_dict, metadata=metadata) + config = OpenMetadataApplicationConfig.parse_obj(config_dict) + return cls(config=config, metadata=metadata) class ApplicationWorkflow(BaseWorkflow, ABC): @@ -114,9 +117,7 @@ def post_init(self) -> None: try: self.runner = runner_class( - config=self.config.appConfig.__root__ - if self.config.appConfig - else None, + config=self.config, metadata=self.metadata, ) except Exception as exc: diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index 58beadb9a0af..1404a530edd2 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -51,11 +51,9 @@ AuthProvider, OpenMetadataConnection, ) -from metadata.generated.schema.entity.services.connections.pipeline.airflowConnection import ( - AirflowConnection, -) -from metadata.generated.schema.entity.services.connections.pipeline.backendConnection import ( - BackendConnection, +from metadata.generated.schema.entity.services.connections.pipeline.customPipelineConnection import ( + CustomPipelineConnection, + CustomPipelineType, ) from metadata.generated.schema.entity.services.databaseService import ( DatabaseConnection, @@ -92,7 +90,6 @@ def int_admin_ometa(url: str = "http://localhost:8585/api") -> OpenMetadata: ) metadata = OpenMetadata(server_config) assert metadata.health_check() - return metadata @@ -123,12 +120,21 @@ def _(name: EntityName) -> C: """Prepare a Create service request""" return CreatePipelineServiceRequest( name=name, - serviceType=PipelineServiceType.Airflow, + serviceType=PipelineServiceType.CustomPipeline, connection=PipelineConnection( - config=AirflowConnection( - hostPort="http://localhost:8080", - connection=BackendConnection(), - ), + config=CustomPipelineConnection(type=CustomPipelineType.CustomPipeline) + ), + ) + + +@create_service_registry.add(DatabaseService) +def _(name: EntityName) -> C: + """Prepare a Create service request""" + return CreateDatabaseServiceRequest( + name=name, + serviceType=DatabaseServiceType.CustomDatabase, + connection=DatabaseConnection( + config=CustomDatabaseConnection(type=CustomDatabaseType.CustomDatabase) ), ) diff --git a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py new file mode 100644 index 000000000000..0d3da0ab62bc --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py @@ -0,0 +1,197 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenMetadata high-level API Suggestion test +""" +from unittest import TestCase + +from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.entity.teams.user import User +from metadata.generated.schema.type.basic import EntityLink +from metadata.generated.schema.type.tagLabel import ( + LabelType, + State, + TagLabel, + TagSource, +) +from metadata.utils.entity_link import get_entity_link + +from ..integration_base import ( + generate_name, + get_create_entity, + get_create_service, + int_admin_ometa, +) + + +class OMetaSuggestionTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + metadata = int_admin_ometa() + + service_name = generate_name() + db_name = generate_name() + schema_name = generate_name() + table_name = generate_name() + + @classmethod + def setUpClass(cls) -> None: + """ + Prepare ingredients: Pipeline Entity + """ + create_service = get_create_service( + entity=DatabaseService, name=cls.service_name + ) + cls.metadata.create_or_update(create_service) + + create_database = get_create_entity( + entity=Database, name=cls.schema_name, reference=cls.service_name.__root__ + ) + cls.database: Database = cls.metadata.create_or_update(create_database) + + create_schema = get_create_entity( + entity=DatabaseSchema, + name=cls.schema_name, + reference=cls.database.fullyQualifiedName.__root__, + ) + cls.schema: DatabaseSchema = cls.metadata.create_or_update(create_schema) + + create_table = get_create_entity( + entity=Table, + name=cls.table_name, + reference=cls.schema.fullyQualifiedName.__root__, + ) + cls.table: Table = cls.metadata.create_or_update(create_table) + + @classmethod + def tearDownClass(cls) -> None: + """ + Clean up + """ + + service_id = str( + cls.metadata.get_by_name( + entity=DatabaseService, fqn=cls.service_name.__root__ + ).id.__root__ + ) + + cls.metadata.delete( + entity=DatabaseService, + entity_id=service_id, + recursive=True, + hard_delete=True, + ) + + def test_create_description_suggestion(self): + """We can create a suggestion""" + suggestion_request = CreateSuggestionRequest( + description="something", + type=SuggestionType.SuggestDescription, + entityLink=EntityLink( + __root__=get_entity_link( + Table, fqn=self.table.fullyQualifiedName.__root__ + ) + ), + ) + + # Suggestions only support POST (not PUT) + self.metadata.create(suggestion_request) + + def test_create_tag_suggestion(self): + """We can create a suggestion""" + suggestion_request = CreateSuggestionRequest( + tagLabels=[ + TagLabel( + tagFQN="PII.Sensitive", + labelType=LabelType.Automated, + state=State.Suggested.value, + source=TagSource.Classification, + ) + ], + type=SuggestionType.SuggestTagLabel, + entityLink=EntityLink( + __root__=get_entity_link( + Table, fqn=self.table.fullyQualifiedName.__root__ + ) + ), + ) + + # Suggestions only support POST (not PUT) + self.metadata.create(suggestion_request) + + def test_list(self): + """List filtering by creator""" + + admin_user: User = self.metadata.get_by_name( + entity=User, fqn="admin", nullable=False + ) + + create_table = get_create_entity( + entity=Table, + reference=self.schema.fullyQualifiedName.__root__, + ) + table: Table = self.metadata.create_or_update(create_table) + + suggestion_request = CreateSuggestionRequest( + description="something", + type=SuggestionType.SuggestDescription, + entityLink=EntityLink( + __root__=get_entity_link(Table, fqn=table.fullyQualifiedName.__root__) + ), + ) + + # Suggestions only support POST (not PUT) + self.metadata.create(suggestion_request) + + suggestions = self.metadata.list_all_entities( + entity=Suggestion, + params={ + "entityFQN": table.fullyQualifiedName.__root__, + "userId": str(admin_user.id.__root__), + }, + ) + + self.assertEqual(len(list(suggestions)), 1) + + def test_update_suggestion(self): + """Update an existing suggestion""" + + create_table = get_create_entity( + entity=Table, + name=self.schema_name, + reference=self.schema.fullyQualifiedName.__root__, + ) + table: Table = self.metadata.create_or_update(create_table) + + suggestion_request = CreateSuggestionRequest( + description="something", + type=SuggestionType.SuggestDescription, + entityLink=EntityLink( + __root__=get_entity_link(Table, fqn=table.fullyQualifiedName.__root__) + ), + ) + + # Suggestions only support POST (not PUT) + res: Suggestion = self.metadata.create(suggestion_request) + self.assertEqual(res.description, "something") + + res.description = "new" + new = self.metadata.update_suggestion(res) + self.assertEqual(new.description, "new") diff --git a/ingestion/tests/integration/utils/test_helpers.py b/ingestion/tests/integration/utils/test_helpers.py index 004842afda34..53ef680c0c86 100644 --- a/ingestion/tests/integration/utils/test_helpers.py +++ b/ingestion/tests/integration/utils/test_helpers.py @@ -63,3 +63,6 @@ def test_find_column_in_table(self): ) self.assertIsNone(not_found) self.assertIsNone(not_found_idx) + + col = find_column_in_table(column_name="FOO", table=table, case_sensitive=False) + self.assertEqual(col, Column(name="foo", dataType=DataType.BIGINT)) diff --git a/ingestion/tests/unit/test_entity_link.py b/ingestion/tests/unit/test_entity_link.py index ca45c858b1a6..fe02499c8997 100644 --- a/ingestion/tests/unit/test_entity_link.py +++ b/ingestion/tests/unit/test_entity_link.py @@ -13,7 +13,10 @@ """ from unittest import TestCase +from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.table import Table from metadata.utils import entity_link +from metadata.utils.entity_link import get_entity_link class TestEntityLink(TestCase): @@ -107,3 +110,19 @@ def validate(self, fn_resp, check_split): ] for x in xs: x.validate(entity_link.split(x.entitylink), x.split_list) + + def test_get_entity_link(self): + """We can get entity link for different entities""" + + table_link = get_entity_link(Table, fqn="service.db.schema.table") + self.assertEqual(table_link, "<#E::table::service.db.schema.table>") + + dashboard_link = get_entity_link(Dashboard, fqn="service.dashboard") + self.assertEqual(dashboard_link, "<#E::dashboard::service.dashboard>") + + column_link = get_entity_link( + Table, fqn="service.db.schema.table", column_name="col" + ) + self.assertEqual( + column_link, "<#E::table::service.db.schema.table::columns::col>" + ) diff --git a/ingestion/tests/unit/test_helpers.py b/ingestion/tests/unit/test_helpers.py index d84f3a97360a..ded406b82923 100644 --- a/ingestion/tests/unit/test_helpers.py +++ b/ingestion/tests/unit/test_helpers.py @@ -15,6 +15,8 @@ from unittest import TestCase from metadata.generated.schema.entity.data.table import Column, DataType, Table +from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType +from metadata.generated.schema.type.basic import EntityLink from metadata.generated.schema.type.tagLabel import ( LabelType, State, @@ -24,6 +26,7 @@ from metadata.utils.helpers import ( clean_up_starting_ending_double_quotes_in_string, deep_size_of_dict, + find_suggestion, format_large_string_numbers, get_entity_tier_from_tags, is_safe_sql_query, @@ -154,3 +157,50 @@ def test_format_large_string_numbers(self): assert format_large_string_numbers(1000000) == "1.000M" assert format_large_string_numbers(1000000000) == "1.000B" assert format_large_string_numbers(1000000000000) == "1.000T" + + def test_find_suggestion(self): + """we can get one possible suggestion""" + suggestions = [ + Suggestion( + id=uuid.uuid4(), + type=SuggestionType.SuggestDescription, + entityLink=EntityLink(__root__="<#E::table::tableFQN>"), + description="something", + ), + Suggestion( + id=uuid.uuid4(), + type=SuggestionType.SuggestDescription, + entityLink=EntityLink(__root__="<#E::table::tableFQN::columns::col>"), + description="something", + ), + ] + + self.assertIsNone( + find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestTagLabel, + entity_link=..., + ) + ) + + self.assertIsNone( + find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestDescription, + entity_link=..., + ) + ) + + suggestion_table = find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestDescription, + entity_link=EntityLink(__root__="<#E::table::tableFQN>"), + ) + self.assertEqual(suggestion_table, suggestions[0]) + + suggestion_col = find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestDescription, + entity_link=EntityLink(__root__="<#E::table::tableFQN::columns::col>"), + ) + self.assertEqual(suggestion_col, suggestions[1]) diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py index 3b8a770ce209..dea9b0d48c12 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py @@ -22,6 +22,7 @@ from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( AppConfig, + PrivateConfig, ) from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( IngestionPipeline, @@ -75,7 +76,14 @@ def build_application_workflow_config( # We pass the generic class and let each app cast the actual object appConfig=AppConfig( __root__=application_pipeline_conf.appConfig.__root__, - ), + ) + if application_pipeline_conf.appConfig + else None, + appPrivateConfig=PrivateConfig( + __root__=application_pipeline_conf.appPrivateConfig.__root__ + ) + if application_pipeline_conf.appPrivateConfig + else None, workflowConfig=build_workflow_config_property(ingestion_pipeline), ingestionPipelineFQN=ingestion_pipeline.fullyQualifiedName.__root__, ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 3c733aa1312e..557942fa30a1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -60,6 +60,7 @@ import org.openmetadata.service.jdbi3.FeedRepository; import org.openmetadata.service.jdbi3.LineageRepository; import org.openmetadata.service.jdbi3.Repository; +import org.openmetadata.service.jdbi3.SuggestionRepository; import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.jdbi3.TokenRepository; import org.openmetadata.service.jdbi3.UsageRepository; @@ -88,7 +89,7 @@ public final class Entity { @Getter @Setter private static SystemRepository systemRepository; @Getter @Setter private static ChangeEventRepository changeEventRepository; @Getter @Setter private static SearchRepository searchRepository; - + @Getter @Setter private static SuggestionRepository suggestionRepository; // List of all the entities private static final Set ENTITY_LIST = new TreeSet<>(); @@ -194,6 +195,7 @@ public final class Entity { // Other entities public static final String EVENT_SUBSCRIPTION = "eventsubscription"; public static final String THREAD = "THREAD"; + public static final String SUGGESTION = "SUGGESTION"; public static final String WORKFLOW = "workflow"; // diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java index 2874d41d6a0b..ca145ca63d89 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java @@ -22,6 +22,7 @@ import javax.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration; import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; import org.openmetadata.schema.api.fernet.FernetConfiguration; @@ -93,6 +94,9 @@ public class OpenMetadataApplicationConfig extends Configuration { @JsonProperty("web") private OMWebConfiguration webConfiguration = new OMWebConfiguration(); + @JsonProperty("applications") + private AppsPrivateConfiguration appsPrivateConfiguration; + @Override public String toString() { return "catalogConfig{" diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index cfba9b91aee6..cd01c50638fb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -163,7 +163,8 @@ private void createAndBindIngestionPipeline( .withConfig( new ApplicationPipeline() .withSourcePythonClass(this.getApp().getSourcePythonClass()) - .withAppConfig(config))) + .withAppConfig(config) + .withAppPrivateConfig(this.getApp().getPrivateConfiguration()))) .withAirflowConfig( new AirflowConfig() .withScheduleInterval(this.getCronMapper().map(quartzCron).asString())) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index a05f64f31f8b..f50c977df569 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -4,7 +4,6 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; -import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; @@ -20,7 +19,6 @@ import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; -import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.util.FullyQualifiedName; @Slf4j @@ -31,12 +29,6 @@ public class DataInsightsApp extends AbstractNativeApplication { private static final String SERVICE_TYPE = "Metadata"; private static final String PIPELINE_DESCRIPTION = "OpenMetadata DataInsight Pipeline"; - @Override - public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { - super.init(app, dao, searchRepository); - LOG.info("Data Insights App is initialized"); - } - @Override public void install() { IngestionPipelineRepository ingestionPipelineRepository = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index c6f69e00af9e..8939e652ab7b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -208,6 +208,11 @@ public static String taskOperationNotAllowed(String user, String operations) { "Principal: CatalogPrincipal{name='%s'} operations %s not allowed", user, operations); } + public static String suggestionOperationNotAllowed(String user, String operations) { + return String.format( + "Principal: CatalogPrincipal{name='%s'} operations %s not allowed", user, operations); + } + public static String entityIsNotEmpty(String entityType) { return String.format("%s is not empty", entityType); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index e757945909e7..d7138985b110 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -329,6 +329,9 @@ public interface CollectionDAO { @CreateSqlObject DocStoreDAO docStoreDAO(); + @CreateSqlObject + SuggestionDAO suggestionDAO(); + interface DashboardDAO extends EntityDAO { @Override default String getTableName() { @@ -3531,8 +3534,7 @@ default List listAfterTsOrder(ListFilter filter, int limit, Inte } default int countOfTestCases(List testCaseIds) { - return countOfTestCases( - getTableName(), testCaseIds.stream().map(Object::toString).collect(Collectors.toList())); + return countOfTestCases(getTableName(), testCaseIds.stream().map(Object::toString).toList()); } @SqlQuery("SELECT count(*) FROM WHERE id IN ()") @@ -4320,4 +4322,81 @@ int listCount( @Define("mysqlCond") String mysqlCond, @Define("psqlCond") String psqlCond); } + + interface SuggestionDAO { + default String getTableName() { + return "suggestions"; + } + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json :: jsonb)", + connectionType = POSTGRES) + void insert(@BindFQN("fqnHash") String fullyQualifiedName, @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE suggestions SET json = :json where id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE suggestions SET json = (:json :: jsonb) where id = :id", + connectionType = POSTGRES) + void update(@BindUUID("id") UUID id, @Bind("json") String json); + + @SqlQuery("SELECT json FROM suggestions WHERE id = :id") + String findById(@BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM suggestions WHERE id = :id") + void delete(@BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM suggestions WHERE fqnHash = :fqnHash") + void deleteByFQN(@BindUUID("fqnHash") String fullyQualifiedName); + + @SqlQuery("SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit") + List list(@Bind("limit") int limit, @Define("condition") String condition); + + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM suggestions ", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM suggestions ", + connectionType = POSTGRES) + int listCount( + @Define("mysqlCond") String mysqlCond, @Define("postgresCond") String postgresCond); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT updatedAt, json FROM suggestions " + + "ORDER BY updatedAt DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY updatedAt", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT updatedAt, json FROM suggestions " + + "ORDER BY updatedAt DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY updatedAt", + connectionType = POSTGRES) + List listBefore( + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("before") String before); + + @ConnectionAwareSqlQuery( + value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", + connectionType = POSTGRES) + List listAfter( + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("after") String after); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index dd5f230d2e8b..f0d8fe4a04fb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -109,6 +109,7 @@ import org.openmetadata.schema.api.feed.ResolveTask; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.system.EntityError; @@ -1681,7 +1682,7 @@ protected List getExperts(T entity) { } public final EntityReference getOwner(EntityReference ref) { - return !supportsOwner ? null : Entity.getEntityReferenceById(ref.getType(), ref.getId(), ALL); + return !supportsOwner ? null : getFromEntityRef(ref.getId(), Relationship.OWNS, null, false); } public final void inheritDomain(T entity, Fields fields, EntityInterface parent) { @@ -1933,6 +1934,15 @@ public TaskWorkflow getTaskWorkflow(ThreadContext threadContext) { } } + public SuggestionRepository.SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { + return new SuggestionRepository.SuggestionWorkflow(suggestion); + } + + public EntityInterface applySuggestion( + EntityInterface entity, String childFQN, Suggestion suggestion) { + return entity; + } + public final void validateTaskThread(ThreadContext threadContext) { ThreadType threadType = threadContext.getThread().getType(); if (threadType != ThreadType.Task) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index e4d07730ce40..2f91705141b3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -488,7 +488,7 @@ public DeleteResponse deletePost(Thread thread, Post post, String userName @Transaction public DeleteResponse deleteThread(Thread thread, String deletedByUser) { deleteThreadInternal(thread.getId()); - LOG.info("{} deleted thread with id {}", deletedByUser, thread.getId()); + LOG.debug("{} deleted thread with id {}", deletedByUser, thread.getId()); return new DeleteResponse<>(thread, ENTITY_DELETED); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java new file mode 100644 index 000000000000..3bfd327433c2 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java @@ -0,0 +1,52 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.util.RestUtil.decodeCursor; + +import java.util.UUID; +import lombok.Builder; +import lombok.Getter; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.service.util.FullyQualifiedName; + +@Getter +@Builder +public class SuggestionFilter { + private SuggestionType suggestionType; + private SuggestionStatus suggestionStatus; + private UUID createdBy; + private String entityFQN; + private SuggestionRepository.PaginationType paginationType; + private String before; + private String after; + + public String getCondition(boolean includePagination) { + StringBuilder condition = new StringBuilder(); + condition.append("WHERE TRUE "); + if (suggestionType != null) { + condition.append(String.format(" AND type = '%s' ", suggestionType.value())); + } + if (suggestionStatus != null) { + condition.append(String.format(" AND status = '%s' ", suggestionStatus.value())); + } + if (entityFQN != null) { + condition.append( + String.format(" AND fqnHash = '%s' ", FullyQualifiedName.buildHash(entityFQN))); + } + if (createdBy != null) { + condition.append( + String.format( + " AND id in (select toId from entity_relationship where fromId = '%s') ", createdBy)); + } + if (paginationType != null && includePagination) { + String paginationCondition = + paginationType == SuggestionRepository.PaginationType.BEFORE + ? String.format(" AND updatedAt > %s ", Long.parseLong(decodeCursor(before))) + : String.format( + " AND updatedAt < %s ", + after != null ? Long.parseLong(decodeCursor(after)) : Long.MAX_VALUE); + condition.append(paginationCondition); + } + return condition.toString(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java new file mode 100644 index 000000000000..7b442f655416 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -0,0 +1,336 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.schema.type.EventType.SUGGESTION_ACCEPTED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_DELETED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_REJECTED; +import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.schema.type.Relationship.CREATED; +import static org.openmetadata.schema.type.Relationship.IS_ABOUT; +import static org.openmetadata.service.Entity.TEAM; +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.jdbi3.UserRepository.TEAMS_FIELD; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.ResourceRegistry; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.resources.feeds.SuggestionsResource; +import org.openmetadata.service.security.AuthorizationException; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; + +@Slf4j +@Repository +public class SuggestionRepository { + private final CollectionDAO dao; + + public enum PaginationType { + BEFORE, + AFTER + } + + public SuggestionRepository() { + this.dao = Entity.getCollectionDAO(); + Entity.setSuggestionRepository(this); + ResourceRegistry.addResource("suggestion", null, Entity.getEntityFields(Suggestion.class)); + } + + @Transaction + public Suggestion create(Suggestion suggestion) { + store(suggestion); + storeRelationships(suggestion); + return suggestion; + } + + @Transaction + public Suggestion update(Suggestion suggestion, String userName) { + suggestion.setUpdatedBy(userName); + dao.suggestionDAO().update(suggestion.getId(), JsonUtils.pojoToJson(suggestion)); + storeRelationships(suggestion); + return suggestion; + } + + @Transaction + public void store(Suggestion suggestion) { + // Insert a new Suggestion + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + dao.suggestionDAO().insert(entityLink.getEntityFQN(), JsonUtils.pojoToJson(suggestion)); + } + + @Transaction + public void storeRelationships(Suggestion suggestion) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + // Add relationship User -- created --> Suggestion relationship + dao.relationshipDAO() + .insert( + suggestion.getCreatedBy().getId(), + suggestion.getId(), + suggestion.getCreatedBy().getType(), + Entity.SUGGESTION, + CREATED.ordinal()); + + // Add field relationship for data asset - Suggestion -- entityLink ---> entity/entityField + dao.fieldRelationshipDAO() + .insert( + suggestion.getId().toString(), // from FQN + entityLink.getFullyQualifiedFieldValue(), // to FQN, + suggestion.getId().toString(), + entityLink.getFullyQualifiedFieldValue(), + Entity.SUGGESTION, // From type + entityLink.getFullyQualifiedFieldType(), // to Type + IS_ABOUT.ordinal(), + null); + } + + public Suggestion get(UUID id) { + return EntityUtil.validate(id, dao.suggestionDAO().findById(id), Suggestion.class); + } + + @Transaction + public RestUtil.DeleteResponse deleteSuggestion( + Suggestion suggestion, String deletedByUser) { + deleteSuggestionInternal(suggestion.getId()); + LOG.debug("{} deleted suggestion with id {}", deletedByUser, suggestion.getId()); + return new RestUtil.DeleteResponse<>(suggestion, SUGGESTION_DELETED); + } + + @Transaction + public RestUtil.DeleteResponse deleteSuggestionsForAnEntity( + EntityInterface entity, String deletedByUser) { + deleteSuggestionInternalForAnEntity(entity); + LOG.debug("{} deleted suggestions for the entity id {}", deletedByUser, entity.getId()); + return new RestUtil.DeleteResponse<>(entity, SUGGESTION_DELETED); + } + + @Transaction + public void deleteSuggestionInternal(UUID id) { + // Delete all the relationships to other entities + dao.relationshipDAO().deleteAll(id, Entity.SUGGESTION); + + // Delete all the field relationships to other entities + dao.fieldRelationshipDAO().deleteAllByPrefix(id.toString()); + + // Finally, delete the suggestion + dao.suggestionDAO().delete(id); + } + + @Transaction + public void deleteSuggestionInternalForAnEntity(EntityInterface entity) { + // Delete all the field relationships to other entities + dao.fieldRelationshipDAO().deleteAllByPrefix(entity.getId().toString()); + + // Finally, delete the suggestion + dao.suggestionDAO().deleteByFQN(entity.getFullyQualifiedName()); + } + + @Getter + public static class SuggestionWorkflow { + protected final Suggestion suggestion; + protected final MessageParser.EntityLink entityLink; + + SuggestionWorkflow(Suggestion suggestion) { + this.suggestion = suggestion; + this.entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); + } + + public EntityInterface acceptSuggestions( + EntityRepository repository, EntityInterface entityInterface) { + if (entityLink.getFieldName() != null) { + entityInterface = + repository.applySuggestion( + entityInterface, entityLink.getFullyQualifiedFieldValue(), suggestion); + return entityInterface; + } else { + if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { + List tags = new ArrayList<>(entityInterface.getTags()); + tags.addAll(suggestion.getTagLabels()); + entityInterface.setTags(tags); + return entityInterface; + } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { + entityInterface.setDescription(suggestion.getDescription()); + return entityInterface; + } else { + throw new WebApplicationException("Invalid suggestion Type"); + } + } + } + } + + public RestUtil.PutResponse acceptSuggestion( + UriInfo uriInfo, + Suggestion suggestion, + SecurityContext securityContext, + Authorizer authorizer) { + suggestion.setStatus(SuggestionStatus.Accepted); + acceptSuggestion(suggestion, securityContext, authorizer); + Suggestion updatedHref = SuggestionsResource.addHref(uriInfo, suggestion); + return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_ACCEPTED); + } + + protected void acceptSuggestion( + Suggestion suggestion, SecurityContext securityContext, Authorizer authorizer) { + String user = securityContext.getUserPrincipal().getName(); + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityInterface entity = + Entity.getEntity( + entityLink, suggestion.getType() == SuggestionType.SuggestTagLabel ? "tags" : "", ALL); + String origJson = JsonUtils.pojoToJson(entity); + SuggestionWorkflow suggestionWorkflow = getSuggestionWorkflow(suggestion); + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); + EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestions(repository, entity); + String updatedEntityJson = JsonUtils.pojoToJson(updatedEntity); + JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); + OperationContext operationContext = new OperationContext(entityLink.getEntityType(), patch); + authorizer.authorize( + securityContext, + operationContext, + new ResourceContext<>(entityLink.getEntityType(), entity.getId(), null)); + repository.patch(null, entity.getId(), user, patch); + suggestion.setStatus(SuggestionStatus.Accepted); + update(suggestion, user); + } + + public RestUtil.PutResponse rejectSuggestion( + UriInfo uriInfo, Suggestion suggestion, String user) { + suggestion.setStatus(SuggestionStatus.Rejected); + update(suggestion, user); + Suggestion updatedHref = SuggestionsResource.addHref(uriInfo, suggestion); + return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_REJECTED); + } + + public void checkPermissionsForUpdateSuggestion( + Suggestion suggestion, SecurityContext securityContext) { + String userName = securityContext.getUserPrincipal().getName(); + User user = Entity.getEntityByName(USER, userName, TEAMS_FIELD, NON_DELETED); + if (Boolean.FALSE.equals(user.getIsAdmin()) + && !userName.equalsIgnoreCase(suggestion.getCreatedBy().getName())) { + throw new AuthorizationException( + CatalogExceptionMessage.suggestionOperationNotAllowed(userName, "Update")); + } + } + + public void checkPermissionsForAcceptOrRejectSuggestion( + Suggestion suggestion, SuggestionStatus status, SecurityContext securityContext) { + String userName = securityContext.getUserPrincipal().getName(); + User user = Entity.getEntityByName(USER, userName, TEAMS_FIELD, NON_DELETED); + MessageParser.EntityLink about = MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityReference aboutRef = EntityUtil.validateEntityLink(about); + EntityReference ownerRef = Entity.getOwner(aboutRef); + List ownerTeamNames = new ArrayList<>(); + if (ownerRef != null) { + try { + User owner = + Entity.getEntityByName( + USER, ownerRef.getFullyQualifiedName(), TEAMS_FIELD, NON_DELETED); + ownerTeamNames = + owner.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + } catch (EntityNotFoundException e) { + Team owner = + Entity.getEntityByName(TEAM, ownerRef.getFullyQualifiedName(), "", NON_DELETED); + ownerTeamNames.add(owner.getFullyQualifiedName()); + } + } + + List userTeamNames = + user.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + + if (Boolean.FALSE.equals(user.getIsAdmin()) + && (ownerRef != null && !ownerRef.getName().equals(userName)) + && Collections.disjoint(userTeamNames, ownerTeamNames)) { + throw new AuthorizationException( + CatalogExceptionMessage.suggestionOperationNotAllowed(userName, status.value())); + } + } + + public SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); + return repository.getSuggestionWorkflow(suggestion); + } + + public int listCount(SuggestionFilter filter) { + String mySqlCondition = filter.getCondition(false); + String postgresCondition = filter.getCondition(false); + return dao.suggestionDAO().listCount(mySqlCondition, postgresCondition); + } + + public ResultList listBefore(SuggestionFilter filter, int limit, String before) { + int total = listCount(filter); + String mySqlCondition = filter.getCondition(true); + String postgresCondition = filter.getCondition(true); + List jsons = + dao.suggestionDAO() + .listBefore( + mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(before)); + List suggestions = getSuggestionList(jsons); + String beforeCursor = null; + String afterCursor; + if (suggestions.size() > limit) { + suggestions.remove(0); + beforeCursor = suggestions.get(0).getUpdatedAt().toString(); + } + afterCursor = + !suggestions.isEmpty() + ? suggestions.get(suggestions.size() - 1).getUpdatedAt().toString() + : null; + return new ResultList<>(suggestions, beforeCursor, afterCursor, total); + } + + public ResultList listAfter(SuggestionFilter filter, int limit, String after) { + int total = listCount(filter); + String mySqlCondition = filter.getCondition(true); + String postgresCondition = filter.getCondition(true); + List jsons = + dao.suggestionDAO() + .listAfter(mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(after)); + List suggestions = getSuggestionList(jsons); + String beforeCursor; + String afterCursor = null; + beforeCursor = after == null ? null : suggestions.get(0).getUpdatedAt().toString(); + if (suggestions.size() > limit) { + suggestions.remove(limit); + afterCursor = suggestions.get(limit - 1).getUpdatedAt().toString(); + } + return new ResultList<>(suggestions, beforeCursor, afterCursor, total); + } + + private List getSuggestionList(List jsons) { + List suggestions = new ArrayList<>(); + for (String json : jsons) { + Suggestion suggestion = JsonUtils.readValue(json, Suggestion.class); + suggestions.add(suggestion); + } + return suggestions; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 56a30143bf82..4f5493d1c716 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -45,6 +45,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.ws.rs.WebApplicationException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; @@ -58,6 +59,7 @@ import org.openmetadata.schema.api.feed.ResolveTask; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.tests.CustomMetric; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.Column; @@ -70,6 +72,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.JoinedWith; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.SuggestionType; import org.openmetadata.schema.type.SystemProfile; import org.openmetadata.schema.type.TableConstraint; import org.openmetadata.schema.type.TableData; @@ -737,6 +740,25 @@ public TaskWorkflow getTaskWorkflow(ThreadContext threadContext) { return super.getTaskWorkflow(threadContext); } + @Override + public Table applySuggestion(EntityInterface entity, String columnFQN, Suggestion suggestion) { + Table table = Entity.getEntity(TABLE, entity.getId(), "columns,tags", ALL); + for (Column col : table.getColumns()) { + if (col.getFullyQualifiedName().equals(columnFQN)) { + if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { + List tags = new ArrayList<>(col.getTags()); + tags.addAll(suggestion.getTagLabels()); + col.setTags(tags); + } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { + col.setDescription(suggestion.getDescription()); + } else { + throw new WebApplicationException("Invalid suggestion Type"); + } + } + } + return table; + } + @Override public String exportToCsv(String name, String user) throws IOException { // Validate table diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index b96ec1df2efb..34189dba23d3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -46,6 +46,8 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; @@ -97,6 +99,7 @@ public class AppResource extends EntityResource { public static final String COLLECTION_PATH = "v1/apps/"; private OpenMetadataApplicationConfig openMetadataApplicationConfig; + private AppsPrivateConfiguration privateConfiguration; private PipelineServiceClient pipelineServiceClient; static final String FIELDS = "owner"; private SearchRepository searchRepository; @@ -104,6 +107,7 @@ public class AppResource extends EntityResource { @Override public void initialize(OpenMetadataApplicationConfig config) { this.openMetadataApplicationConfig = config; + this.privateConfiguration = config.getAppsPrivateConfiguration(); this.pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); @@ -139,6 +143,7 @@ public void initialize(OpenMetadataApplicationConfig config) { // Schedule if (app.getScheduleType().equals(ScheduleType.Scheduled)) { + setAppRuntimeProperties(app); ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } } @@ -159,6 +164,32 @@ public static class AppRunList extends ResultList { /* Required for serde */ } + /** + * Load the apps' OM configuration and private parameters + */ + private void setAppRuntimeProperties(App app) { + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); + + if (privateConfiguration != null + && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { + for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { + if (app.getName().equals(appPrivateConfig.getName())) { + app.setPrivateConfiguration(appPrivateConfig.getParameters()); + } + } + } + } + + /** + * We don't want to store runtime information into the DB + */ + private void unsetAppRuntimeProperties(App app) { + app.setOpenMetadataServerConnection(null); + app.setPrivateConfiguration(null); + } + @GET @Operation( operationId = "listInstalledApplications", @@ -530,15 +561,13 @@ public Response create( create.getName(), new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information - app.setOpenMetadataServerConnection(null); + unsetAppRuntimeProperties(app); return create(uriInfo, securityContext, app); } @@ -571,10 +600,14 @@ public Response patchApplication( App app = repository.get(null, id, repository.getFields("bot,pipelines")); AppScheduler.getInstance().deleteScheduledApplication(app); Response response = patchInternal(uriInfo, securityContext, id, patch); + App updatedApp = (App) response.getEntity(); + setAppRuntimeProperties(updatedApp); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication( - (App) response.getEntity(), Entity.getCollectionDAO(), searchRepository); + updatedApp, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + unsetAppRuntimeProperties(updatedApp); return response; } @@ -604,9 +637,12 @@ public Response createOrUpdate( new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); AppScheduler.getInstance().deleteScheduledApplication(app); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + unsetAppRuntimeProperties(app); return createOrUpdate(uriInfo, securityContext, app); } @@ -684,9 +720,12 @@ public Response restoreApp( Response response = restoreEntity(uriInfo, securityContext, restore.getId()); if (response.getStatus() == Response.Status.OK.getStatusCode()) { App app = (App) response.getEntity(); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + unsetAppRuntimeProperties(app); } return response; } @@ -717,6 +756,7 @@ public Response scheduleApplication( @Context SecurityContext securityContext) { App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); @@ -752,9 +792,7 @@ public Response configureApplication( repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); // The application will have the updated appConfiguration we can use to run the `configure` // logic - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); try { ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App has been configured.").build(); @@ -788,6 +826,7 @@ public Response triggerApplicationRun( String name) { EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); + setAppRuntimeProperties(app); if (app.getAppType().equals(AppType.Internal)) { ApplicationHandler.triggerApplicationOnDemand( app, Entity.getCollectionDAO(), searchRepository); @@ -801,9 +840,7 @@ public Response triggerApplicationRun( IngestionPipeline ingestionPipeline = ingestionPipelineRepository.get( uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNER)); - ingestionPipeline.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + ingestionPipeline.setOpenMetadataServerConnection(app.getOpenMetadataServerConnection()); decryptOrNullify(securityContext, ingestionPipeline, app.getBot().getName(), true); ServiceEntityInterface service = Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); @@ -838,6 +875,7 @@ public Response deployApplicationFlow( String name) { EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); + setAppRuntimeProperties(app); if (app.getAppType().equals(AppType.Internal)) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); return Response.status(Response.Status.OK).entity("Application Deployed").build(); @@ -851,9 +889,7 @@ public Response deployApplicationFlow( ingestionPipelineRepository.get( uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNER)); - ingestionPipeline.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + ingestionPipeline.setOpenMetadataServerConnection(app.getOpenMetadataServerConnection()); decryptOrNullify(securityContext, ingestionPipeline, app.getBot().getName(), true); ServiceEntityInterface service = Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java new file mode 100644 index 000000000000..6e74397708d8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -0,0 +1,430 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.resources.feeds; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.schema.type.EventType.SUGGESTION_CREATED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_UPDATED; +import static org.openmetadata.service.util.RestUtil.CHANGE_CUSTOM_HEADER; + +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.List; +import java.util.UUID; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.feed.CreateSuggestion; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.sdk.exception.SuggestionException; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.SuggestionFilter; +import org.openmetadata.service.jdbi3.SuggestionRepository; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.tags.TagLabelUtil; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.PostResourceContext; +import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; +import org.openmetadata.service.util.UserUtil; + +@Path("/v1/suggestions") +@Tag( + name = "Suggestions", + description = + "Suggestions API supports ability to add suggestion for descriptions or tag labels for Entities.") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "suggestions") +public class SuggestionsResource { + public static final String COLLECTION_PATH = "/v1/suggestions/"; + private final SuggestionRepository dao; + private final Authorizer authorizer; + private final String INVALID_SUGGESTION_REQUEST = "INVALID_SUGGESTION_REQUEST"; + + public static void addHref(UriInfo uriInfo, List suggestions) { + if (uriInfo != null) { + suggestions.forEach(t -> addHref(uriInfo, t)); + } + } + + public static Suggestion addHref(UriInfo uriInfo, Suggestion suggestion) { + if (uriInfo != null) { + suggestion.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, suggestion.getId())); + } + return suggestion; + } + + public SuggestionsResource(Authorizer authorizer) { + this.dao = Entity.getSuggestionRepository(); + this.authorizer = authorizer; + } + + public static class SuggestionList extends ResultList { + /* Required for serde */ + } + + @GET + @Operation( + operationId = "listSuggestions", + summary = "List Suggestions", + description = + "Get a list of suggestions, optionally filtered by `entityLink` or `entityFQN`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of Suggestions", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SuggestionList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Parameter( + description = + "Limit the number of suggestions returned. (1 to 1000000, default = 10)") + @DefaultValue("10") + @Min(1) + @Max(1000000) + @QueryParam("limit") + int limitParam, + @Parameter( + description = "Returns list of threads before this cursor", + schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter( + description = "Returns list of threads after this cursor", + schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter(description = "Filter suggestions by entityFQN", schema = @Schema(type = "string")) + @QueryParam("entityFQN") + String entityFQN, + @Parameter( + description = + "Filter threads by user id or bot id. This filter requires a 'filterType' query param.", + schema = @Schema(type = "string")) + @QueryParam("userId") + UUID userId, + @Parameter( + description = + "Filter threads by whether they are accepted or rejected. By default status is OPEN.") + @DefaultValue("Open") + @QueryParam("status") + String status) { + RestUtil.validateCursors(before, after); + SuggestionFilter filter = + SuggestionFilter.builder() + .suggestionStatus(SuggestionStatus.valueOf(status)) + .entityFQN(entityFQN) + .createdBy(userId) + .paginationType( + before != null + ? SuggestionRepository.PaginationType.BEFORE + : SuggestionRepository.PaginationType.AFTER) + .before(before) + .after(after) + .build(); + ResultList suggestions; + if (before != null) { + suggestions = dao.listBefore(filter, limitParam, before); + } else { + suggestions = dao.listAfter(filter, limitParam, after); + } + addHref(uriInfo, suggestions.getData()); + return suggestions; + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getSuggestionByID", + summary = "Get a suggestion by Id", + description = "Get a suggestion by `Id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Suggestion", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse( + responseCode = "404", + description = "Suggestion for instance {id} is not found") + }) + public Suggestion get( + @Context UriInfo uriInfo, + @Parameter(description = "Id of the Thread", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + return addHref(uriInfo, dao.get(id)); + } + + @PUT + @Path("/{id}/accept") + @Operation( + operationId = "acceptSuggestion", + summary = "Close a task", + description = "Close a task without making any changes to the entity.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The task thread.", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response acceptSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the suggestion", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + Suggestion suggestion = dao.get(id); + dao.checkPermissionsForAcceptOrRejectSuggestion( + suggestion, SuggestionStatus.Accepted, securityContext); + return dao.acceptSuggestion(uriInfo, suggestion, securityContext, authorizer).toResponse(); + } + + @PUT + @Path("/{id}/reject") + @Operation( + operationId = "rejectSuggestion", + summary = "Reject a Suggestion", + description = "Close a Suggestion without making any changes to the entity.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Suggestion.", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response rejectSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the suggestion", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + Suggestion suggestion = dao.get(id); + dao.checkPermissionsForAcceptOrRejectSuggestion( + suggestion, SuggestionStatus.Rejected, securityContext); + return dao.rejectSuggestion(uriInfo, suggestion, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + + @PUT + @Path("/{id}") + @Operation( + operationId = "updateSuggestion", + summary = "Update a suggestion by `Id`.", + description = "Update an existing suggestion using JsonPatch.", + externalDocs = + @ExternalDocumentation( + description = "JsonPatch RFC", + url = "https://tools.ietf.org/html/rfc6902")) + public Response updateSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Suggestion", schema = @Schema(type = "string")) + @PathParam("id") + UUID id, + @Valid Suggestion suggestion) { + Suggestion origSuggestion = dao.get(id); + dao.checkPermissionsForUpdateSuggestion(origSuggestion, securityContext); + suggestion.setCreatedAt(origSuggestion.getCreatedAt()); + suggestion.setCreatedBy(origSuggestion.getCreatedBy()); + addHref(uriInfo, dao.update(suggestion, securityContext.getUserPrincipal().getName())); + return Response.created(suggestion.getHref()) + .entity(suggestion) + .header(CHANGE_CUSTOM_HEADER, SUGGESTION_UPDATED) + .build(); + } + + @POST + @Operation( + operationId = "createSuggestion", + summary = "Create a Suggestion", + description = + "Create a new Suggestion. A Suggestion is created about a data asset when a user suggests an update.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The thread", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateSuggestion create) { + Suggestion suggestion = getSuggestion(securityContext, create); + addHref(uriInfo, dao.create(suggestion)); + return Response.created(suggestion.getHref()) + .entity(suggestion) + .header(CHANGE_CUSTOM_HEADER, SUGGESTION_CREATED) + .build(); + } + + @DELETE + @Path("/{suggestionId}") + @Operation( + operationId = "deleteSuggestion", + summary = "Delete a Suggestion by Id", + description = "Delete an existing Suggestion and all its relationships.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "thread with {threadId} is not found"), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response deleteSuggestion( + @Context SecurityContext securityContext, + @Parameter( + description = "ThreadId of the thread to be deleted", + schema = @Schema(type = "string")) + @PathParam("suggestionId") + UUID suggestionId) { + // validate and get the thread + Suggestion suggestion = dao.get(suggestionId); + // delete thread only if the admin/bot/author tries to delete it + OperationContext operationContext = + new OperationContext(Entity.SUGGESTION, MetadataOperation.DELETE); + ResourceContextInterface resourceContext = + new PostResourceContext(suggestion.getCreatedBy().getName()); + authorizer.authorize(securityContext, operationContext, resourceContext); + return dao.deleteSuggestion(suggestion, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + + @DELETE + @Path("/{entityType}/name/{entityFQN}") + @Operation( + operationId = "deleteSuggestions", + summary = "Delete a Suggestions by entityFQN", + description = "Delete an existing Suggestions and all its relationships.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "thread with {threadId} is not found"), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response deleteSuggestions( + @Context SecurityContext securityContext, + @Parameter(description = "entity type", schema = @Schema(type = "string")) + @PathParam("entityType") + String entityType, + @Parameter(description = "fullyQualifiedName of entity", schema = @Schema(type = "string")) + @PathParam("entityFQN") + String entityFQN) { + // validate and get the thread + EntityInterface entity = + Entity.getEntityByName(entityType, entityFQN, "owner", Include.NON_DELETED); + // delete thread only if the admin/bot/author tries to delete it + OperationContext operationContext = + new OperationContext(Entity.SUGGESTION, MetadataOperation.DELETE); + ResourceContextInterface resourceContext = new PostResourceContext(entity.getOwner().getName()); + authorizer.authorize(securityContext, operationContext, resourceContext); + return dao.deleteSuggestionsForAnEntity(entity, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + + private Suggestion getSuggestion(SecurityContext securityContext, CreateSuggestion create) { + validate(create); + return new Suggestion() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withEntityLink(create.getEntityLink()) + .withType(create.getType()) + .withDescription(create.getDescription()) + .withTagLabels(create.getTagLabels()) + .withStatus(SuggestionStatus.Open) + .withCreatedBy(UserUtil.getUserOrBot(securityContext.getUserPrincipal().getName())) + .withCreatedAt(System.currentTimeMillis()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } + + private void validate(CreateSuggestion suggestion) { + if (suggestion.getEntityLink() == null) { + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's entityLink cannot be null."); + } + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + Entity.getEntityReferenceByName( + entityLink.getEntityType(), entityLink.getEntityFQN(), Include.NON_DELETED); + + if (suggestion.getType() == SuggestionType.SuggestDescription) { + if (suggestion.getDescription() == null || suggestion.getDescription().isEmpty()) { + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's description cannot be empty."); + } + } else if (suggestion.getType() == SuggestionType.SuggestTagLabel) { + if (suggestion.getTagLabels().isEmpty()) { + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's tag label's cannot be empty."); + } else { + for (TagLabel label : listOrEmpty(suggestion.getTagLabels())) { + TagLabelUtil.applyTagCommonFields(label); + } + } + } else { + throw new SuggestionException( + Response.Status.BAD_REQUEST, INVALID_SUGGESTION_REQUEST, "Invalid Suggestion Type."); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java index bb24c81ef28a..eee5948b8b09 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java @@ -16,6 +16,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthType.JWT; +import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import at.favre.lib.crypto.bcrypt.BCrypt; @@ -228,4 +229,14 @@ public static List getRoleForBot(String botName) { }; return listOf(RoleResource.getRole(botRole)); } + + public static EntityReference getUserOrBot(String name) { + EntityReference userOrBot; + try { + userOrBot = Entity.getEntityReferenceByName(Entity.USER, name, NON_DELETED); + } catch (EntityNotFoundException e) { + userOrBot = Entity.getEntityReferenceByName(Entity.BOT, name, NON_DELETED); + } + return userOrBot; + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java index faad55062af3..a2212ec285f9 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java @@ -34,6 +34,7 @@ import static org.openmetadata.service.exception.CatalogExceptionMessage.permissionNotAllowed; import static org.openmetadata.service.resources.EntityResourceTest.C1; import static org.openmetadata.service.resources.EntityResourceTest.USER1; +import static org.openmetadata.service.resources.EntityResourceTest.USER2_REF; import static org.openmetadata.service.resources.EntityResourceTest.USER_ADDRESS_TAG_LABEL; import static org.openmetadata.service.security.SecurityUtil.authHeaders; import static org.openmetadata.service.security.SecurityUtil.getPrincipalName; @@ -559,12 +560,15 @@ void post_invalidAnnouncement_400() throws IOException { } @Test - void put_resolveTaskByUser_description_200() throws IOException { + void put_resolveTaskByUser_description_200(TestInfo testInfo) throws IOException { + TableResourceTest tableResourceTest = new TableResourceTest(); + CreateTable createTable = tableResourceTest.createRequest(testInfo).withOwner(USER2_REF); + Table table = tableResourceTest.createAndCheckEntity(createTable, ADMIN_AUTH_HEADERS); // Create a task from User to User2 String about = String.format( "<#E::%s::%s::columns::%s::description>", - Entity.TABLE, TABLE.getFullyQualifiedName(), C1); + Entity.TABLE, table.getFullyQualifiedName(), C1); Thread taskThread = createTaskThread( USER.getName(), @@ -588,7 +592,7 @@ void put_resolveTaskByUser_description_200() throws IOException { // User2 who is assigned the task can resolve the task resolveTask(taskId, resolveTask, USER2_AUTH_HEADERS); - Table table = TABLE_RESOURCE_TEST.getEntity(TABLE.getId(), null, USER_AUTH_HEADERS); + table = TABLE_RESOURCE_TEST.getEntity(table.getId(), null, USER_AUTH_HEADERS); assertEquals("accepted", EntityUtil.getColumn(table, (C1)).getDescription()); taskThread = getTask(taskId, USER_AUTH_HEADERS); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java new file mode 100644 index 000000000000..ae90ddb55083 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java @@ -0,0 +1,498 @@ +package org.openmetadata.service.resources.feeds; + +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; +import static org.openmetadata.service.resources.EntityResourceTest.C1; +import static org.openmetadata.service.resources.EntityResourceTest.C2; +import static org.openmetadata.service.resources.EntityResourceTest.PERSONAL_DATA_TAG_LABEL; +import static org.openmetadata.service.resources.EntityResourceTest.PII_SENSITIVE_TAG_LABEL; +import static org.openmetadata.service.security.SecurityUtil.authHeaders; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.assertResponse; +import static org.openmetadata.service.util.TestUtils.assertResponseContains; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.feed.CreateSuggestion; +import org.openmetadata.schema.api.teams.CreateTeam; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.teams.TeamResourceTest; +import org.openmetadata.service.resources.teams.UserResourceTest; +import org.openmetadata.service.security.CatalogOpenIdAuthorizationRequestFilter; +import org.openmetadata.service.util.TestUtils; + +@Slf4j +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class SuggestionsResourceTest extends OpenMetadataApplicationTest { + public static Table TABLE; + public static Table TABLE2; + + public static Table TABLE_WITHOUT_OWNER; + public static String TABLE_LINK; + public static String TABLE2_LINK; + public static String TABLE_WITHOUT_OWNER_LINK; + public static String TABLE_COLUMN1_LINK; + public static String TABLE_COLUMN2_LINK; + public static List COLUMNS; + public static User USER; + public static String USER_LINK; + public static Map USER_AUTH_HEADERS; + public static User USER2; + public static Map USER2_AUTH_HEADERS; + public static Team TEAM; + public static Team TEAM2; + public static String TEAM_LINK; + + public static TableResourceTest TABLE_RESOURCE_TEST; + + @BeforeAll + public void setup(TestInfo test) throws IOException, URISyntaxException { + TABLE_RESOURCE_TEST = new TableResourceTest(); + TABLE_RESOURCE_TEST.setup(test); // Initialize TableResourceTest for using helper methods + + UserResourceTest userResourceTest = new UserResourceTest(); + USER2 = + userResourceTest.createEntity(userResourceTest.createRequest(test, 4), ADMIN_AUTH_HEADERS); + USER2_AUTH_HEADERS = authHeaders(USER2.getName()); + + CreateTable createTable = + TABLE_RESOURCE_TEST.createRequest(test).withOwner(TableResourceTest.USER1_REF); + TABLE = TABLE_RESOURCE_TEST.createAndCheckEntity(createTable, ADMIN_AUTH_HEADERS); + + TeamResourceTest teamResourceTest = new TeamResourceTest(); + CreateTeam createTeam = + teamResourceTest + .createRequest(test, 4) + .withDisplayName("Team2") + .withDescription("Team2 description") + .withUsers(List.of(USER2.getId())); + TEAM2 = teamResourceTest.createAndCheckEntity(createTeam, ADMIN_AUTH_HEADERS); + EntityReference TEAM2_REF = TEAM2.getEntityReference(); + + CreateTable createTable2 = TABLE_RESOURCE_TEST.createRequest(test); + createTable2.withName("table2").withOwner(TEAM2_REF); + TABLE2 = TABLE_RESOURCE_TEST.createAndCheckEntity(createTable2, ADMIN_AUTH_HEADERS); + + CreateTable createTable3 = TABLE_RESOURCE_TEST.createRequest(test); + createTable3.withName("table_without_owner").withOwner(null); + TABLE_WITHOUT_OWNER = + TABLE_RESOURCE_TEST.createAndCheckEntity(createTable3, ADMIN_AUTH_HEADERS); + + COLUMNS = + Collections.singletonList( + new Column().withName("column1").withDataType(ColumnDataType.BIGINT)); + TABLE_LINK = String.format("<#E::table::%s>", TABLE.getFullyQualifiedName()); + TABLE2_LINK = String.format("<#E::table::%s>", TABLE2.getFullyQualifiedName()); + TABLE_WITHOUT_OWNER_LINK = + String.format("<#E::table::%s>", TABLE_WITHOUT_OWNER.getFullyQualifiedName()); + TABLE_COLUMN1_LINK = + String.format("<#E::table::%s::columns::" + C1 + ">", TABLE.getFullyQualifiedName()); + TABLE_COLUMN2_LINK = + String.format("<#E::table::%s::columns::" + C2 + ">", TABLE.getFullyQualifiedName()); + + USER = TableResourceTest.USER1; + USER_LINK = String.format("<#E::user::%s>", USER.getFullyQualifiedName()); + USER_AUTH_HEADERS = authHeaders(USER.getName()); + + TEAM = TableResourceTest.TEAM1; + TEAM_LINK = String.format("<#E::team::%s>", TEAM.getFullyQualifiedName()); + } + + @Test + void post_suggestionWithoutEntityLink_4xx() { + // Create thread without addressed to entity in the request + CreateSuggestion create = create().withEntityLink(null); + assertResponse( + () -> createSuggestion(create, USER_AUTH_HEADERS), + BAD_REQUEST, + "Suggestion's entityLink cannot be null."); + } + + @Test + void post_suggestionWithInvalidAbout_4xx() { + // Create Suggestion without addressed to entity in the request + CreateSuggestion create = create().withEntityLink("<>"); // Invalid EntityLink + + String failureReason = + "[entityLink must match \"(?U)^<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#%]+>$\"]"; + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + + create.withEntityLink("<#E::>"); // Invalid EntityLink - missing entityType and entityId + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + + create.withEntityLink("<#E::table::>"); // Invalid EntityLink - missing entityId + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + + create.withEntityLink( + "<#E::table::tableName"); // Invalid EntityLink - missing closing bracket ">" + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + } + + @Test + void post_suggestionWithoutDescriptionOrTags_4xx() { + CreateSuggestion create = create().withDescription(null); + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), + BAD_REQUEST, + "Suggestion's description cannot be empty"); + } + + @Test + void post_feedWithNonExistentEntity_404() { + CreateSuggestion create = create().withEntityLink("<#E::table::invalidTableName>"); + assertResponse( + () -> createSuggestion(create, USER_AUTH_HEADERS), + NOT_FOUND, + entityNotFound(Entity.TABLE, "invalidTableName")); + } + + @Test + void post_validSuggestionAndList_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + create = create().withEntityLink(TABLE_LINK); + int suggestionCount = 1; + for (int i = 0; i < 10; i++) { + createAndCheck(create, USER_AUTH_HEADERS); + // List all the threads and make sure the number of threads increased by 1 + assertEquals( + ++suggestionCount, + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + } + SuggestionsResource.SuggestionList suggestionList = + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS); + assertEquals(suggestionCount, suggestionList.getPaging().getTotal()); + assertEquals(10, suggestionList.getData().size()); + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + 10, + null, + suggestionList.getPaging().getAfter(), + USER_AUTH_HEADERS); + assertEquals(1, suggestionList.getData().size()); + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + suggestionList.getPaging().getBefore(), + null, + USER_AUTH_HEADERS); + assertEquals(10, suggestionList.getData().size()); + create = create().withEntityLink(TABLE_COLUMN1_LINK); + createAndCheck(create, USER2_AUTH_HEADERS); + create = create().withEntityLink(TABLE_COLUMN2_LINK); + createAndCheck(create, USER2_AUTH_HEADERS); + assertEquals( + suggestionCount + 2, + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + create = create().withEntityLink(TABLE2_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + assertEquals( + suggestionCount + 2, + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + assertEquals( + 1, + listSuggestions(TABLE2.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + USER2.getId(), + null, + null, + null, + null); + assertEquals(2, suggestionList.getPaging().getTotal()); + assertNull(suggestionList.getPaging().getBefore()); + assertNull(suggestionList.getPaging().getAfter()); + create = create().withEntityLink(TABLE_WITHOUT_OWNER_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + assertEquals( + 1, + listSuggestions( + TABLE_WITHOUT_OWNER.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + /* deleteSuggestions("table", TABLE.getFullyQualifiedName(), USER_AUTH_HEADERS); + assertEquals( + 0, + listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS).getPaging().getTotal()); + deleteSuggestions("table", TABLE2.getFullyQualifiedName(), USER_AUTH_HEADERS); + assertEquals( + 0, + listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS).getPaging().getTotal());*/ + } + + @Test + void put_updateSuggestion_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + suggestion.setDescription("updated description"); + updateSuggestion(suggestion.getId(), suggestion, USER_AUTH_HEADERS); + Suggestion updatedSuggestion = getSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + assertEquals(suggestion.getId(), updatedSuggestion.getId()); + assertEquals(suggestion.getDescription(), updatedSuggestion.getDescription()); + updatedSuggestion.setDescription("updated description with different user"); + assertResponse( + () -> updateSuggestion(updatedSuggestion.getId(), updatedSuggestion, USER2_AUTH_HEADERS), + FORBIDDEN, + CatalogExceptionMessage.taskOperationNotAllowed(USER2.getName(), "Update")); + } + + @Test + @Order(1) + void put_acceptSuggestion_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + acceptSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + TableResourceTest tableResourceTest = new TableResourceTest(); + Table table = tableResourceTest.getEntity(TABLE.getId(), "", USER_AUTH_HEADERS); + assertEquals(suggestion.getDescription(), table.getDescription()); + suggestion = getSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + assertEquals(SuggestionStatus.Accepted, suggestion.getStatus()); + create = createTagSuggestion(); + Suggestion suggestion1 = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + assertResponse( + () -> acceptSuggestion(suggestion1.getId(), USER2_AUTH_HEADERS), + FORBIDDEN, + CatalogExceptionMessage.taskOperationNotAllowed(USER2.getName(), "Accepted")); + + acceptSuggestion(suggestion1.getId(), USER_AUTH_HEADERS); + table = tableResourceTest.getEntity(TABLE.getId(), "tags", USER_AUTH_HEADERS); + List expectedTags = new ArrayList<>(table.getTags()); + expectedTags.addAll(suggestion1.getTagLabels()); + validateAppliedTags(expectedTags, table.getTags()); + + create = createTagSuggestion().withEntityLink(TABLE_COLUMN1_LINK); + Suggestion suggestion2 = createSuggestion(create, USER_AUTH_HEADERS); + acceptSuggestion(suggestion2.getId(), USER_AUTH_HEADERS); + table = tableResourceTest.getEntity(TABLE.getId(), "columns,tags", USER_AUTH_HEADERS); + Column column = null; + for (Column col : table.getColumns()) { + if (col.getName().equals(C1)) { + column = col; + } + } + if (column != null) { + expectedTags = new ArrayList<>(column.getTags()); + expectedTags.addAll(suggestion2.getTagLabels()); + validateAppliedTags(expectedTags, column.getTags()); + } + String description = "Table without owner"; + create = create().withEntityLink(TABLE_WITHOUT_OWNER_LINK).withDescription(description); + Suggestion suggestion3 = createSuggestion(create, USER_AUTH_HEADERS); + acceptSuggestion(suggestion3.getId(), USER2_AUTH_HEADERS); + table = tableResourceTest.getEntity(TABLE_WITHOUT_OWNER.getId(), "", USER_AUTH_HEADERS); + assertEquals(description, table.getDescription()); + } + + @Test + @Order(2) + void put_rejectSuggestion_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + assertEquals( + 1, + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + rejectSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + suggestion = getSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + assertEquals(SuggestionStatus.Rejected, suggestion.getStatus()); + CreateSuggestion create1 = create().withEntityLink(TABLE2_LINK); + final Suggestion suggestion1 = createSuggestion(create1, USER2_AUTH_HEADERS); + Assertions.assertEquals(create1.getEntityLink(), suggestion1.getEntityLink()); + assertEquals( + 1, + listSuggestions(TABLE2.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + assertResponse( + () -> rejectSuggestion(suggestion1.getId(), USER_AUTH_HEADERS), + FORBIDDEN, + CatalogExceptionMessage.taskOperationNotAllowed(USER.getName(), "Rejected")); + rejectSuggestion(suggestion1.getId(), USER2_AUTH_HEADERS); + Suggestion suggestion2 = getSuggestion(suggestion1.getId(), USER2_AUTH_HEADERS); + assertEquals(SuggestionStatus.Rejected, suggestion2.getStatus()); + } + + public Suggestion createSuggestion(CreateSuggestion create, Map authHeaders) + throws HttpResponseException { + return TestUtils.post(getResource("suggestions"), create, Suggestion.class, authHeaders); + } + + public void updateSuggestion(UUID id, Suggestion update, Map authHeaders) + throws HttpResponseException { + TestUtils.put(getResource("suggestions/" + id), update, CREATED, authHeaders); + } + + public CreateSuggestion create() { + return new CreateSuggestion() + .withDescription("Update description") + .withType(SuggestionType.SuggestDescription) + .withEntityLink(TABLE_LINK); + } + + public CreateSuggestion createTagSuggestion() { + return new CreateSuggestion() + .withTagLabels(List.of(PII_SENSITIVE_TAG_LABEL, PERSONAL_DATA_TAG_LABEL)) + .withType(SuggestionType.SuggestTagLabel) + .withEntityLink(TABLE_LINK); + } + + public Suggestion getSuggestion(UUID id, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/" + id); + return TestUtils.get(target, Suggestion.class, authHeaders); + } + + public void acceptSuggestion(UUID id, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/" + id + "/accept"); + TestUtils.put(target, null, Response.Status.OK, authHeaders); + } + + public void rejectSuggestion(UUID id, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/" + id + "/reject"); + TestUtils.put(target, null, Response.Status.OK, authHeaders); + } + + public void deleteSuggestions( + String entityType, String entityFQN, Map authHeaders) + throws HttpResponseException { + WebTarget target = + getResource("suggestions/" + entityType + "/name/" + URLEncoder.encode(entityFQN)); + TestUtils.delete(target, authHeaders); + } + + public SuggestionsResource.SuggestionList listSuggestions( + String entityFQN, + Integer limit, + Map authHeaders, + UUID userId, + String suggestionType, + String status, + String before, + String after) + throws HttpResponseException { + WebTarget target = getResource("suggestions"); + target = entityFQN != null ? target.queryParam("entityFQN", entityFQN) : target; + target = userId != null ? target.queryParam("userId", userId) : target; + target = suggestionType != null ? target.queryParam("suggestionType", suggestionType) : target; + target = status != null ? target.queryParam("status", status) : target; + target = before != null ? target.queryParam("before", before) : target; + target = after != null ? target.queryParam("after", after) : target; + target = limit != null ? target.queryParam("limit", limit) : target; + return TestUtils.get(target, SuggestionsResource.SuggestionList.class, authHeaders); + } + + public SuggestionsResource.SuggestionList listSuggestions( + String entityFQN, Integer limit, String before, String after, Map authHeaders) + throws HttpResponseException { + return listSuggestions(entityFQN, limit, authHeaders, null, null, null, before, after); + } + + public Suggestion createAndCheck(CreateSuggestion create, Map authHeaders) + throws HttpResponseException { + // Validate returned thread from POST + Suggestion suggestion = createSuggestion(create, authHeaders); + validateSuggestion( + suggestion, + create.getEntityLink(), + authHeaders.get(CatalogOpenIdAuthorizationRequestFilter.X_AUTH_PARAMS_EMAIL_HEADER), + create.getType(), + create.getDescription(), + create.getTagLabels()); + + // Validate returned thread again from GET + Suggestion getSuggestion = getSuggestion(suggestion.getId(), authHeaders); + validateSuggestion( + getSuggestion, + create.getEntityLink(), + authHeaders.get(CatalogOpenIdAuthorizationRequestFilter.X_AUTH_PARAMS_EMAIL_HEADER), + create.getType(), + create.getDescription(), + create.getTagLabels()); + return suggestion; + } + + private void validateSuggestion( + Suggestion suggestion, + String entityLink, + String createdBy, + SuggestionType type, + String description, + List tags) { + assertNotNull(suggestion.getId()); + assertEquals(entityLink, suggestion.getEntityLink()); + assertEquals(createdBy, suggestion.getCreatedBy().getName()); + assertEquals(type, suggestion.getType()); + assertEquals(tags, suggestion.getTagLabels()); + assertEquals(description, suggestion.getDescription()); + } + + private void validateAppliedTags(List appliedTags, List entityTags) { + for (TagLabel tagLabel : appliedTags) { + Assertions.assertTrue(entityTags.contains(tagLabel)); + } + } +} diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java new file mode 100644 index 000000000000..76d31abf2271 --- /dev/null +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java @@ -0,0 +1,40 @@ +package org.openmetadata.sdk.exception; + +import javax.ws.rs.core.Response; + +public class SuggestionException extends WebServiceException { + private static final String BY_NAME_MESSAGE = + "Search Index Not Found Exception [%s] due to [%s]."; + private static final String ERROR_TYPE = "SUGGESTION_EXCEPTION"; + + public SuggestionException(String message) { + super(Response.Status.INTERNAL_SERVER_ERROR, ERROR_TYPE, message); + } + + private SuggestionException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public SuggestionException(Response.Status status, String errorType, String message) { + super(status, errorType, message); + } + + public static SuggestionException byMessage( + String name, String errorMessage, Response.Status status) { + return new SuggestionException(status, buildMessageByName(name, errorMessage)); + } + + public static SuggestionException byMessage( + String name, String errorType, String errorMessage, Response.Status status) { + return new SuggestionException(status, errorType, buildMessageByName(name, errorMessage)); + } + + public static SuggestionException byMessage(String name, String errorMessage) { + return new SuggestionException( + Response.Status.BAD_REQUEST, buildMessageByName(name, errorMessage)); + } + + private static String buildMessageByName(String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, name, errorMessage); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json b/openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json new file mode 100644 index 000000000000..91b4b8c4a958 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json @@ -0,0 +1,31 @@ +{ + "$id": "https://open-metadata.org/schema/api/feed/createSuggestion.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateSuggestionRequest", + "description": "Create Suggestion request", + "type": "object", + "properties": { + "description": { + "description": "Message in Markdown format. See markdown support for more details.", + "type": "string" + }, + "tagLabels": { + "description": "Tags or Glossary Terms.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "type": { + "$ref": "../../entity/feed/suggestion.json#/definitions/suggestionType" + }, + "entityLink": { + "description": "Data asset about which this thread is created for with format <#E::{entities}::{entityName}::{field}::{fieldValue}.", + "$ref": "../../type/basic.json#/definitions/entityLink" + } + }, + "oneOf": [{"required": ["suggestionType", "entityLink", "description"]}, + {"required": ["suggestionType", "entityLink","tagLabels"]}], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json new file mode 100644 index 000000000000..f0e1672ea96a --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json @@ -0,0 +1,43 @@ +{ + "$id": "https://open-metadata.org/schema/entity/configuration/appsPrivateConfiguration.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AppsPrivateConfiguration", + "description": "This schema defines a list of configurations for the Application Framework", + "type": "object", + "javaType": "org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration", + "definitions": { + "appPrivateConfig": { + "type": "object", + "javaType": "org.openmetadata.schema.api.configuration.apps.AppPrivateConfig", + "title": "AppPrivateConfig", + "description": "Single Application Configuration Definition", + "properties": { + "name": { + "type": "string", + "description": "Application Name" + }, + "parameters": { + "javaType": "org.openmetadata.schema.api.configuration.apps.Parameters", + "description": "Parameters to initialize the Applications.", + "type": "object", + "additionalProperties": { + ".{1,}": { "type": "string" } + } + } + }, + "required": ["name", "parameters"], + "additionalProperties": false + } + }, + "properties": { + "appsPrivateConfiguration": { + "description": "List of configuration for apps", + "type": "array", + "items": { + "$ref": "#/definitions/appPrivateConfig" + } + } + }, + "required": ["appsPrivateConfiguration"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index 13651561eeae..2546da2a2bab 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -199,6 +199,10 @@ "description": "Application Configuration object.", "$ref": "./configuration/applicationConfig.json#/definitions/appConfig" }, + "privateConfiguration": { + "description": "Application Private configuration loaded at runtime.", + "$ref": "./configuration/applicationConfig.json#/definitions/privateConfig" + }, "pipelines": { "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", "$ref": "../../type/entityReferenceList.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 2a85c58f4d1c..d963dc9be84a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -20,6 +20,13 @@ "$ref": "internal/searchIndexingAppConfig.json" } ] + }, + "privateConfig": { + "oneOf": [ + { + "$ref": "./private/external/metaPilotAppPrivateConfig.json" + } + ] } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json index c9f085d19a4d..4ac75978557f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -48,25 +48,6 @@ "$ref": "#/definitions/metaPilotAppType", "default": "MetaPilot" }, - "waiiInstance": { - "title": "WAII Instance", - "description": "WAII API host URL", - "type": "string", - "format": "URI", - "default": "https://tweakit.waii.ai/api/" - }, - "collateURL": { - "title": "Collate URL", - "description": "Collate Server public URL. WAII will use this information to interact with the server. E.g., https://sandbox.getcollate.io", - "type": "string", - "format": "URI" - }, - "token": { - "title": "WAII API Token", - "description": "WAII API Token", - "type": "string", - "format": "password" - }, "serviceDatabases": { "title": "Service Databases", "description": "Services and Databases configured to get the descriptions from.", @@ -76,6 +57,5 @@ } } }, - "additionalProperties": false, - "required": ["token"] + "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json new file mode 100644 index 000000000000..61b26a069ce8 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json @@ -0,0 +1,31 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/private/external/metaPilotAppConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MetaPilotAppPrivateConfig", + "description": "PRivate Configuration for the MetaPilot External Application.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppPrivateConfig", + "properties": { + "waiiInstance": { + "title": "WAII Instance", + "description": "WAII API host URL", + "type": "string", + "format": "URI", + "default": "https://tweakit.waii.ai/api/" + }, + "collateURL": { + "title": "Collate URL", + "description": "Collate Server public URL. WAII will use this information to interact with the server. E.g., https://sandbox.getcollate.io", + "type": "string", + "format": "URI" + }, + "token": { + "title": "WAII API Token", + "description": "WAII API Token", + "type": "string", + "format": "password" + } + }, + "additionalProperties": false, + "required": ["waiiInstance", "collateURL", "token"] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json b/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json new file mode 100644 index 000000000000..271b7731d1da --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json @@ -0,0 +1,99 @@ +{ + "$id": "https://open-metadata.org/schema/entity/feed/thread.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Suggestion", + "description": "This schema defines the Suggestion entity. A suggestion can be applied to an asset to give the owner context about possible changes or improvements to descriptions, tags,...", + "type": "object", + "definitions": { + "suggestionType": { + "javaType": "org.openmetadata.schema.type.SuggestionType", + "description": "Type of a Suggestion.", + "type": "string", + "enum": [ + "SuggestDescription", + "SuggestTagLabel" + ], + "javaEnums": [ + { + "name": "SuggestDescription" + }, + { + "name": "SuggestTagLabel" + } + ] + }, + "suggestionStatus": { + "javaType": "org.openmetadata.schema.type.SuggestionStatus", + "type": "string", + "description": "Status of a Suggestion.", + "enum": [ + "Open", + "Accepted", + "Rejected" + ], + "javaEnums": [ + { + "name": "Open" + }, + { + "name": "Accepted" + }, + { + "name": "Rejected" + } + ], + "default": "Open" + } + }, + "properties": { + "id": { + "description": "Unique identifier that identifies an entity instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "type": { + "$ref": "#/definitions/suggestionType" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "entityLink": { + "description": "Data asset about which this thread is created for with format <#E::{entities}::{entityName}::{field}::{fieldValue}.", + "$ref": "../../type/basic.json#/definitions/entityLink" + }, + "createdAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "createdBy": { + "description": "User or Bot who made the suggestion.", + "$ref": "../../type/entityReference.json" + }, + "updatedAt": { + "description": "Last update time corresponding to the update version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User or Bot who updated the suggestion.", + "type": "string" + }, + "status": { + "$ref": "#/definitions/suggestionStatus" + }, + "description": { + "description": "The main message of the thread in Markdown format.", + "type": "string" + }, + "tagLabels": { + "description": "Tags or Glossary Terms.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + } + }, + "oneOf": [{"required": ["id", "entityLink", "description", "suggestionType"]}, + {"required": ["id", "entityLink", "tagLabels", "suggestionType"]}], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json index 2ee79748248c..bd37ce5d74e2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json @@ -18,6 +18,10 @@ "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/appConfig", "description": "External Application configuration" }, + "appPrivateConfig": { + "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/privateConfig", + "description": "External Application Private configuration" + }, "ingestionPipelineFQN": { "description": "Fully qualified name of ingestion pipeline, used to identify the current ingestion pipeline", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json index 72248e41ce4c..dc029f796a96 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json @@ -25,6 +25,10 @@ "appConfig": { "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/appConfig", "description": "Application configuration" + }, + "appPrivateConfig": { + "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/privateConfig", + "description": "Application private configuration" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json b/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json index e7cefd016f56..ae0bd94aec56 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json @@ -19,6 +19,11 @@ "postUpdated", "taskResolved", "taskClosed", - "logicalTestCaseAdded" + "logicalTestCaseAdded", + "suggestionCreated", + "suggestionUpdated", + "suggestionAccepted", + "suggestionRejected", + "suggestionDeleted" ] } diff --git a/openmetadata-ui/src/main/resources/ui/src/App.tsx b/openmetadata-ui/src/main/resources/ui/src/App.tsx index 3579fb6730f9..3c923647cdc2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/App.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/App.tsx @@ -25,6 +25,7 @@ import DomainProvider from './components/Domain/DomainProvider/DomainProvider'; import { EntityExportModalProvider } from './components/Entity/EntityExportModalProvider/EntityExportModalProvider.component'; import ErrorBoundary from './components/ErrorBoundary/ErrorBoundary'; import GlobalSearchProvider from './components/GlobalSearchProvider/GlobalSearchProvider'; +import MetaPilotProvider from './components/MetaPilot/MetaPilotProvider/MetaPilotProvider'; import PermissionProvider from './components/PermissionProvider/PermissionProvider'; import TourProvider from './components/TourProvider/TourProvider'; import WebAnalyticsProvider from './components/WebAnalytics/WebAnalyticsProvider'; @@ -54,9 +55,11 @@ const App: FC = ({ routeElements }) => { - - - + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg index 0ed0c62850cc..64708284d223 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg @@ -1,22 +1,20 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg index 0beefce29883..5aeecd41f534 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg @@ -1,22 +1,14 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg new file mode 100644 index 000000000000..ad5a5f7d5193 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg index 840ece058f4a..45d0613f5a8b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg @@ -1,21 +1,23 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg new file mode 100644 index 000000000000..7cc52b60ab00 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx index 804782833c82..92f6b667cb18 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx @@ -21,6 +21,8 @@ import SignUpPage from '../../pages/SignUp/SignUpPage'; import Appbar from '../AppBar/Appbar'; import AuthenticatedAppRouter from '../AppRouter/AuthenticatedAppRouter'; import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; +import { useMetaPilotContext } from '../MetaPilot/MetaPilotProvider/MetaPilotProvider'; +import MetaPilotSidebar from '../MetaPilot/MetaPilotSidebar/MetaPilotSidebar'; import LeftSidebar from '../MyData/LeftSidebar/LeftSidebar.component'; import './app-container.less'; @@ -28,6 +30,7 @@ const AppContainer = () => { const { i18n } = useTranslation(); const { Header, Sider, Content } = Layout; const { currentUser } = useAuthContext(); + const { suggestionsVisible } = useMetaPilotContext(); const isDirectionRTL = useMemo(() => i18n.dir() === 'rtl', [i18n]); @@ -49,9 +52,16 @@ const AppContainer = () => {
- - - + + + + + {suggestionsVisible && ( + + + + )} + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx index b601ffea562d..e73d72732c8f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx @@ -43,6 +43,7 @@ import { ReactComponent as IconDropdown } from '../../../assets/svg/menu.svg'; import Loader from '../../../components/Loader/Loader'; import PageLayoutV1 from '../../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../../components/TabsLabel/TabsLabel.component'; +import { APP_UI_SCHEMA } from '../../../constants/Applications.constant'; import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { GlobalSettingOptions } from '../../../constants/GlobalSettings.constants'; import { ServiceCategory } from '../../../enums/service.enum'; @@ -325,6 +326,7 @@ const AppDetails = () => { okText={t('label.submit')} schema={jsonSchema} serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + uiSchema={APP_UI_SCHEMA} validator={validator} onCancel={noop} onSubmit={onConfigSave} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx index 50c974cf9bfe..6ddafbb8e08f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx @@ -28,7 +28,7 @@ const AppLogo = ({ const Icon = data.ReactComponent as React.ComponentType< JSX.IntrinsicElements['svg'] >; - setAppLogo(); + setAppLogo(); }); } else { setAppLogo(logo); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx index 3d562cefd175..1c36b0b6dc24 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx @@ -53,6 +53,10 @@ jest.mock( } ); +jest.mock('../../common/EntityDescription/DescriptionV1', () => + jest.fn().mockImplementation(() =>
DescriptionV1
) +); + const mockProps = { glossary: mockedGlossaries[0], glossaryTerms: [], diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx index d598dcf96a50..16546b1a5562 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx @@ -23,10 +23,10 @@ import { getEntityVersionByField, getEntityVersionTags, } from '../../../../utils/EntityVersionUtils'; +import DescriptionV1 from '../../../common/EntityDescription/DescriptionV1'; import { OperationPermission } from '../../../PermissionProvider/PermissionProvider.interface'; import TagsContainerV2 from '../../../Tag/TagsContainerV2/TagsContainerV2'; import { DisplayType } from '../../../Tag/TagsViewer/TagsViewer.interface'; -import DescriptionV1 from '../../../common/EntityDescription/DescriptionV1'; import GlossaryDetailsRightPanel from '../../GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component'; import { GlossaryUpdateConfirmationModal } from '../../GlossaryUpdateConfirmationModal/GlossaryUpdateConfirmationModal'; import GlossaryTermReferences from './GlossaryTermReferences'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx new file mode 100644 index 000000000000..4dc382a80feb --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx @@ -0,0 +1,95 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; +import { Button, Card, Space, Typography } from 'antd'; +import React, { useLayoutEffect } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as SuggestionsIcon } from '../../../assets/svg/ic-suggestions.svg'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; +import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import { useMetaPilotContext } from '../MetaPilotProvider/MetaPilotProvider'; +import { SuggestionAction } from '../MetaPilotProvider/MetaPilotProvider.interface'; +import { MetaPilotDescriptionAlertProps } from './MetaPilotDescriptionAlert.interface'; + +const MetaPilotDescriptionAlert = ({ + showHeading = true, + suggestion, + hasEditAccess = false, +}: MetaPilotDescriptionAlertProps) => { + const { t } = useTranslation(); + const { onUpdateActiveSuggestion, acceptRejectSuggestion } = + useMetaPilotContext(); + + useLayoutEffect(() => { + const element = document.querySelector('.suggested-description-card'); + if (element) { + element.scrollIntoView({ block: 'center', behavior: 'smooth' }); + } + }, []); + + if (!suggestion) { + return null; + } + + return ( + + {showHeading && ( + + + {t('label.description')} + + + + )} + +
+
+ + + {t('label.metapilot-suggested-description')} + +
+ onUpdateActiveSuggestion(undefined)} /> +
+ + {hasEditAccess && ( +
+ + +
+ )} + + + ); +}; + +export default MetaPilotDescriptionAlert; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts new file mode 100644 index 000000000000..b47460693138 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Suggestion } from '../../../generated/entity/feed/suggestion'; + +export interface MetaPilotDescriptionAlertProps { + showHeading?: boolean; + suggestion: Suggestion; + hasEditAccess?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts new file mode 100644 index 000000000000..3f853076ab7e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ReactNode } from 'react'; +import { Suggestion } from '../../../generated/entity/feed/suggestion'; + +export interface MetaPilotContextType { + suggestionsVisible: boolean; + isMetaPilotEnabled: boolean; + onToggleSuggestionsVisible: (state: boolean) => void; + activeSuggestion?: Suggestion; + suggestions: Suggestion[]; + loading: boolean; + entityFqn: string; + refreshEntity: (() => void) | undefined; + onUpdateActiveSuggestion: (suggestion?: Suggestion) => void; + fetchSuggestions: (entityFqn: string) => void; + acceptRejectSuggestion: ( + suggestion: Suggestion, + action: SuggestionAction + ) => void; + onUpdateEntityFqn: (entityFqn: string) => void; + resetMetaPilot: () => void; + initMetaPilot: (entityFqn: string, refreshEntity?: () => void) => void; +} + +export interface MetaPilotContextProps { + children: ReactNode; +} + +export enum SuggestionAction { + Accept = 'accept', + Reject = 'reject', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx new file mode 100644 index 000000000000..ebc3ed89eeaf --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx @@ -0,0 +1,196 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button } from 'antd'; +import { AxiosError } from 'axios'; +import { isEmpty } from 'lodash'; +import React, { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; +import { Suggestion } from '../../../generated/entity/feed/suggestion'; +import { Include } from '../../../generated/type/include'; +import { getApplicationByName } from '../../../rest/applicationAPI'; +import { + getMetaPilotSuggestionsList, + updateSuggestionStatus, +} from '../../../rest/suggestionsAPI'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { usePermissionProvider } from '../../PermissionProvider/PermissionProvider'; +import { + MetaPilotContextProps, + MetaPilotContextType, + SuggestionAction, +} from './MetaPilotProvider.interface'; + +export const MetaPilotContext = createContext({} as MetaPilotContextType); + +const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { + const { t } = useTranslation(); + const [suggestionsVisible, setSuggestionsVisible] = useState(false); + const [isMetaPilotEnabled, setIsMetaPilotEnabled] = useState(false); + const [activeSuggestion, setActiveSuggestion] = useState< + Suggestion | undefined + >(); + const [entityFqn, setEntityFqn] = useState(''); + const [suggestions, setSuggestions] = useState([]); + const [loading, setLoading] = useState(false); + const [refreshEntity, setRefreshEntity] = useState<() => void>(); + const { permissions } = usePermissionProvider(); + + const fetchMetaPilotAppDetails = useCallback(async () => { + try { + await getApplicationByName('MetaPilotApplication', { + fields: 'owner', + include: Include.All, + }); + setIsMetaPilotEnabled(true); + } catch (error) { + setIsMetaPilotEnabled(false); + } + }, []); + + const fetchSuggestions = useCallback(async (entityFQN: string) => { + setLoading(true); + try { + const res = await getMetaPilotSuggestionsList({ + entityFQN, + }); + setSuggestions(res.data); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } finally { + setLoading(false); + } + }, []); + + const acceptRejectSuggestion = useCallback( + async (suggestion: Suggestion, status: SuggestionAction) => { + try { + await updateSuggestionStatus(suggestion, status); + await fetchSuggestions(entityFqn); + + setActiveSuggestion(undefined); + if (status === SuggestionAction.Accept) { + refreshEntity?.(); + } + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [entityFqn, refreshEntity] + ); + + const onToggleSuggestionsVisible = useCallback((state: boolean) => { + setSuggestionsVisible(state); + }, []); + + const onUpdateActiveSuggestion = useCallback((suggestion?: Suggestion) => { + setActiveSuggestion(suggestion); + }, []); + + const onUpdateEntityFqn = useCallback((entityFqn: string) => { + setEntityFqn(entityFqn); + }, []); + + const resetMetaPilot = useCallback(() => { + setSuggestionsVisible(false); + setActiveSuggestion(undefined); + setEntityFqn(''); + }, []); + + const initMetaPilot = useCallback( + (entityFqn: string, refreshEntity?: () => void) => { + setEntityFqn(entityFqn); + setRefreshEntity(() => refreshEntity); + }, + [] + ); + + useEffect(() => { + if (isMetaPilotEnabled && !isEmpty(entityFqn)) { + fetchSuggestions(entityFqn); + } + }, [isMetaPilotEnabled, entityFqn]); + + useEffect(() => { + if (!isEmpty(permissions)) { + fetchMetaPilotAppDetails(); + } + }, [permissions]); + + const metaPilotContextObj = useMemo(() => { + return { + suggestionsVisible, + isMetaPilotEnabled, + suggestions, + activeSuggestion, + entityFqn, + loading, + refreshEntity, + onToggleSuggestionsVisible, + onUpdateEntityFqn, + onUpdateActiveSuggestion, + fetchSuggestions, + acceptRejectSuggestion, + initMetaPilot, + resetMetaPilot, + }; + }, [ + suggestionsVisible, + isMetaPilotEnabled, + suggestions, + activeSuggestion, + entityFqn, + loading, + refreshEntity, + onToggleSuggestionsVisible, + onUpdateEntityFqn, + onUpdateActiveSuggestion, + fetchSuggestions, + acceptRejectSuggestion, + initMetaPilot, + resetMetaPilot, + ]); + + return ( + + {children} + {isMetaPilotEnabled && ( +
+
+ )} + + ); +}; + +export const useMetaPilotContext = () => useContext(MetaPilotContext); + +export default MetaPilotProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx new file mode 100644 index 000000000000..ff5b9c54cbd5 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CloseOutlined } from '@ant-design/icons'; +import { Card, Drawer, Typography } from 'antd'; +import React, { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; +import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; +import { Suggestion } from '../../../generated/entity/feed/suggestion'; +import EntityLink from '../../../utils/EntityLink'; +import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; +import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import Loader from '../../Loader/Loader'; +import { useMetaPilotContext } from '../MetaPilotProvider/MetaPilotProvider'; +import './meta-pilot-sidebar.less'; + +const MetaPilotSidebar = () => { + const { t } = useTranslation(); + const { + onUpdateActiveSuggestion, + suggestions, + loading, + suggestionsVisible, + onToggleSuggestionsVisible, + } = useMetaPilotContext(); + + const descriptionsView = useMemo(() => { + return suggestions.map((item: Suggestion) => { + return ( + onUpdateActiveSuggestion(item)}> + + + {EntityLink.getTableColumnName(item.entityLink) ?? + EntityLink.getEntityFqn(item.entityLink)} + + + ); + }); + }, [suggestions]); + + return ( + onToggleSuggestionsVisible(false)} + /> + } + getContainer={false} + headerStyle={{ padding: 16 }} + mask={false} + open={suggestionsVisible} + title={ +
+ + + {t('label.metapilot')} + +
+ } + width={340}> + {loading ? ( + + ) : ( + <> + {suggestions?.length === 0 && ( + + )} + {suggestions.length > 0 && ( + <> + + {t('label.suggested-description-plural')} + + {descriptionsView} + + )} + + )} + + ); +}; + +export default MetaPilotSidebar; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less new file mode 100644 index 000000000000..7bfa1b0da7bc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import url('../../../styles/variables.less'); + +.meta-pilot-drawer { + .ant-drawer-content { + background-color: #f6f9fd; + } + .ant-drawer-body { + padding: 0 16px 16px; + } + .suggestion-card { + border-radius: 10px; + cursor: pointer; + &:hover { + p { + color: @primary-color; + } + } + } + .ant-drawer-content-wrapper { + box-shadow: none !important; + } +} + +.suggested-description-card { + border-radius: 10px !important; + border-color: @grey-4 !important; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx index b2c0a76f0cf9..7ac871d623f1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx @@ -230,6 +230,7 @@ const SchemaTable = ({ columnData={{ fqn: record.fullyQualifiedName ?? '', field: record.description, + record, }} entityFqn={decodedEntityFqn} entityType={EntityType.TABLE} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx index 8c3ae8427839..0dfcfec65905 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx @@ -12,13 +12,19 @@ */ import { Button, Space } from 'antd'; -import React from 'react'; +import { isEmpty } from 'lodash'; +import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../assets/svg/edit-new.svg'; import RichTextEditorPreviewer from '../../components/common/RichTextEditor/RichTextEditorPreviewer'; import { DE_ACTIVE_COLOR } from '../../constants/constants'; import { EntityField } from '../../constants/Feeds.constants'; +import { EntityType } from '../../enums/entity.enum'; import EntityTasks from '../../pages/TasksPage/EntityTasks/EntityTasks.component'; +import EntityLink from '../../utils/EntityLink'; +import { getEntityFeedLink } from '../../utils/EntityUtils'; +import MetaPilotDescriptionAlert from '../MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component'; +import { useMetaPilotContext } from '../MetaPilot/MetaPilotProvider/MetaPilotProvider'; import { TableDescriptionProps } from './TableDescription.interface'; const TableDescription = ({ @@ -32,6 +38,48 @@ const TableDescription = ({ onThreadLinkSelect, }: TableDescriptionProps) => { const { t } = useTranslation(); + const { activeSuggestion, suggestions } = useMetaPilotContext(); + + const entityLink = useMemo( + () => + entityType === EntityType.TABLE + ? EntityLink.getTableEntityLink( + entityFqn, + columnData.record?.name ?? '' + ) + : getEntityFeedLink(entityType, columnData.fqn), + [entityType, entityFqn] + ); + + const suggestionForEmptyData = useMemo(() => { + if (isEmpty(columnData.field ?? ''.trim())) { + return suggestions.find( + (suggestion) => suggestion.entityLink === entityLink + ); + } + + return null; + }, [suggestions, columnData.field, entityLink]); + + if (activeSuggestion?.entityLink === entityLink) { + return ( + + ); + } + + if (suggestionForEmptyData) { + return ( + + ); + } return ( )) ); +jest.mock('../../MetaPilot/MetaPilotProvider/MetaPilotProvider', () => ({ + useMetaPilotContext: jest.fn().mockReturnValue({ + suggestions: [], + }), +})); + jest.mock('../../common/ErrorWithPlaceholder/ErrorPlaceHolder', () => jest .fn() diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx index 52beca237e8b..ba28e7bd49a0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx @@ -43,6 +43,14 @@ jest.mock('react-router-dom', () => ({ useParams: jest.fn().mockImplementation(() => mockParams), })); +jest.mock('../MetaPilot/MetaPilotProvider/MetaPilotProvider', () => ({ + useMetaPilotContext: jest.fn().mockReturnValue({ + suggestions: [], + initMetaPilot: jest.fn(), + resetMetaPilot: jest.fn(), + }), +})); + jest.mock('../../rest/rolesAPIV1', () => ({ getRoles: jest.fn().mockImplementation(() => Promise.resolve(mockUserRole)), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx index 404b8b1e4361..5120f429e5bb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx @@ -14,6 +14,7 @@ import Icon from '@ant-design/icons'; import { Card, Space, Tooltip, Typography } from 'antd'; import { t } from 'i18next'; +import { isEmpty } from 'lodash'; import React, { useCallback, useMemo } from 'react'; import { useHistory } from 'react-router'; import { ReactComponent as CommentIcon } from '../../../assets/svg/comment.svg'; @@ -29,6 +30,8 @@ import { getUpdateDescriptionPath, TASK_ENTITIES, } from '../../../utils/TasksUtils'; +import MetaPilotDescriptionAlert from '../../MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component'; +import { useMetaPilotContext } from '../../MetaPilot/MetaPilotProvider/MetaPilotProvider'; import { ModalWithMarkdownEditor } from '../../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import RichTextEditorPreviewer from '../RichTextEditor/RichTextEditorPreviewer'; const { Text } = Typography; @@ -73,6 +76,7 @@ const DescriptionV1 = ({ reduceDescription, }: Props) => { const history = useHistory(); + const { activeSuggestion, suggestions } = useMetaPilotContext(); const handleRequestDescription = useCallback(() => { history.push( @@ -86,10 +90,19 @@ const DescriptionV1 = ({ ); }, [entityType, entityFqn]); - const entityLink = useMemo( - () => getEntityFeedLink(entityType, entityFqn, EntityField.DESCRIPTION), - [entityType, entityFqn] - ); + const { entityLink, entityLinkWithoutField } = useMemo(() => { + const entityLink = getEntityFeedLink( + entityType, + entityFqn, + EntityField.DESCRIPTION + ); + const entityLinkWithoutField = getEntityFeedLink(entityType, entityFqn); + + return { + entityLink, + entityLinkWithoutField, + }; + }, [entityType, entityFqn]); const taskActionButton = useMemo(() => { const hasDescription = Boolean(description.trim()); @@ -159,6 +172,34 @@ const DescriptionV1 = ({ ] ); + const suggestionForEmptyData = useMemo(() => { + if (isEmpty(description.trim())) { + return suggestions.find( + (suggestion) => suggestion.entityLink === entityLinkWithoutField + ); + } + + return null; + }, [suggestions, description]); + + if (activeSuggestion?.entityLink === entityLinkWithoutField) { + return ( + + ); + } + + if (suggestionForEmptyData) { + return ( + + ); + } + const content = ( = [ { name: t('label.configure'), step: 2 }, { name: t('label.schedule'), step: 3 }, ]; + +export const APP_UI_SCHEMA = { metaPilotAppType: { 'ui:widget': 'hidden' } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 24c46c215a5c..fdd440106bd3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -657,6 +657,8 @@ "metadata-lowercase": "metadaten", "metadata-plural": "Metadaten", "metadata-to-es-config-optional": "Metadaten-zu-ES-Konfiguration (optional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Metriktyp", "metric-value": "Metrikwert", "metrics-summary": "Metrikzusammenfassung", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Erfolgreich hochgeladen", "suggest": "Vorschlagen", "suggest-entity": "{{entity}} vorschlagen", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Vorschlag", "suggestion-lowercase-plural": "Vorschläge", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index aeff8b672a76..4d552f788b84 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -657,6 +657,8 @@ "metadata-lowercase": "metadata", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Metadata To ES Config (Optional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Metric Type", "metric-value": "Metric Value", "metrics-summary": "Metrics Summary", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Successfully Uploaded", "suggest": "Suggest", "suggest-entity": "Suggest {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Suggestion", "suggestion-lowercase-plural": "suggestions", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index bd1a49c0fd95..5435a927f8b3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -657,6 +657,8 @@ "metadata-lowercase": "metadatos", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Configuración de Metadatos a ES (Opcional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Tipo de Métrica", "metric-value": "Valor de Métrica", "metrics-summary": "Resumen de Métricas", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Cargado Exitosamente", "suggest": "Sugerir", "suggest-entity": "Sugerir {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Sugerencia", "suggestion-lowercase-plural": "sugerencias", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index ed6ec4cb7626..e75dee19a575 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -657,6 +657,8 @@ "metadata-lowercase": "métadonnées", "metadata-plural": "Métadonnées", "metadata-to-es-config-optional": "Configuration de Métadonnées vers ES (Optionnelle)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Type de Mesure", "metric-value": "Valeur de la Mesure", "metrics-summary": "Résumé des Mesures", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Téléchargé avec succès", "suggest": "Suggérer", "suggest-entity": "Suggérer {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Suggestion", "suggestion-lowercase-plural": "suggestions", "suite": "Ensemble", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json index 59075bf4137b..7219ca67f255 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json @@ -657,6 +657,8 @@ "metadata-lowercase": "מטא-דאטה", "metadata-plural": "מטא-דאטה", "metadata-to-es-config-optional": "קונפיגורציית מטא-דאטה ל-Elasticsearch (אופציונלי)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "סוג מדד", "metric-value": "ערך מדד", "metrics-summary": "סיכום מדדים", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "הועלה בהצלחה", "suggest": "הצע", "suggest-entity": "הצע {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "הצעה", "suggestion-lowercase-plural": "הצעות", "suite": "יחידת בדיקה", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index c506be903d5c..10d850728d74 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -657,6 +657,8 @@ "metadata-lowercase": "メタデータ", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Metadata To ES Config (Optional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "メトリクスのタイプ", "metric-value": "メトリクスの値", "metrics-summary": "メトリクスの要約", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "アップロード成功", "suggest": "提案", "suggest-entity": "{{entity}}を提案", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "提案", "suggestion-lowercase-plural": "提案", "suite": "スイート", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json index 9709c75b7cf7..7020e6f9a3b6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json @@ -657,6 +657,8 @@ "metadata-lowercase": "metadata", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Metadata naar ES-configuratie (Optioneel)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Metriektype", "metric-value": "Metriekwaarde", "metrics-summary": "Samenvatting van metingen", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Succesvol Geüpload", "suggest": "Suggestie", "suggest-entity": "Suggereer {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Suggestie", "suggestion-lowercase-plural": "suggesties", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index af84e359430c..34a54e504d04 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -657,6 +657,8 @@ "metadata-lowercase": "metadados", "metadata-plural": "Metadados", "metadata-to-es-config-optional": "Metadados para Configuração ES (Opcional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Tipo de Métrica", "metric-value": "Valor da Métrica", "metrics-summary": "Resumo de Métricas", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Carregado com Sucesso", "suggest": "Sugerir", "suggest-entity": "Sugerir {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Sugestão", "suggestion-lowercase-plural": "sugestões", "suite": "Conjuto de Testes", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index 6b8eb66d7e19..3ae3852035f8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -657,6 +657,8 @@ "metadata-lowercase": "метаданные", "metadata-plural": "Метаданные", "metadata-to-es-config-optional": "Метаданные для конфигурации ES (необязательно)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Тип метрики", "metric-value": "Значение метрики", "metrics-summary": "Сводка метрик", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "Успешно загружено", "suggest": "Предложить", "suggest-entity": "Предложить {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Предложение", "suggestion-lowercase-plural": "предложения", "suite": "Набор", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 6f8880123edc..3edcec36abe6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -657,6 +657,8 @@ "metadata-lowercase": "元数据", "metadata-plural": "元数据", "metadata-to-es-config-optional": "元数据到 ES 配置（可选）", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "指标类型", "metric-value": "指标值", "metrics-summary": "指标概要", @@ -1040,6 +1042,7 @@ "successfully-uploaded": "上传成功", "suggest": "建议", "suggest-entity": "建议{{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "建议", "suggestion-lowercase-plural": "建议", "suite": "套件", diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx index a4306e2fdcb9..b6ee11899a67 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx @@ -25,7 +25,10 @@ import FormBuilder from '../../components/common/FormBuilder/FormBuilder'; import IngestionStepper from '../../components/IngestionStepper/IngestionStepper.component'; import Loader from '../../components/Loader/Loader'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; -import { STEPS_FOR_APP_INSTALL } from '../../constants/Applications.constant'; +import { + APP_UI_SCHEMA, + STEPS_FOR_APP_INSTALL, +} from '../../constants/Applications.constant'; import { GlobalSettingOptions } from '../../constants/GlobalSettings.constants'; import { ServiceCategory } from '../../enums/service.enum'; import { AppType } from '../../generated/entity/applications/app'; @@ -163,6 +166,7 @@ const AppInstall = () => { okText={t('label.submit')} schema={jsonSchema} serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + uiSchema={APP_UI_SCHEMA} validator={validator} onCancel={() => setActiveServiceStep(1)} onSubmit={onSaveConfiguration} diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx index 0abaa7d2e046..6eae69704600 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx @@ -30,6 +30,17 @@ jest.mock('../../components/PermissionProvider/PermissionProvider', () => ({ })), })); +jest.mock( + '../../components/MetaPilot/MetaPilotProvider/MetaPilotProvider', + () => ({ + useMetaPilotContext: jest.fn().mockReturnValue({ + suggestions: [], + initMetaPilot: jest.fn(), + resetMetaPilot: jest.fn(), + }), + }) +); + jest.mock('../../rest/tableAPI', () => ({ getTableDetailsByFQN: jest.fn().mockImplementation(() => Promise.resolve({ @@ -215,7 +226,7 @@ describe('TestDetailsPageV1 component', () => { }); expect(getTableDetailsByFQN).toHaveBeenCalledWith('fqn', { - fields: `${COMMON_API_FIELDS},usageSummary`, + fields: `${COMMON_API_FIELDS}`, }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx index 9e6b2c64d21e..3e2bdd9980b6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx @@ -33,6 +33,7 @@ import EntityRightPanel from '../../components/Entity/EntityRightPanel/EntityRig import Lineage from '../../components/Lineage/Lineage.component'; import LineageProvider from '../../components/LineageProvider/LineageProvider'; import Loader from '../../components/Loader/Loader'; +import { useMetaPilotContext } from '../../components/MetaPilot/MetaPilotProvider/MetaPilotProvider'; import { EntityName } from '../../components/Modals/EntityNameModal/EntityNameModal.interface'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import { usePermissionProvider } from '../../components/PermissionProvider/PermissionProvider'; @@ -114,6 +115,7 @@ const TableDetailsPageV1 = () => { ThreadType.Conversation ); const [queryCount, setQueryCount] = useState(0); + const { resetMetaPilot, initMetaPilot } = useMetaPilotContext(); const [loading, setLoading] = useState(!isTourOpen); const [tablePermissions, setTablePermissions] = useState( @@ -135,7 +137,7 @@ const TableDetailsPageV1 = () => { [datasetFQN] ); - const fetchTableDetails = async () => { + const fetchTableDetails = useCallback(async () => { setLoading(true); try { let fields = defaultFields; @@ -159,7 +161,7 @@ const TableDetailsPageV1 = () => { } finally { setLoading(false); } - }; + }, [tableFqn]); const fetchQueryCount = async () => { if (!tableDetails?.id) { @@ -278,10 +280,15 @@ const TableDetailsPageV1 = () => { ); useEffect(() => { - if (tableFqn) { + if (tableFqn && fetchTableDetails) { fetchResourcePermission(tableFqn); + initMetaPilot(tableFqn, fetchTableDetails); } - }, [tableFqn]); + + return () => { + resetMetaPilot(); + }; + }, [tableFqn, fetchTableDetails]); const handleFeedCount = useCallback((data: FeedCounts) => { setFeedCount(data); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts new file mode 100644 index 000000000000..812877e4f6fa --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { AxiosResponse } from 'axios'; +import { PagingResponse } from 'Models'; +import { SuggestionAction } from '../components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface'; +import { Suggestion } from '../generated/entity/feed/suggestion'; +import { ListParams } from '../interface/API.interface'; +import APIClient from './index'; + +const BASE_URL = '/suggestions'; + +export type ListSuggestionsParams = ListParams & { + entityFQN?: string; +}; + +export const getMetaPilotSuggestionsList = async ( + params?: ListSuggestionsParams +) => { + const response = await APIClient.get>(BASE_URL, { + params, + }); + + return response.data; +}; + +export const updateSuggestionStatus = ( + data: Suggestion, + action: SuggestionAction +): Promise => { + const url = `${BASE_URL}/${data.id}/${action}`; + + return APIClient.put(url, {}); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/app.less b/openmetadata-ui/src/main/resources/ui/src/styles/app.less index 49fc4d988c37..2e667bdda506 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/app.less +++ b/openmetadata-ui/src/main/resources/ui/src/styles/app.less @@ -726,3 +726,10 @@ a[href].link-text-grey, width: 20px; height: 20px; } + +.floating-button-container { + position: fixed; + bottom: 20px; + right: 20px; + z-index: 1001; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json index 099d382bf49d..ea86671af3b2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json @@ -51,25 +51,6 @@ "$ref": "#/definitions/metaPilotAppType", "default": "MetaPilot" }, - "waiiInstance": { - "title": "WAII Instance", - "description": "WAII API host URL", - "type": "string", - "format": "URI", - "default": "https://tweakit.waii.ai/api/" - }, - "collateURL": { - "title": "Collate URL", - "description": "Collate Server public URL. WAII will use this information to interact with the server. E.g., https://sandbox.getcollate.io", - "type": "string", - "format": "URI" - }, - "token": { - "title": "WAII API Token", - "description": "WAII API Token", - "type": "string", - "format": "password" - }, "serviceDatabases": { "title": "Service Databases", "description": "Services and Databases configured to get the descriptions from.", @@ -79,6 +60,5 @@ } } }, - "additionalProperties": false, - "required": ["token"] + "additionalProperties": false }