From 66e44056b75a58842f7052c2f820911fc77e6305 Mon Sep 17 00:00:00 2001 From: Akash Jain <15995028+akash-jain-10@users.noreply.github.com> Date: Wed, 22 Nov 2023 22:05:43 +0530 Subject: [PATCH 1/8] fix(docs): Update contents to current release version 1.2.2 (#14075) --- .../content/v1.2.x/deployment/docker/index.md | 4 ++-- .../content/v1.2.x/deployment/ingestion/mwaa.md | 6 +++--- .../content/v1.2.x/deployment/kubernetes/faqs.md | 4 ++-- .../v1.2.x/deployment/upgrade/kubernetes.md | 8 ++++---- .../v1.2.x/quick-start/local-docker-deployment.md | 14 +++++++------- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/openmetadata-docs/content/v1.2.x/deployment/docker/index.md b/openmetadata-docs/content/v1.2.x/deployment/docker/index.md index d20155597f25..7d6151b44d38 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/docker/index.md +++ b/openmetadata-docs/content/v1.2.x/deployment/docker/index.md @@ -102,7 +102,7 @@ This docker compose file contains only the docker compose services for OpenMetad You can also run the below command to fetch the docker compose file directly from the terminal - ```bash -wget https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/docker-compose-openmetadata.yml +wget https://github.com/open-metadata/OpenMetadata/releases/download/1.2.2-release/docker-compose-openmetadata.yml ``` ### 3. Update Environment Variables required for OpenMetadata Dependencies @@ -191,7 +191,7 @@ You can validate that all containers are up by running with command `docker ps`. ```commandline ❯ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -470cc8149826 openmetadata/server:1.2.0 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server +470cc8149826 openmetadata/server:1.2.2 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server ``` In a few seconds, you should be able to access the OpenMetadata UI at [http://localhost:8585](http://localhost:8585) diff --git a/openmetadata-docs/content/v1.2.x/deployment/ingestion/mwaa.md b/openmetadata-docs/content/v1.2.x/deployment/ingestion/mwaa.md index f1f5fcd8e20f..cb8ec536c1c0 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/ingestion/mwaa.md +++ b/openmetadata-docs/content/v1.2.x/deployment/ingestion/mwaa.md @@ -32,9 +32,9 @@ To install the package, we need to update the `requirements.txt` file from the M openmetadata-ingestion[]==x.y.z ``` -Where `x.y.z` is the version of the OpenMetadata ingestion package. Note that the version needs to match the server version. If we are using the server at 1.2.0, then the ingestion package needs to also be 1.2.0. +Where `x.y.z` is the version of the OpenMetadata ingestion package. Note that the version needs to match the server version. If we are using the server at 1.2.2, then the ingestion package needs to also be 1.2.2. -The plugin parameter is a list of the sources that we want to ingest. An example would look like this `openmetadata-ingestion[mysql,snowflake,s3]==1.2.0`. +The plugin parameter is a list of the sources that we want to ingest. An example would look like this `openmetadata-ingestion[mysql,snowflake,s3]==1.2.2`. A DAG deployed using a Python Operator would then look like follows @@ -106,7 +106,7 @@ We will now describe the steps, following the official AWS documentation. - The cluster needs a task to run in `FARGATE` mode. - The required image is `docker.getcollate.io/openmetadata/ingestion-base:x.y.z` - - The same logic as above applies. The `x.y.z` version needs to match the server version. For example, `docker.getcollate.io/openmetadata/ingestion-base:1.2.0` + - The same logic as above applies. The `x.y.z` version needs to match the server version. For example, `docker.getcollate.io/openmetadata/ingestion-base:1.2.2` We have tested this process with a Task Memory of 512MB and Task CPU (unit) of 256. This can be tuned depending on the amount of metadata that needs to be ingested. diff --git a/openmetadata-docs/content/v1.2.x/deployment/kubernetes/faqs.md b/openmetadata-docs/content/v1.2.x/deployment/kubernetes/faqs.md index 02ff65bfee57..492cd23b9b91 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/kubernetes/faqs.md +++ b/openmetadata-docs/content/v1.2.x/deployment/kubernetes/faqs.md @@ -48,7 +48,7 @@ WORKDIR /home/ COPY . RUN update-ca-certificates ``` -where `docker.getcollate.io/openmetadata/server:x.y.z` needs to point to the same version of the OpenMetadata server, for example `docker.getcollate.io/openmetadata/server:1.2.0`. +where `docker.getcollate.io/openmetadata/server:x.y.z` needs to point to the same version of the OpenMetadata server, for example `docker.getcollate.io/openmetadata/server:1.2.2`. This image needs to be built and published to the container registry of your choice. ### 2. Update your openmetadata helm values yaml @@ -95,7 +95,7 @@ COPY setup.py . RUN pip install --no-deps . ``` -where `docker.getcollate.io/openmetadata/ingestion:x.y.z` needs to point to the same version of the OpenMetadata server, for example `docker.getcollate.io/openmetadata/ingestion:1.2.0`. +where `docker.getcollate.io/openmetadata/ingestion:x.y.z` needs to point to the same version of the OpenMetadata server, for example `docker.getcollate.io/openmetadata/ingestion:1.2.2`. This image needs to be built and published to the container registry of your choice. ### 2. Update the airflow in openmetadata dependencies values YAML diff --git a/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md b/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md index c8a30c62049c..ba72e6960c9b 100644 --- a/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md +++ b/openmetadata-docs/content/v1.2.x/deployment/upgrade/kubernetes.md @@ -45,12 +45,12 @@ Verify with the below command to see the latest release available locally. ```commandline helm search repo open-metadata --versions > NAME CHART VERSION APP VERSION DESCRIPTION -open-metadata/openmetadata 1.2.1 1.2.0 A Helm chart for OpenMetadata on Kubernetes -open-metadata/openmetadata 1.2.0 1.2.0 A Helm chart for OpenMetadata on Kubernetes +open-metadata/openmetadata 1.2.4 1.2.2 A Helm chart for OpenMetadata on Kubernetes +open-metadata/openmetadata 1.2.3 1.2.2 A Helm chart for OpenMetadata on Kubernetes ... -open-metadata/openmetadata-dependencies 1.2.1 1.2.0 Helm Dependencies for OpenMetadata -open-metadata/openmetadata-dependencies 1.2.0 1.2.0 Helm Dependencies for OpenMetadata +open-metadata/openmetadata-dependencies 1.2.4 1.2.2 Helm Dependencies for OpenMetadata +open-metadata/openmetadata-dependencies 1.2.3 1.2.2 Helm Dependencies for OpenMetadata ... ``` diff --git a/openmetadata-docs/content/v1.2.x/quick-start/local-docker-deployment.md b/openmetadata-docs/content/v1.2.x/quick-start/local-docker-deployment.md index 7d809d8e84ac..05b3d87ea3f4 100644 --- a/openmetadata-docs/content/v1.2.x/quick-start/local-docker-deployment.md +++ b/openmetadata-docs/content/v1.2.x/quick-start/local-docker-deployment.md @@ -119,15 +119,15 @@ The latest version is at the top of the page You can use the curl or wget command as well to fetch the docker compose files from your terminal - ```commandline -curl -sL -o docker-compose.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/docker-compose.yml +curl -sL -o docker-compose.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.2.2-release/docker-compose.yml -curl -sL -o docker-compose-postgres.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/docker-compose-postgres.yml +curl -sL -o docker-compose-postgres.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.2.2-release/docker-compose-postgres.yml ``` ```commandline -wget -O https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/docker-compose.yml +wget -O https://github.com/open-metadata/OpenMetadata/releases/download/1.2.2-release/docker-compose.yml -wget -O https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/docker-compose-postgres.yml +wget -O https://github.com/open-metadata/OpenMetadata/releases/download/1.2.2-release/docker-compose-postgres.yml ``` ### 3. Start the Docker Compose Services @@ -166,10 +166,10 @@ You can validate that all containers are up by running with command `docker ps`. ```commandline ❯ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -470cc8149826 openmetadata/server:1.2.0 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server -63578aacbff5 openmetadata/ingestion:1.2.0 "./ingestion_depende…" 45 seconds ago Up 43 seconds 0.0.0.0:8080->8080/tcp openmetadata_ingestion +470cc8149826 openmetadata/server:1.2.2 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server +63578aacbff5 openmetadata/ingestion:1.2.2 "./ingestion_depende…" 45 seconds ago Up 43 seconds 0.0.0.0:8080->8080/tcp openmetadata_ingestion 9f5ee8334f4b docker.elastic.co/elasticsearch/elasticsearch:7.16.3 "/tini -- /usr/local…" 45 seconds ago Up 44 seconds 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp openmetadata_elasticsearch -08947ab3424b openmetadata/db:1.2.0 "/entrypoint.sh mysq…" 45 seconds ago Up 44 seconds (healthy) 3306/tcp, 33060-33061/tcp openmetadata_mysql +08947ab3424b openmetadata/db:1.2.2 "/entrypoint.sh mysq…" 45 seconds ago Up 44 seconds (healthy) 3306/tcp, 33060-33061/tcp openmetadata_mysql ``` In a few seconds, you should be able to access the OpenMetadata UI at [http://localhost:8585](http://localhost:8585) From 4bce6a9aaf1781a373cad0003ab2e59336c5f850 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:13:35 +0530 Subject: [PATCH 2/8] Athena: Add support for work groups in usage & lineage (#14072) --- .../source/database/athena/models.py | 10 +++ .../source/database/athena/query_parser.py | 80 +++++++++++++++---- .../ingestion/source/database/athena/usage.py | 4 +- .../connectors/database/athena/index.md | 1 + 4 files changed, 77 insertions(+), 18 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/athena/models.py b/ingestion/src/metadata/ingestion/source/database/athena/models.py index d833893846de..51c9403e46ae 100644 --- a/ingestion/src/metadata/ingestion/source/database/athena/models.py +++ b/ingestion/src/metadata/ingestion/source/database/athena/models.py @@ -39,3 +39,13 @@ class AthenaQueryExecution(BaseModel): class AthenaQueryExecutionList(BaseModel): QueryExecutions: Optional[List[AthenaQueryExecution]] + + +class WorkGroup(BaseModel): + Name: Optional[str] + State: Optional[str] + + +class WorkGroupsList(BaseModel): + WorkGroups: Optional[List[WorkGroup]] = [] + NextToken: Optional[str] diff --git a/ingestion/src/metadata/ingestion/source/database/athena/query_parser.py b/ingestion/src/metadata/ingestion/source/database/athena/query_parser.py index a9e2bb1beaa4..725af32ca0eb 100644 --- a/ingestion/src/metadata/ingestion/source/database/athena/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/athena/query_parser.py @@ -12,6 +12,7 @@ Athena Query parser module """ +import traceback from abc import ABC from math import ceil @@ -27,6 +28,7 @@ from metadata.ingestion.source.database.athena.models import ( AthenaQueryExecutionList, QueryExecutionIdsResponse, + WorkGroupsList, ) from metadata.ingestion.source.database.query_parser_source import QueryParserSource from metadata.utils.constants import QUERY_WITH_DBT, QUERY_WITH_OM_VERSION @@ -36,6 +38,8 @@ ATHENA_QUERY_PAGINATOR_LIMIT = 50 +ATHENA_ENABLED_WORK_GROUP_STATE = "ENABLED" + class AthenaQueryParserSource(QueryParserSource, ABC): """ @@ -59,23 +63,67 @@ def create(cls, config_dict, metadata: OpenMetadata): ) return cls(config, metadata) - def get_queries(self): - query_limit = ceil( - self.source_config.resultLimit / ATHENA_QUERY_PAGINATOR_LIMIT - ) - paginator = self.client.get_paginator("list_query_executions") - paginator_response = paginator.paginate() - for response in paginator_response: - response_obj = QueryExecutionIdsResponse(**response) - if response_obj.QueryExecutionIds: - query_details_response = self.client.batch_get_query_execution( - QueryExecutionIds=response_obj.QueryExecutionIds + def _get_work_group_response(self, next_token: str, is_first_call: bool = False): + if is_first_call: + return self.client.list_work_groups() + return self.client.list_work_groups(NextToken=next_token) + + def get_work_groups(self) -> str: + """ + Method to get list of names of athena work groups + """ + next_token = None + is_first_call = True + try: + while True: + work_group_list = self._get_work_group_response( + next_token, is_first_call ) - query_details_list = AthenaQueryExecutionList(**query_details_response) - yield query_details_list - query_limit -= 1 - if not query_limit: - break + response_obj = WorkGroupsList(**work_group_list) + for work_group in response_obj.WorkGroups: + if ( + work_group.State + and work_group.State.upper() == ATHENA_ENABLED_WORK_GROUP_STATE + ): + yield work_group.Name + next_token = response_obj.NextToken + is_first_call = False + if next_token is None: + break + except Exception as exc: + logger.debug(f"Failed to fetch work groups due to: {exc}") + logger.debug(traceback.format_exc()) + if is_first_call: + # if it fails for the first api call, most likely due to insufficient + # permissions then still fetch the queries with default workgroup + yield None + + def get_queries(self): + """ + Method to fetch queries from all work groups + """ + for work_group in self.get_work_groups(): + query_limit = ceil( + self.source_config.resultLimit / ATHENA_QUERY_PAGINATOR_LIMIT + ) + paginator = self.client.get_paginator("list_query_executions") + if work_group: + paginator_response = paginator.paginate(WorkGroup=work_group) + else: + paginator_response = paginator.paginate() + for response in paginator_response: + response_obj = QueryExecutionIdsResponse(**response) + if response_obj.QueryExecutionIds: + query_details_response = self.client.batch_get_query_execution( + QueryExecutionIds=response_obj.QueryExecutionIds + ) + query_details_list = AthenaQueryExecutionList( + **query_details_response + ) + yield query_details_list + query_limit -= 1 + if not query_limit: + break def is_not_dbt_or_om_query(self, query_text: str) -> bool: return not ( diff --git a/ingestion/src/metadata/ingestion/source/database/athena/usage.py b/ingestion/src/metadata/ingestion/source/database/athena/usage.py index bfeb9401b41c..edc63f28917a 100644 --- a/ingestion/src/metadata/ingestion/source/database/athena/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/athena/usage.py @@ -11,7 +11,7 @@ """ Athena usage module """ -from typing import Iterable, Optional +from typing import Iterable from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery from metadata.ingestion.source.database.athena.query_parser import ( @@ -32,7 +32,7 @@ class AthenaUsageSource(AthenaQueryParserSource, UsageSource): Athena Usage Source """ - def yield_table_queries(self) -> Optional[Iterable[TableQuery]]: + def yield_table_queries(self) -> Iterable[TableQueries]: """ Method to yield TableQueries """ diff --git a/openmetadata-docs/content/v1.2.x/connectors/database/athena/index.md b/openmetadata-docs/content/v1.2.x/connectors/database/athena/index.md index 361ad0f75661..5a4692db6e0a 100644 --- a/openmetadata-docs/content/v1.2.x/connectors/database/athena/index.md +++ b/openmetadata-docs/content/v1.2.x/connectors/database/athena/index.md @@ -74,6 +74,7 @@ And is defined as: "athena:ListQueryExecutions", "athena:StartQueryExecution", "athena:GetQueryExecution", + "athena:ListWorkGroups", "athena:GetQueryResults", "athena:BatchGetQueryExecution" ], From 274a3d177368ddfc032d2f4c349efc6b89b2f949 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Thu, 23 Nov 2023 11:38:30 +0530 Subject: [PATCH 3/8] fixed viewparsing query (#14076) --- .../1.2.3/mysql/postDataMigrationSQLScript.sql | 0 .../migrations/native/1.2.3/mysql/schemaChanges.sql | 0 .../1.2.3/postgres/postDataMigrationSQLScript.sql | 0 .../native/1.2.3/postgres/schemaChanges.sql | 12 ++++++++++++ 4 files changed, 12 insertions(+) create mode 100644 bootstrap/sql/migrations/native/1.2.3/mysql/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.2.3/mysql/schemaChanges.sql create mode 100644 bootstrap/sql/migrations/native/1.2.3/postgres/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.2.3/postgres/schemaChanges.sql diff --git a/bootstrap/sql/migrations/native/1.2.3/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.2.3/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.2.3/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.3/mysql/schemaChanges.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.2.3/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.2.3/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.2.3/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.2.3/postgres/schemaChanges.sql new file mode 100644 index 000000000000..399e5d62f61d --- /dev/null +++ b/bootstrap/sql/migrations/native/1.2.3/postgres/schemaChanges.sql @@ -0,0 +1,12 @@ + +-- fixed Query for updating viewParsingTimeoutLimit +UPDATE ingestion_pipeline_entity +SET json = jsonb_set( + json::jsonb #- '{sourceConfig,config,viewParsingTimeoutLimit}', + '{sourceConfig,config,queryParsingTimeoutLimit}', + (json #> '{sourceConfig,config,viewParsingTimeoutLimit}')::jsonb, + true +) +WHERE json #>> '{pipelineType}' = 'metadata' +AND json #>> '{sourceConfig,config,type}' = 'DatabaseMetadata' +AND json #>> '{sourceConfig,config,viewParsingTimeoutLimit}' is not null; From cabc346a6bd6d03f712adb3c41ce18596f0ac9f8 Mon Sep 17 00:00:00 2001 From: Teddy Date: Thu, 23 Nov 2023 07:11:10 +0100 Subject: [PATCH 4/8] feat: db migration for custom metrics (#14036) --- .../native/1.3.0/mysql/postDataMigrationSQLScript.sql | 8 ++++++++ .../native/1.3.0/postgres/postDataMigrationSQLScript.sql | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e66612d8f5fa --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/postDataMigrationSQLScript.sql @@ -0,0 +1,8 @@ +-- Rename customMetricsProfile to customMetrics +UPDATE profiler_data_time_series +SET json = REPLACE(json, '"customMetricsProfile"', '"customMetrics"'); + +-- Delete customMetricsProfile from entity_extension +-- This was not supported on the processing side before 1.3. +DELETE FROM openmetadata_db.entity_extension ee +where extension like '%customMetrics'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..6d65e5f77057 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/postDataMigrationSQLScript.sql @@ -0,0 +1,8 @@ +-- Rename customMetricsProfile to customMetrics +UPDATE profiler_data_time_series +SET json = REPLACE(json::text, '"customMetricsProfile"', '"customMetrics"')::jsonb; + +-- Delete customMetricsProfile from entity_extension +-- This was not supported on the processing side before 1.3. +DELETE FROM entity_extension ee +where extension like '%customMetrics'; \ No newline at end of file From ab3440938a169fef6e1aa2874dde629e837561be Mon Sep 17 00:00:00 2001 From: Akash Jain <15995028+akash-jain-10@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:42:20 +0530 Subject: [PATCH 5/8] fix: make ingestion build docker image using Dockerfile.ci (#14080) --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b345cfdee9e4..9c16def705eb 100644 --- a/Makefile +++ b/Makefile @@ -69,8 +69,8 @@ SNYK_ARGS := --severity-threshold=high .PHONY: snyk-ingestion-report snyk-ingestion-report: ## Uses Snyk CLI to validate the ingestion code and container. Don't stop the execution @echo "Validating Ingestion container..." - docker build -t openmetadata-ingestion:scan -f ingestion/Dockerfile . - snyk container test openmetadata-ingestion:scan --file=ingestion/Dockerfile $(SNYK_ARGS) --json > security-report/ingestion-docker-scan.json | true; + docker build -t openmetadata-ingestion:scan -f ingestion/Dockerfile.ci . + snyk container test openmetadata-ingestion:scan --file=ingestion/Dockerfile.ci $(SNYK_ARGS) --json > security-report/ingestion-docker-scan.json | true; @echo "Validating ALL ingestion dependencies. Make sure the venv is activated." cd ingestion; \ pip freeze > scan-requirements.txt; \ From c95de19a094e4ed93d03c281cdae64aa6f955803 Mon Sep 17 00:00:00 2001 From: VolkovGeoPhy <117853378+VolkovGeoPhy@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:33:07 +0100 Subject: [PATCH 6/8] great-expectations~=0.18.0 (#14056) --- ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/setup.py b/ingestion/setup.py index e169ad8d180e..b3194a3b41c6 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -24,7 +24,7 @@ "boto3": "boto3>=1.20,<2.0", # No need to add botocore separately. It's a dep from boto3 "geoalchemy2": "GeoAlchemy2~=0.12", "google-cloud-storage": "google-cloud-storage==1.43.0", - "great-expectations": "great-expectations~=0.17.0", + "great-expectations": "great-expectations~=0.18.0", "grpc-tools": "grpcio-tools>=1.47.2", "msal": "msal~=1.2", "neo4j": "neo4j~=5.3.0", From 631145d09073505a0ba9b2ba7b70b324d463d28e Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Thu, 23 Nov 2023 18:46:25 +0530 Subject: [PATCH 7/8] chore(ui): add `initialOptions` prop in tag suggestion component (#14085) * chore(ui): add initialData prop in tag suggestion component * address comment --- .../components/AsyncSelectList/AsyncSelectList.interface.ts | 2 +- .../ui/src/components/AsyncSelectList/AsyncSelectList.tsx | 6 +++--- .../Tag/TagsSelectForm/TagsSelectForm.component.tsx | 2 +- .../ui/src/pages/TasksPage/shared/TagSuggestion.tsx | 4 ++++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.interface.ts index b81456b4db3d..be6ec2bdfa93 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.interface.ts @@ -28,7 +28,7 @@ export interface AsyncSelectListProps { placeholder?: string; debounceTimeout?: number; defaultValue?: string[]; - initialData?: SelectOption[]; + initialOptions?: SelectOption[]; onChange?: (option: DefaultOptionType | DefaultOptionType[]) => void; fetchOptions: ( search: string, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.tsx index 22d4b55f2c82..2b5ff83332ab 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AsyncSelectList/AsyncSelectList.tsx @@ -42,7 +42,7 @@ const AsyncSelectList: FC = ({ onChange, fetchOptions, debounceTimeout = 800, - initialData, + initialOptions, className, ...props }) => { @@ -52,7 +52,7 @@ const AsyncSelectList: FC = ({ const [searchValue, setSearchValue] = useState(''); const [paging, setPaging] = useState({} as Paging); const [currentPage, setCurrentPage] = useState(1); - const selectedTagsRef = useRef(initialData ?? []); + const selectedTagsRef = useRef(initialOptions ?? []); const loadOptions = useCallback( async (value: string) => { @@ -204,7 +204,7 @@ const AsyncSelectList: FC = ({ data ?? { value, label: value, - data: initialData?.find((item) => item.value === value)?.data, + data: initialOptions?.find((item) => item.value === value)?.data, } ); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsSelectForm/TagsSelectForm.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsSelectForm/TagsSelectForm.component.tsx index b808554185f1..f3127bf9fd0c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsSelectForm/TagsSelectForm.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsSelectForm/TagsSelectForm.component.tsx @@ -66,7 +66,7 @@ const TagSelectForm = ({ diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/shared/TagSuggestion.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/shared/TagSuggestion.tsx index ad8541c8bd0b..9dc8ab831310 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/shared/TagSuggestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/shared/TagSuggestion.tsx @@ -17,6 +17,7 @@ import { isArray, isEmpty } from 'lodash'; import { EntityTags } from 'Models'; import React, { useMemo } from 'react'; import AsyncSelectList from '../../../components/AsyncSelectList/AsyncSelectList'; +import { SelectOption } from '../../../components/AsyncSelectList/AsyncSelectList.interface'; import { TagSource } from '../../../generated/entity/data/container'; import { TagLabel } from '../../../generated/type/tagLabel'; import { @@ -28,6 +29,7 @@ export interface TagSuggestionProps { placeholder?: string; tagType?: TagSource; value?: TagLabel[]; + initialOptions?: SelectOption[]; onChange?: (newTags: TagLabel[]) => void; } @@ -35,6 +37,7 @@ const TagSuggestion: React.FC = ({ onChange, value, placeholder, + initialOptions, tagType = TagSource.Classification, }) => { const isGlossaryType = useMemo( @@ -82,6 +85,7 @@ const TagSuggestion: React.FC = ({ item.tagFQN) || []} fetchOptions={isGlossaryType ? fetchGlossaryList : fetchTagsElasticSearch} + initialOptions={initialOptions} mode="multiple" placeholder={ placeholder ?? From 312b038b2b6c6ea1f95fa1f98ae48e921a49fcc9 Mon Sep 17 00:00:00 2001 From: karanh37 <33024356+karanh37@users.noreply.github.com> Date: Thu, 23 Nov 2023 18:46:46 +0530 Subject: [PATCH 8/8] fix: add request tags and desc for glossary and terms (#14071) * fix: add request tags and desc for glossary and terms * fix: breadcrumbs * fix: add request tags cypress --- .../ui/cypress/e2e/Pages/Glossary.spec.js | 70 +++++++++++++++++-- .../GlossaryDetails.component.tsx | 8 ++- .../GlossaryDetails.interface.ts | 1 + .../GlossaryDetails/GlossaryDetails.test.tsx | 1 + .../GlossaryDetailsRightPanel.component.tsx | 21 ++++-- .../GlossaryDetailsRightPanel.test.tsx | 1 + .../GlossaryTermsV1.component.tsx | 2 + .../GlossaryTermsV1.interface.ts | 1 + .../GlossaryTerms/GlossaryTermsV1.test.tsx | 1 + .../tabs/GlossaryOverviewTab.component.tsx | 29 +++++--- .../tabs/GlossaryOverviewTab.test.tsx | 1 + .../Glossary/GlossaryV1.component.tsx | 51 ++++++++++++++ .../Task/TaskTab/TaskTab.component.tsx | 1 + .../pages/TasksPage/TasksPage.interface.ts | 6 +- .../resources/ui/src/utils/GlossaryUtils.ts | 30 +++++++- .../main/resources/ui/src/utils/TasksUtils.ts | 42 +++++++++-- 16 files changed, 235 insertions(+), 31 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js index 1097d7d72df4..c491a5b921ae 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js @@ -39,10 +39,12 @@ import { SEARCH_ENTITY_TABLE, } from '../../constants/constants'; +const name = `test_dataconsumer${uuid()}`; + const CREDENTIALS = { firstName: 'Cypress', lastName: 'UserDC', - email: `test_dataconsumer${uuid()}@openmetadata.org`, + email: `${name}@openmetadata.org`, password: 'User@OMD123', username: 'CypressUserDC', }; @@ -278,9 +280,7 @@ const updateTags = (inTerm) => { '/api/v1/search/query?q=*&index=tag_search_index&from=0&size=*&query_filter=*', 'tags' ); - cy.get( - '[data-testid="tags-input-container"] [data-testid="add-tag"]' - ).click(); + cy.get('[data-testid="tags-container"] [data-testid="add-tag"]').click(); verifyResponseStatusCode('@tags', 200); @@ -294,7 +294,7 @@ const updateTags = (inTerm) => { cy.get('[data-testid="saveAssociatedTag"]').scrollIntoView().click(); const container = inTerm - ? '[data-testid="tags-input-container"]' + ? '[data-testid="tags-container"]' : '[data-testid="glossary-details"]'; cy.wait(1000); cy.get(container).scrollIntoView().contains('Personal').should('be.visible'); @@ -636,7 +636,7 @@ describe('Glossary page should work properly', () => { // Remove Tag cy.get( - '[data-testid="tags-input-container"] [data-testid="edit-button"]' + '[data-testid="tags-container"] [data-testid="edit-button"]' ).click(); cy.get('[data-testid="remove-tags"]').should('be.visible').click(); @@ -768,6 +768,64 @@ describe('Glossary page should work properly', () => { voteGlossary(); }); + it('Request Tags workflow for Glossary', function () { + cy.get('[data-testid="glossary-left-panel"]') + .contains(NEW_GLOSSARY_1.name) + .click(); + + interceptURL( + 'GET', + `/api/v1/search/query?q=*%20AND%20disabled:false&index=tag_search_index*`, + 'suggestTag' + ); + interceptURL('POST', '/api/v1/feed', 'taskCreated'); + interceptURL('PUT', '/api/v1/feed/tasks/*/resolve', 'taskResolve'); + + cy.get('[data-testid="request-entity-tags"]').should('exist').click(); + + // set assignees for task + cy.get( + '[data-testid="select-assignee"] > .ant-select-selector > .ant-select-selection-overflow' + ) + .click() + .type(name); + cy.get(`[data-testid="assignee-option-${name}"]`).click(); + cy.clickOutside(); + + cy.get('[data-testid="tag-selector"]') + .click() + .type('{backspace}') + .type('{backspace}') + .type('Personal'); + + verifyResponseStatusCode('@suggestTag', 200); + cy.get( + '.ant-select-dropdown [data-testid="tag-PersonalData.Personal"]' + ).click(); + cy.clickOutside(); + + cy.get('[data-testid="submit-tag-request"]').click(); + verifyResponseStatusCode('@taskCreated', 201); + + // Accept the tag suggestion which is created + cy.get('.ant-btn-compact-first-item').contains('Accept Suggestion').click(); + + verifyResponseStatusCode('@taskResolve', 200); + + cy.reload(); + + cy.get('[data-testid="glossary-left-panel"]') + .contains(NEW_GLOSSARY_1.name) + .click(); + + checkDisplayName(NEW_GLOSSARY_1.name); + + // Verify Tags which is added at the time of creating glossary + cy.get('[data-testid="tags-container"]') + .contains('Personal') + .should('be.visible'); + }); + it('Assets Tab should work properly', () => { selectActiveGlossary(NEW_GLOSSARY.name); const glossary = NEW_GLOSSARY.name; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx index 9450f6d2dd75..c668132847a4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx @@ -46,6 +46,7 @@ const GlossaryDetails = ({ onAddGlossaryTerm, onEditGlossaryTerm, isVersionView, + onThreadLinkSelect, }: GlossaryDetailsProps) => { const { t } = useTranslation(); const history = useHistory(); @@ -131,15 +132,19 @@ const GlossaryDetails = ({ setIsDescriptionEditable(false)} onDescriptionEdit={() => setIsDescriptionEditable(true)} onDescriptionUpdate={onDescriptionUpdate} + onThreadLinkSelect={onThreadLinkSelect} /> + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts index f79d24151aa0..c9ffb55067e1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts @@ -33,4 +33,5 @@ export type GlossaryDetailsProps = { refreshGlossaryTerms: () => void; onAddGlossaryTerm: (glossaryTerm: GlossaryTerm | undefined) => void; onEditGlossaryTerm: (glossaryTerm: GlossaryTerm) => void; + onThreadLinkSelect: (value: string) => void; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx index 4eba4a5b30e0..441212f5e6b6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx @@ -65,6 +65,7 @@ const mockProps = { onAddGlossaryTerm: jest.fn(), onEditGlossaryTerm: jest.fn(), updateVote: jest.fn(), + onThreadLinkSelect: jest.fn(), }; describe('Test Glossary-details component', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx index e9c64eaaff36..b6c6ad725d21 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx @@ -22,7 +22,6 @@ import { UserSelectableList } from '../../../components/common/UserSelectableLis import { UserTeamSelectableList } from '../../../components/common/UserTeamSelectableList/UserTeamSelectableList.component'; import { OperationPermission } from '../../../components/PermissionProvider/PermissionProvider.interface'; import TagButton from '../../../components/TagButton/TagButton.component'; -import TagsInput from '../../../components/TagsInput/TagsInput.component'; import { DE_ACTIVE_COLOR, getTeamAndUserDetailsPath, @@ -31,7 +30,7 @@ import { } from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { EntityType } from '../../../enums/entity.enum'; -import { Glossary } from '../../../generated/entity/data/glossary'; +import { Glossary, TagSource } from '../../../generated/entity/data/glossary'; import { GlossaryTerm, TagLabel, @@ -47,6 +46,8 @@ import { getEntityVersionTags, } from '../../../utils/EntityVersionUtils'; import { DomainLabel } from '../../common/DomainLabel/DomainLabel.component'; +import TagsContainerV2 from '../../Tag/TagsContainerV2/TagsContainerV2'; +import { DisplayType } from '../../Tag/TagsViewer/TagsViewer.interface'; import GlossaryReviewers from './GlossaryReviewers'; type Props = { @@ -55,6 +56,7 @@ type Props = { selectedData: Glossary | GlossaryTerm; isGlossary: boolean; onUpdate: (data: GlossaryTerm | Glossary) => void; + onThreadLinkSelect: (value: string) => void; }; const GlossaryDetailsRightPanel = ({ @@ -63,6 +65,7 @@ const GlossaryDetailsRightPanel = ({ isGlossary, onUpdate, isVersionView, + onThreadLinkSelect, }: Props) => { const hasEditReviewerAccess = useMemo(() => { return permissions.EditAll || permissions.EditReviewers; @@ -307,11 +310,15 @@ const GlossaryDetailsRightPanel = ({
{isGlossary && ( - )}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx index dfc567c8e4da..c9db49c24f54 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx @@ -48,6 +48,7 @@ describe('GlossaryDetailsRightPanel', () => { isGlossary permissions={mockPermissions} selectedData={mockedGlossaries[0]} + onThreadLinkSelect={jest.fn()} onUpdate={jest.fn()} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx index dd420b940f5b..f3e43fca3c2e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx @@ -63,6 +63,7 @@ const GlossaryTermsV1 = ({ updateVote, refreshActiveGlossaryTerm, isVersionView, + onThreadLinkSelect, }: GlossaryTermsV1Props) => { const { fqn: glossaryFqn, @@ -130,6 +131,7 @@ const GlossaryTermsV1 = ({ isVersionView={isVersionView} permissions={permissions} selectedData={glossaryTerm} + onThreadLinkSelect={onThreadLinkSelect} onUpdate={(data) => handleGlossaryTermUpdate(data as GlossaryTerm)} /> ), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.interface.ts index a653b3d7ea3e..6bcb02186cc8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.interface.ts @@ -30,4 +30,5 @@ export interface GlossaryTermsV1Props { onEditGlossaryTerm: (glossaryTerm: GlossaryTerm) => void; updateVote?: (data: VotingDataProps) => Promise; refreshActiveGlossaryTerm?: () => void; + onThreadLinkSelect: (value: string) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx index da64b2c299a3..21a6ad52bd55 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx @@ -83,6 +83,7 @@ const mockProps = { refreshActiveGlossaryTerm: jest.fn(), onAddGlossaryTerm: jest.fn(), onEditGlossaryTerm: jest.fn(), + onThreadLinkSelect: jest.fn(), }; describe('Test Glossary-term component', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx index 004538048529..30a0336e9527 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx @@ -17,7 +17,7 @@ import { EntityType } from '../../../../enums/entity.enum'; import { Glossary } from '../../../../generated/entity/data/glossary'; import { GlossaryTerm } from '../../../../generated/entity/data/glossaryTerm'; import { ChangeDescription } from '../../../../generated/entity/type'; -import { TagLabel } from '../../../../generated/type/tagLabel'; +import { TagLabel, TagSource } from '../../../../generated/type/tagLabel'; import { getEntityName } from '../../../../utils/EntityUtils'; import { getEntityVersionByField, @@ -25,7 +25,8 @@ import { } from '../../../../utils/EntityVersionUtils'; import DescriptionV1 from '../../../common/EntityDescription/DescriptionV1'; import { OperationPermission } from '../../../PermissionProvider/PermissionProvider.interface'; -import TagsInput from '../../../TagsInput/TagsInput.component'; +import TagsContainerV2 from '../../../Tag/TagsContainerV2/TagsContainerV2'; +import { DisplayType } from '../../../Tag/TagsViewer/TagsViewer.interface'; import GlossaryDetailsRightPanel from '../../GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component'; import GlossaryTermReferences from './GlossaryTermReferences'; import GlossaryTermSynonyms from './GlossaryTermSynonyms'; @@ -37,6 +38,7 @@ type Props = { onUpdate: (data: GlossaryTerm | Glossary) => Promise; isGlossary: boolean; isVersionView?: boolean; + onThreadLinkSelect: (value: string) => void; }; const GlossaryOverviewTab = ({ @@ -45,6 +47,7 @@ const GlossaryOverviewTab = ({ onUpdate, isGlossary, isVersionView, + onThreadLinkSelect, }: Props) => { const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); @@ -110,14 +113,17 @@ const GlossaryOverviewTab = ({ setIsDescriptionEditable(false)} onDescriptionEdit={() => setIsDescriptionEditable(true)} onDescriptionUpdate={onDescriptionUpdate} + onThreadLinkSelect={onThreadLinkSelect} /> @@ -153,11 +159,15 @@ const GlossaryOverviewTab = ({ - @@ -171,6 +181,7 @@ const GlossaryOverviewTab = ({ isVersionView={isVersionView} permissions={permissions} selectedData={selectedData} + onThreadLinkSelect={onThreadLinkSelect} onUpdate={onUpdate} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx index 6a6a8c19cb5a..300077f9ede2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx @@ -50,6 +50,7 @@ describe('GlossaryOverviewTab', () => { isGlossary={isGlossary} permissions={permissions} selectedData={selectedData} + onThreadLinkSelect={jest.fn()} onUpdate={onUpdate} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx index 7955d6233108..0e441bcd446b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx @@ -25,9 +25,14 @@ import { getGlossaryTermDetailsPath, } from '../../constants/constants'; import { EntityAction } from '../../enums/entity.enum'; +import { + CreateThread, + ThreadType, +} from '../../generated/api/feed/createThread'; import { Glossary } from '../../generated/entity/data/glossary'; import { GlossaryTerm } from '../../generated/entity/data/glossaryTerm'; import { VERSION_VIEW_GLOSSARY_PERMISSION } from '../../mocks/Glossary.mock'; +import { postThread } from '../../rest/feedsAPI'; import { addGlossaryTerm, getGlossaryTerms, @@ -37,6 +42,8 @@ import { import { getEntityDeleteMessage } from '../../utils/CommonUtils'; import { DEFAULT_ENTITY_PERMISSION } from '../../utils/PermissionsUtils'; import { showErrorToast } from '../../utils/ToastUtils'; +import { useActivityFeedProvider } from '../ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; +import ActivityThreadPanel from '../ActivityFeed/ActivityThreadPanel/ActivityThreadPanel'; import EntityDeleteModal from '../Modals/EntityDeleteModal/EntityDeleteModal'; import { usePermissionProvider } from '../PermissionProvider/PermissionProvider'; import { @@ -69,6 +76,11 @@ const GlossaryV1 = ({ const { action, tab } = useParams<{ action: EntityAction; glossaryName: string; tab: string }>(); const history = useHistory(); + const [threadLink, setThreadLink] = useState(''); + const [threadType, setThreadType] = useState( + ThreadType.Conversation + ); + const { postFeed, deleteFeed, updateFeed } = useActivityFeedProvider(); const { getEntityPermission } = usePermissionProvider(); const [isLoading, setIsLoading] = useState(true); @@ -96,6 +108,17 @@ const GlossaryV1 = ({ [action] ); + const onThreadPanelClose = () => { + setThreadLink(''); + }; + + const onThreadLinkSelect = (link: string, threadType?: ThreadType) => { + setThreadLink(link); + if (threadType) { + setThreadType(threadType); + } + }; + const fetchGlossaryTerm = async ( params?: ListGlossaryTermsParams, refresh?: boolean @@ -139,6 +162,19 @@ const GlossaryV1 = ({ } }; + const createThread = async (data: CreateThread) => { + try { + await postThread(data); + } catch (error) { + showErrorToast( + error as AxiosError, + t('server.create-entity-error', { + entity: t('label.conversation'), + }) + ); + } + }; + const handleDelete = () => { const { id } = selectedData; if (isGlossaryActive) { @@ -329,6 +365,7 @@ const GlossaryV1 = ({ onEditGlossaryTerm={(term) => handleGlossaryTermModalAction(true, term) } + onThreadLinkSelect={onThreadLinkSelect} /> ) : ( handleGlossaryTermModalAction(true, term) } + onThreadLinkSelect={onThreadLinkSelect} /> ))} @@ -376,6 +414,19 @@ const GlossaryV1 = ({ onSave={handleGlossaryTermSave} /> )} + + {threadLink ? ( + + ) : null} ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Task/TaskTab/TaskTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Task/TaskTab/TaskTab.component.tsx index a5e481602a81..1548c199f66b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Task/TaskTab/TaskTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Task/TaskTab/TaskTab.component.tsx @@ -342,6 +342,7 @@ export const TaskTab = ({ ) : ( } menu={{ items: TASK_ACTION_LIST, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/TasksPage.interface.ts b/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/TasksPage.interface.ts index 0fc43a30a3aa..7ba42936193d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/TasksPage.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TasksPage/TasksPage.interface.ts @@ -16,6 +16,8 @@ import { Dashboard } from '../../generated/entity/data/dashboard'; import { DashboardDataModel } from '../../generated/entity/data/dashboardDataModel'; import { Database } from '../../generated/entity/data/database'; import { DatabaseSchema } from '../../generated/entity/data/databaseSchema'; +import { Glossary } from '../../generated/entity/data/glossary'; +import { GlossaryTerm } from '../../generated/entity/data/glossaryTerm'; import { Mlmodel } from '../../generated/entity/data/mlmodel'; import { Pipeline } from '../../generated/entity/data/pipeline'; import { SearchIndex } from '../../generated/entity/data/searchIndex'; @@ -34,7 +36,9 @@ export type EntityData = | Database | DatabaseSchema | DashboardDataModel - | SearchIndex; + | SearchIndex + | Glossary + | GlossaryTerm; export interface Option { label: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.ts index 59481045250d..31b438c3ef28 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/GlossaryUtils.ts @@ -15,7 +15,10 @@ import { AxiosError } from 'axios'; import { isUndefined, omit } from 'lodash'; import { StatusType } from '../components/common/StatusBadge/StatusBadge.interface'; import { ModifiedGlossaryTerm } from '../components/Glossary/GlossaryTermTab/GlossaryTermTab.interface'; -import { WILD_CARD_CHAR } from '../constants/char.constants'; +import { + FQN_SEPARATOR_CHAR, + WILD_CARD_CHAR, +} from '../constants/char.constants'; import { SearchIndex } from '../enums/search.enum'; import { Glossary } from '../generated/entity/data/glossary'; import { GlossaryTerm, Status } from '../generated/entity/data/glossaryTerm'; @@ -24,6 +27,8 @@ import { SearchResponse } from '../interface/search.interface'; import { ListGlossaryTermsParams } from '../rest/glossaryAPI'; import { searchData } from '../rest/miscAPI'; import { formatSearchGlossaryTermResponse } from './APIUtils'; +import Fqn from './Fqn'; +import { getGlossaryPath } from './RouterUtils'; export interface GlossaryTermTreeNode { children?: GlossaryTermTreeNode[]; @@ -234,3 +239,26 @@ export const StatusFilters = Object.values(Status) text: status, value: status, })); + +export const getGlossaryBreadcrumbs = (fqn: string) => { + const arr = Fqn.split(fqn); + const dataFQN: Array = []; + const breadcrumbList = [ + { + name: 'Glossaries', + url: getGlossaryPath(''), + activeTitle: false, + }, + ...arr.map((d) => { + dataFQN.push(d); + + return { + name: d, + url: getGlossaryPath(dataFQN.join(FQN_SEPARATOR_CHAR)), + activeTitle: false, + }; + }), + ]; + + return breadcrumbList; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts index e504e5d4a5f8..bdc7f1bb4c1e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TasksUtils.ts @@ -53,6 +53,7 @@ import { getDatabaseSchemaDetailsByFQN, } from '../rest/databaseAPI'; import { getDataModelDetailsByFQN } from '../rest/dataModelsAPI'; +import { getGlossariesByName, getGlossaryTermByFQN } from '../rest/glossaryAPI'; import { getUserSuggestions } from '../rest/miscAPI'; import { getMlModelByFQN } from '../rest/mlModelAPI'; import { getPipelineByFqn } from '../rest/pipelineAPI'; @@ -73,11 +74,12 @@ import { defaultFields as DataModelFields } from './DataModelsUtils'; import { defaultFields as TableFields } from './DatasetDetailsUtils'; import { getEntityName } from './EntityUtils'; import { getEntityFQN, getEntityType } from './FeedUtils'; +import { getGlossaryBreadcrumbs } from './GlossaryUtils'; import { defaultFields as MlModelFields } from './MlModelDetailsUtils'; import { defaultFields as PipelineFields } from './PipelineDetailsUtils'; import serviceUtilClassBase from './ServiceUtilClassBase'; import { STORED_PROCEDURE_DEFAULT_FIELDS } from './StoredProceduresUtils'; -import { getEncodedFqn } from './StringsUtils'; +import { getDecodedFqn, getEncodedFqn } from './StringsUtils'; import { getEntityLink } from './TableUtils'; import { showErrorToast } from './ToastUtils'; @@ -275,6 +277,8 @@ export const TASK_ENTITIES = [ EntityType.DASHBOARD_DATA_MODEL, EntityType.STORED_PROCEDURE, EntityType.SEARCH_INDEX, + EntityType.GLOSSARY, + EntityType.GLOSSARY_TERM, ]; export const getBreadCrumbList = ( @@ -308,12 +312,17 @@ export const getBreadCrumbList = ( const service = (serviceCategory: ServiceCategory) => { return { - name: getEntityName(entityData.service), - url: getEntityName(entityData.service) - ? getServiceDetailsPath(entityData.service?.name || '', serviceCategory) + name: getEntityName((entityData as Table).service), + url: getEntityName((entityData as Table).service) + ? getServiceDetailsPath( + (entityData as Table).service?.name ?? '', + serviceCategory + ) : '', - imgSrc: entityData.serviceType - ? serviceUtilClassBase.getServiceTypeLogo(entityData.serviceType) + imgSrc: (entityData as Table).serviceType + ? serviceUtilClassBase.getServiceTypeLogo( + (entityData as Table).serviceType as string + ) : undefined, }; }; @@ -372,6 +381,11 @@ export const getBreadCrumbList = ( ]; } + case EntityType.GLOSSARY: + case EntityType.GLOSSARY_TERM: { + return getGlossaryBreadcrumbs(entityData.fullyQualifiedName ?? ''); + } + default: return []; } @@ -485,6 +499,22 @@ export const fetchEntityDetail = ( .catch((err: AxiosError) => showErrorToast(err)); break; + case EntityType.GLOSSARY: + getGlossariesByName(entityFQN, TabSpecificField.TAGS) + .then((res) => { + setEntityData(res); + }) + .catch((err: AxiosError) => showErrorToast(err)); + + break; + case EntityType.GLOSSARY_TERM: + getGlossaryTermByFQN(getDecodedFqn(entityFQN), TabSpecificField.TAGS) + .then((res) => { + setEntityData(res); + }) + .catch((err: AxiosError) => showErrorToast(err)); + + break; default: break;