From d8f7bc293adbae5f0746faf420b61bce9255b896 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 22 Jan 2024 23:01:59 -0800 Subject: [PATCH 01/49] Fix #14786: Suggestions API --- .../native/1.3.0/mysql/schemaChanges.sql | 13 +- .../native/1.3.0/postgres/schemaChanges.sql | 11 + .../java/org/openmetadata/service/Entity.java | 4 +- .../service/jdbi3/CollectionDAO.java | 36 ++ .../service/jdbi3/EntityRepository.java | 5 + .../service/jdbi3/FeedRepository.java | 2 +- .../service/jdbi3/SuggestionRepository.java | 203 +++++++++++ .../resources/feeds/SuggestionsResource.java | 339 ++++++++++++++++++ .../openmetadata/service/util/UserUtil.java | 11 + .../feeds/SuggestionResourceTest.java | 125 +++++++ .../schema/api/feed/createSuggestion.json | 31 ++ .../json/schema/entity/feed/suggestion.json | 100 ++++++ .../json/schema/type/changeEventType.json | 7 +- 13 files changed, 883 insertions(+), 4 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql index 40afb8a3ad0e..25697b97178c 100644 --- a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql @@ -167,4 +167,15 @@ where serviceType = 'Mssql'; DELETE FROM event_subscription_entity; DELETE FROM change_event_consumers -DELETE FROM consumers_dlq; \ No newline at end of file +DELETE FROM consumers_dlq; + +CREATE TABLE IF NOT EXISTS suggestions ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, + suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.type') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + status VARCHAR(256) GENERATED ALWAYS AS (json -> '$.status') NOT NULL, + PRIMARY KEY (id) +); \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index ea78941b279b..efc8aefe2d86 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -180,3 +180,14 @@ where serviceType = 'Mssql'; DELETE FROM event_subscription_entity; DELETE FROM change_event_consumers DELETE FROM consumers_dlq; + +CREATE TABLE IF NOT EXISTS suggestions ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, + suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.type') NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, + status VARCHAR(256) GENERATED ALWAYS AS (json -> '$.status') NOT NULL, + PRIMARY KEY (id) +); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 72e049506cde..92d137f93b5b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -60,6 +60,7 @@ import org.openmetadata.service.jdbi3.FeedRepository; import org.openmetadata.service.jdbi3.LineageRepository; import org.openmetadata.service.jdbi3.Repository; +import org.openmetadata.service.jdbi3.SuggestionRepository; import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.jdbi3.TokenRepository; import org.openmetadata.service.jdbi3.UsageRepository; @@ -88,7 +89,7 @@ public final class Entity { @Getter @Setter private static SystemRepository systemRepository; @Getter @Setter private static ChangeEventRepository changeEventRepository; @Getter @Setter private static SearchRepository searchRepository; - + @Getter @Setter private static SuggestionRepository suggestionRepository; // List of all the entities private static final Set ENTITY_LIST = new TreeSet<>(); @@ -194,6 +195,7 @@ public final class Entity { // Other entities public static final String EVENT_SUBSCRIPTION = "eventsubscription"; public static final String THREAD = "THREAD"; + public static final String SUGGESTION = "SUGGESTION"; public static final String WORKFLOW = "workflow"; // diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 7563d056c4cd..d49719945c25 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -331,6 +331,9 @@ public interface CollectionDAO { @CreateSqlObject DocStoreDAO docStoreDAO(); + @CreateSqlObject + SuggestionDAO suggestionDAO(); + interface DashboardDAO extends EntityDAO { @Override default String getTableName() { @@ -4314,4 +4317,37 @@ int listCount( @Define("mysqlCond") String mysqlCond, @Define("psqlCond") String psqlCond); } + + interface SuggestionDAO { + @ConnectionAwareSqlUpdate( + value = "INSERT INTO suggestions(json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO suggestions(json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE suggestions SET json = :json where id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE suggestions SET json = (:json :: jsonb) where id = :id", + connectionType = POSTGRES) + void update(@BindUUID("id") UUID id, @Bind("json") String json); + + @SqlQuery("SELECT json FROM suggestions WHERE id = :id") + String findById(@BindUUID("id") UUID id); + + @SqlQuery("SELECT json FROM suggestions ORDER BY createdAt DESC") + List list(); + + @SqlQuery("SELECT count(id) FROM suggestions ") + int listCount(@Define("condition") String condition); + + @SqlUpdate("DELETE FROM suggestions WHERE id = :id") + void delete(@BindUUID("id") UUID id); + + @SqlQuery("SELECT json FROM thread_entity ORDER BY createdAt DESC LIMIT :limit") + List list(@Bind("limit") int limit, @Define("condition") String condition); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 71974bae8656..0cdd0fd9335d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -109,6 +109,7 @@ import org.openmetadata.schema.api.feed.ResolveTask; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.ApiStatus; @@ -1929,6 +1930,10 @@ public TaskWorkflow getTaskWorkflow(ThreadContext threadContext) { } } + public SuggestionRepository.SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { + return new SuggestionRepository.SuggestionWorkflow(suggestion); + } + public final void validateTaskThread(ThreadContext threadContext) { ThreadType threadType = threadContext.getThread().getType(); if (threadType != ThreadType.Task) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index 6c915007948a..06fd008076cc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -490,7 +490,7 @@ public DeleteResponse deletePost(Thread thread, Post post, String userName @Transaction public DeleteResponse deleteThread(Thread thread, String deletedByUser) { deleteThreadInternal(thread.getId()); - LOG.info("{} deleted thread with id {}", deletedByUser, thread.getId()); + LOG.debug("{} deleted thread with id {}", deletedByUser, thread.getId()); return new DeleteResponse<>(thread, ENTITY_DELETED); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java new file mode 100644 index 000000000000..621b6e985490 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -0,0 +1,203 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.schema.type.EventType.ENTITY_DELETED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_ACCEPTED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_REJECTED; +import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.schema.type.Relationship.CREATED; +import static org.openmetadata.schema.type.Relationship.IS_ABOUT; +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.jdbi3.UserRepository.TEAMS_FIELD; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import javax.json.JsonPatch; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.service.Entity; +import org.openmetadata.service.ResourceRegistry; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.resources.feeds.SuggestionsResource; +import org.openmetadata.service.security.AuthorizationException; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.RestUtil; + +@Slf4j +@Repository +public class SuggestionRepository { + private final CollectionDAO dao; + + public SuggestionRepository() { + this.dao = Entity.getCollectionDAO(); + Entity.setSuggestionRepository(this); + ResourceRegistry.addResource("suggestion", null, Entity.getEntityFields(Suggestion.class)); + } + + @Transaction + public Suggestion create(Suggestion suggestion) { + store(suggestion); + storeRelationships(suggestion); + return suggestion; + } + + @Transaction + public Suggestion update(Suggestion suggestion, String userName) { + suggestion.setUpdatedBy(userName); + dao.suggestionDAO().update(suggestion.getId(), JsonUtils.pojoToJson(suggestion)); + storeRelationships(suggestion); + return suggestion; + } + + @Transaction + public void store(Suggestion suggestion) { + // Insert a new Suggestion + dao.suggestionDAO().insert(JsonUtils.pojoToJson(suggestion)); + } + + @Transaction + public void storeRelationships(Suggestion suggestion) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + // Add relationship User -- created --> Thread relationship + dao.relationshipDAO() + .insert( + suggestion.getCreatedBy().getId(), + suggestion.getId(), + suggestion.getCreatedBy().getType(), + Entity.SUGGESTION, + CREATED.ordinal()); + + // Add field relationship for data asset - Thread -- isAbout ---> entity/entityField + dao.fieldRelationshipDAO() + .insert( + suggestion.getId().toString(), // from FQN + entityLink.getFullyQualifiedFieldValue(), // to FQN, + suggestion.getId().toString(), + entityLink.getFullyQualifiedFieldValue(), + Entity.SUGGESTION, // From type + entityLink.getFullyQualifiedFieldType(), // to Type + IS_ABOUT.ordinal(), + null); + } + + public Suggestion get(UUID id) { + return EntityUtil.validate(id, dao.suggestionDAO().findById(id), Suggestion.class); + } + + @Transaction + public RestUtil.DeleteResponse deleteSuggestion( + Suggestion suggestion, String deletedByUser) { + deleteSuggestionInternal(suggestion.getId()); + LOG.debug("{} deleted suggestion with id {}", deletedByUser, suggestion.getId()); + return new RestUtil.DeleteResponse<>(suggestion, ENTITY_DELETED); + } + + @Transaction + public void deleteSuggestionInternal(UUID id) { + // Delete all the relationships to other entities + dao.relationshipDAO().deleteAll(id, Entity.SUGGESTION); + + // Delete all the field relationships to other entities + dao.fieldRelationshipDAO().deleteAllByPrefix(id.toString()); + + // Finally, delete the suggestion + dao.suggestionDAO().delete(id); + } + + @Getter + public static class SuggestionWorkflow { + protected final Suggestion suggestion; + + SuggestionWorkflow(Suggestion suggestion) { + this.suggestion = suggestion; + } + + public EntityInterface acceptSuggestions(EntityInterface entityInterface) { + if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { + entityInterface.setTags(suggestion.getTagLabels()); + return entityInterface; + } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { + entityInterface.setDescription(suggestion.getDescription()); + return entityInterface; + } else { + throw new WebApplicationException("Invalid suggestion Type"); + } + } + } + + public RestUtil.PutResponse acceptSuggestion( + UriInfo uriInfo, Suggestion suggestion, String user) { + suggestion.setStatus(SuggestionStatus.Accepted); + acceptSuggestion(suggestion, user); + Suggestion updatedHref = SuggestionsResource.addHref(uriInfo, suggestion); + return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_ACCEPTED); + } + + protected void acceptSuggestion(Suggestion suggestion, String user) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityInterface entity = + Entity.getEntity( + entityLink, suggestion.getType() == SuggestionType.SuggestTagLabel ? "tags" : "", ALL); + String origJson = JsonUtils.pojoToJson(entity); + SuggestionWorkflow suggestionWorkflow = getSuggestionWorkflow(suggestion); + EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestions(entity); + String updatedEntityJson = JsonUtils.pojoToJson(updatedEntity); + JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); + repository.patch(null, entity.getId(), user, patch); + suggestion.setStatus(SuggestionStatus.Accepted); + update(suggestion, user); + } + + public RestUtil.PutResponse rejectSuggestion( + UriInfo uriInfo, Suggestion suggestion, String user) { + suggestion.setStatus(SuggestionStatus.Rejected); + update(suggestion, user); + Suggestion updatedHref = SuggestionsResource.addHref(uriInfo, suggestion); + return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_REJECTED); + } + + public void checkPermissionsForAcceptOrRejectSuggestion( + Suggestion suggestion, SuggestionStatus status, SecurityContext securityContext) { + String userName = securityContext.getUserPrincipal().getName(); + User user = Entity.getEntityByName(USER, userName, TEAMS_FIELD, NON_DELETED); + MessageParser.EntityLink about = MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityReference aboutRef = EntityUtil.validateEntityLink(about); + EntityReference ownerRef = Entity.getOwner(aboutRef); + User owner = + Entity.getEntityByName(USER, ownerRef.getFullyQualifiedName(), TEAMS_FIELD, NON_DELETED); + List userTeamNames = + user.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + List ownerTeamNames = + owner.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + if (Boolean.FALSE.equals(user.getIsAdmin()) + && !owner.getName().equals(userName) + && Collections.disjoint(userTeamNames, ownerTeamNames)) { + throw new AuthorizationException( + CatalogExceptionMessage.taskOperationNotAllowed(userName, status.value())); + } + } + + public SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); + return repository.getSuggestionWorkflow(suggestion); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java new file mode 100644 index 000000000000..e4a1e0692db0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -0,0 +1,339 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.resources.feeds; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.schema.type.EventType.SUGGESTION_CREATED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_UPDATED; +import static org.openmetadata.service.util.RestUtil.CHANGE_CUSTOM_HEADER; + +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.List; +import java.util.UUID; +import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.api.feed.CreateSuggestion; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.SuggestionRepository; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.tags.TagLabelUtil; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.PostResourceContext; +import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; +import org.openmetadata.service.util.UserUtil; + +@Path("/v1/suggestions") +@Tag( + name = "Suggestions", + description = + "Suggestions API supports ability to add suggestion for descriptions or tag labels for Entities.") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "suggestions") +public class SuggestionsResource { + public static final String COLLECTION_PATH = "/v1/suggestions/"; + private final SuggestionRepository dao; + private final Authorizer authorizer; + + public static void addHref(UriInfo uriInfo, List suggestions) { + if (uriInfo != null) { + suggestions.forEach(t -> addHref(uriInfo, t)); + } + } + + public static Suggestion addHref(UriInfo uriInfo, Suggestion suggestion) { + if (uriInfo != null) { + suggestion.setHref(RestUtil.getHref(uriInfo, COLLECTION_PATH, suggestion.getId())); + } + return suggestion; + } + + public SuggestionsResource(Authorizer authorizer) { + this.dao = Entity.getSuggestionRepository(); + this.authorizer = authorizer; + } + + public static class SuggestionList extends ResultList { + /* Required for serde */ + } + + @GET + @Operation( + operationId = "listSuggestions", + summary = "List Suggestions", + description = + "Get a list of suggestions, optionally filtered by `entityLink` or `entityFQN`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of Suggestions", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SuggestionList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Parameter( + description = + "Limit the number of suggestions returned. (1 to 1000000, default = 10)") + @DefaultValue("10") + @Min(1) + @Max(1000000) + @QueryParam("limit") + int limitParam, + @Parameter( + description = "Returns list of threads before this cursor", + schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter( + description = "Returns list of threads after this cursor", + schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = + "Filter threads by entity link of entity about which this thread is created", + schema = + @Schema(type = "string", example = "")) + @QueryParam("entityLink") + String entityLink, + @Parameter( + description = + "Filter threads by user id or bot id. This filter requires a 'filterType' query param.", + schema = @Schema(type = "string")) + @QueryParam("userId") + UUID userId, + @Parameter( + description = + "Filter threads by whether they are accepted or rejected. By default status is OPEN.") + @DefaultValue("OPEN") + @QueryParam("status") + String status) { + RestUtil.validateCursors(before, after); + return null; + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getSuggestionByID", + summary = "Get a suggestion by Id", + description = "Get a suggestion by `Id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Suggestion", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse( + responseCode = "404", + description = "Suggestion for instance {id} is not found") + }) + public Suggestion get( + @Context UriInfo uriInfo, + @Parameter(description = "Id of the Thread", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + return addHref(uriInfo, dao.get(id)); + } + + @PUT + @Path("/{id}/accept") + @Operation( + operationId = "acceptSuggestion", + summary = "Close a task", + description = "Close a task without making any changes to the entity.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The task thread.", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response acceptSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the suggestion", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + Suggestion suggestion = dao.get(id); + dao.checkPermissionsForAcceptOrRejectSuggestion( + suggestion, suggestion.getStatus(), securityContext); + return dao.acceptSuggestion(uriInfo, suggestion, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + + @PUT + @Path("/{id}") + @Operation( + operationId = "updateSuggestion", + summary = "Update a suggestion by `Id`.", + description = "Update an existing suggestion using JsonPatch.", + externalDocs = + @ExternalDocumentation( + description = "JsonPatch RFC", + url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response updateSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Suggestion", schema = @Schema(type = "string")) + @PathParam("id") + String id, + @Valid Suggestion suggestion) { + suggestion.setCreatedBy(UserUtil.getUserOrBot(securityContext.getUserPrincipal().getName())); + suggestion.setCreatedAt(System.currentTimeMillis()); + addHref(uriInfo, dao.update(suggestion, securityContext.getUserPrincipal().getName())); + return Response.created(suggestion.getHref()) + .entity(suggestion) + .header(CHANGE_CUSTOM_HEADER, SUGGESTION_UPDATED) + .build(); + } + + @POST + @Operation( + operationId = "createSuggestion", + summary = "Create a Suggestion", + description = + "Create a new Suggestion. A Suggestion is created about a data asset when a user suggests an update.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The thread", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateSuggestion create) { + Suggestion suggestion = getSuggestion(securityContext, create); + addHref(uriInfo, dao.create(suggestion)); + return Response.created(suggestion.getHref()) + .entity(suggestion) + .header(CHANGE_CUSTOM_HEADER, SUGGESTION_CREATED) + .build(); + } + + @DELETE + @Path("/{suggestionId}") + @Operation( + operationId = "deleteSuggestion", + summary = "Delete a Suggestion by Id", + description = "Delete an existing Suggestion and all its relationships.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "thread with {threadId} is not found"), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response deleteSuggestion( + @Context SecurityContext securityContext, + @Parameter( + description = "ThreadId of the thread to be deleted", + schema = @Schema(type = "string")) + @PathParam("suggestionId") + UUID suggestionId) { + // validate and get the thread + Suggestion suggestion = dao.get(suggestionId); + // delete thread only if the admin/bot/author tries to delete it + OperationContext operationContext = + new OperationContext(Entity.SUGGESTION, MetadataOperation.DELETE); + ResourceContextInterface resourceContext = + new PostResourceContext(suggestion.getCreatedBy().getName()); + authorizer.authorize(securityContext, operationContext, resourceContext); + return dao.deleteSuggestion(suggestion, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + + private Suggestion getSuggestion(SecurityContext securityContext, CreateSuggestion create) { + validate(create); + return new Suggestion() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withEntityLink(create.getEntityLink()) + .withType(create.getType()) + .withDescription(create.getDescription()) + .withTagLabels(create.getTagLabels()) + .withStatus(SuggestionStatus.Open) + .withCreatedBy(UserUtil.getUserOrBot(securityContext.getUserPrincipal().getName())) + .withCreatedAt(System.currentTimeMillis()) + .withUpdatedBy(securityContext.getUserPrincipal().getName()) + .withUpdatedAt(System.currentTimeMillis()); + } + + private void validate(CreateSuggestion suggestion) { + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + Entity.getEntityReferenceByName( + entityLink.getEntityType(), entityLink.getEntityFQN(), Include.NON_DELETED); + if (suggestion.getType() == SuggestionType.SuggestDescription) { + if (suggestion.getDescription() == null || suggestion.getDescription().isEmpty()) { + throw new WebApplicationException("Suggestion's description cannot be empty."); + } + } else if (suggestion.getType() == SuggestionType.SuggestTagLabel) { + if (suggestion.getTagLabels().isEmpty()) { + throw new WebApplicationException("Suggestion's tag label's cannot be empty."); + } else { + for (TagLabel label : listOrEmpty(suggestion.getTagLabels())) { + TagLabelUtil.applyTagCommonFields(label); + } + } + } else { + throw new WebApplicationException("Invalid Suggestion Type."); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java index bb24c81ef28a..eee5948b8b09 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java @@ -16,6 +16,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.entity.teams.AuthenticationMechanism.AuthType.JWT; +import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import at.favre.lib.crypto.bcrypt.BCrypt; @@ -228,4 +229,14 @@ public static List getRoleForBot(String botName) { }; return listOf(RoleResource.getRole(botRole)); } + + public static EntityReference getUserOrBot(String name) { + EntityReference userOrBot; + try { + userOrBot = Entity.getEntityReferenceByName(Entity.USER, name, NON_DELETED); + } catch (EntityNotFoundException e) { + userOrBot = Entity.getEntityReferenceByName(Entity.BOT, name, NON_DELETED); + } + return userOrBot; + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java new file mode 100644 index 000000000000..e4ea1172247c --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java @@ -0,0 +1,125 @@ +package org.openmetadata.service.resources.feeds; + +import static org.openmetadata.service.resources.EntityResourceTest.C1; +import static org.openmetadata.service.security.SecurityUtil.authHeaders; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.feed.CreateSuggestion; +import org.openmetadata.schema.api.teams.CreateTeam; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.teams.TeamResourceTest; +import org.openmetadata.service.resources.teams.UserResourceTest; +import org.openmetadata.service.util.TestUtils; + +@Slf4j +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class SuggestionResourceTest extends OpenMetadataApplicationTest { + public static Table TABLE; + public static Table TABLE2; + public static String TABLE_LINK; + public static String TABLE_COLUMN_LINK; + public static String TABLE_DESCRIPTION_LINK; + public static List COLUMNS; + public static User USER; + public static String USER_LINK; + public static Map USER_AUTH_HEADERS; + public static User USER2; + public static Map USER2_AUTH_HEADERS; + public static Team TEAM; + public static Team TEAM2; + public static String TEAM_LINK; + + public static TableResourceTest TABLE_RESOURCE_TEST; + + @BeforeAll + public void setup(TestInfo test) throws IOException, URISyntaxException { + TABLE_RESOURCE_TEST = new TableResourceTest(); + TABLE_RESOURCE_TEST.setup(test); // Initialize TableResourceTest for using helper methods + + UserResourceTest userResourceTest = new UserResourceTest(); + USER2 = + userResourceTest.createEntity(userResourceTest.createRequest(test, 4), ADMIN_AUTH_HEADERS); + USER2_AUTH_HEADERS = authHeaders(USER2.getName()); + + CreateTable createTable = + TABLE_RESOURCE_TEST.createRequest(test).withOwner(TableResourceTest.USER1_REF); + TABLE = TABLE_RESOURCE_TEST.createAndCheckEntity(createTable, ADMIN_AUTH_HEADERS); + + TeamResourceTest teamResourceTest = new TeamResourceTest(); + CreateTeam createTeam = + teamResourceTest + .createRequest(test, 4) + .withDisplayName("Team2") + .withDescription("Team2 description") + .withUsers(List.of(USER2.getId())); + TEAM2 = teamResourceTest.createAndCheckEntity(createTeam, ADMIN_AUTH_HEADERS); + EntityReference TEAM2_REF = TEAM2.getEntityReference(); + + CreateTable createTable2 = TABLE_RESOURCE_TEST.createRequest(test); + createTable2.withName("table2").withOwner(TEAM2_REF); + TABLE2 = TABLE_RESOURCE_TEST.createAndCheckEntity(createTable2, ADMIN_AUTH_HEADERS); + + COLUMNS = + Collections.singletonList( + new Column().withName("column1").withDataType(ColumnDataType.BIGINT)); + TABLE_LINK = String.format("<#E::table::%s>", TABLE.getFullyQualifiedName()); + TABLE_COLUMN_LINK = + String.format( + "<#E::table::%s::columns::" + C1 + "::description>", TABLE.getFullyQualifiedName()); + TABLE_DESCRIPTION_LINK = + String.format("<#E::table::%s::description>", TABLE.getFullyQualifiedName()); + + USER = TableResourceTest.USER1; + USER_LINK = String.format("<#E::user::%s>", USER.getFullyQualifiedName()); + USER_AUTH_HEADERS = authHeaders(USER.getName()); + + TEAM = TableResourceTest.TEAM1; + TEAM_LINK = String.format("<#E::team::%s>", TEAM.getFullyQualifiedName()); + } + + @Test + void post_validThreadAndList_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + } + + public Suggestion createSuggestion(CreateSuggestion create, Map authHeaders) + throws HttpResponseException { + return TestUtils.post(getResource("suggestions"), create, Suggestion.class, authHeaders); + } + + public CreateSuggestion create() { + String entityLink = String.format("<#E::%s::%s>", Entity.TABLE, TABLE.getFullyQualifiedName()); + return new CreateSuggestion() + .withDescription("Update description") + .withType(SuggestionType.SuggestDescription) + .withEntityLink(entityLink); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json b/openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json new file mode 100644 index 000000000000..91b4b8c4a958 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/feed/createSuggestion.json @@ -0,0 +1,31 @@ +{ + "$id": "https://open-metadata.org/schema/api/feed/createSuggestion.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateSuggestionRequest", + "description": "Create Suggestion request", + "type": "object", + "properties": { + "description": { + "description": "Message in Markdown format. See markdown support for more details.", + "type": "string" + }, + "tagLabels": { + "description": "Tags or Glossary Terms.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "type": { + "$ref": "../../entity/feed/suggestion.json#/definitions/suggestionType" + }, + "entityLink": { + "description": "Data asset about which this thread is created for with format <#E::{entities}::{entityName}::{field}::{fieldValue}.", + "$ref": "../../type/basic.json#/definitions/entityLink" + } + }, + "oneOf": [{"required": ["suggestionType", "entityLink", "description"]}, + {"required": ["suggestionType", "entityLink","tagLabels"]}], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json b/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json new file mode 100644 index 000000000000..cb6ebfd171fb --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json @@ -0,0 +1,100 @@ +{ + "$id": "https://open-metadata.org/schema/entity/feed/thread.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Thread", + "description": "This schema defines the Thread entity. A Thread is a collection of posts made by the users. The first post that starts a thread is **about** a data asset **from** a user. Other users can respond to this post by creating new posts in the thread. Note that bot users can also interact with a thread. A post can contains links that mention Users or other Data Assets.", + "type": "object", + + "definitions": { + "suggestionType": { + "javaType": "org.openmetadata.schema.type.SuggestionType", + "description": "Type of a Suggestion.", + "type": "string", + "enum": [ + "SuggestDescription", + "SuggestTagLabel" + ], + "javaEnums": [ + { + "name": "SuggestDescription" + }, + { + "name": "SuggestTagLabel" + } + ] + }, + "suggestionStatus": { + "javaType": "org.openmetadata.schema.type.SuggestionStatus", + "type": "string", + "description": "Status of a Suggestion.", + "enum": [ + "Open", + "Accepted", + "Rejected" + ], + "javaEnums": [ + { + "name": "Open" + }, + { + "name": "Accepted" + }, + { + "name": "Rejected" + } + ], + "default": "Open" + } + }, + "properties": { + "id": { + "description": "Unique identifier that identifies an entity instance.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "type": { + "$ref": "#/definitions/suggestionType" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "entityLink": { + "description": "Data asset about which this thread is created for with format <#E::{entities}::{entityName}::{field}::{fieldValue}.", + "$ref": "../../type/basic.json#/definitions/entityLink" + }, + "createdAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "createdBy": { + "description": "User or Bot who made the suggestion.", + "$ref": "../../type/entityReference.json" + }, + "updatedAt": { + "description": "Last update time corresponding to the update version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User or Bot who updated the suggestion.", + "type": "string" + }, + "status": { + "$ref": "#/definitions/suggestionStatus" + }, + "description": { + "description": "The main message of the thread in Markdown format.", + "type": "string" + }, + "tagLabels": { + "description": "Tags or Glossary Terms.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + } + }, + "oneOf": [{"required": ["id", "entityLink", "description", "suggestionType"]}, + {"required": ["id", "entityLink", "tagLabels", "suggestionType"]}], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json b/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json index e7cefd016f56..ae0bd94aec56 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/changeEventType.json @@ -19,6 +19,11 @@ "postUpdated", "taskResolved", "taskClosed", - "logicalTestCaseAdded" + "logicalTestCaseAdded", + "suggestionCreated", + "suggestionUpdated", + "suggestionAccepted", + "suggestionRejected", + "suggestionDeleted" ] } From cd1125269f404f1aef7ecb49b27385293c59ac8c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 23 Jan 2024 14:52:33 +0100 Subject: [PATCH 02/49] Handle suggestions in ometa --- .../processor/test_case_runner.py | 11 +- .../src/metadata/great_expectations/action.py | 5 +- .../src/metadata/ingestion/ometa/ometa_api.py | 20 ++- .../src/metadata/ingestion/ometa/routes.py | 5 + .../source/database/dbt/dbt_utils.py | 4 +- ingestion/src/metadata/utils/entity_link.py | 26 +++- .../tests/integration/integration_base.py | 85 +++++++++-- .../ometa/test_ometa_suggestion_api.py | 136 ++++++++++++++++++ .../json/schema/entity/feed/suggestion.json | 5 +- 9 files changed, 264 insertions(+), 33 deletions(-) create mode 100644 ingestion/tests/integration/ometa/test_ometa_suggestion_api.py diff --git a/ingestion/src/metadata/data_quality/processor/test_case_runner.py b/ingestion/src/metadata/data_quality/processor/test_case_runner.py index 4434d1049b6d..7c97e074f9bf 100644 --- a/ingestion/src/metadata/data_quality/processor/test_case_runner.py +++ b/ingestion/src/metadata/data_quality/processor/test_case_runner.py @@ -28,6 +28,7 @@ test_suite_source_factory, ) from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest +from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -196,8 +197,9 @@ def compare_and_create_test_cases( ), entityLink=EntityLink( __root__=entity_link.get_entity_link( - table_fqn, - test_case_to_create.columnName, + Table, + fqn=table_fqn, + column_name=test_case_to_create.columnName, ) ), testSuite=test_suite_fqn, @@ -248,8 +250,9 @@ def _update_test_cases( updated_test_case = self.metadata.patch_test_case_definition( source=test_case, entity_link=entity_link.get_entity_link( - table_fqn, - test_case_definition.columnName, + Table, + fqn=table_fqn, + column_name=test_case_definition.columnName, ), test_case_parameter_values=test_case_definition.parameterValues, ) diff --git a/ingestion/src/metadata/great_expectations/action.py b/ingestion/src/metadata/great_expectations/action.py index c34b685bf422..b127f224dc5e 100644 --- a/ingestion/src/metadata/great_expectations/action.py +++ b/ingestion/src/metadata/great_expectations/action.py @@ -420,8 +420,9 @@ def _handle_test_case( test_case = self.ometa_conn.get_or_create_test_case( test_case_fqn, entity_link=get_entity_link( - table_entity.fullyQualifiedName.__root__, - fqn.split_test_case_fqn(test_case_fqn).column, + Table, + fqn=table_entity.fullyQualifiedName.__root__, + column_name=fqn.split_test_case_fqn(test_case_fqn).column, ), test_suite_fqn=test_suite.fullyQualifiedName.__root__, test_definition_fqn=test_definition.fullyQualifiedName.__root__, diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index 881390b8ec9d..e17a302143ca 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -246,11 +246,9 @@ def get_entity_from_create(self, create: Type[C]) -> Type[T]: ) return entity_class - def create_or_update(self, data: C) -> T: + def _create(self, data: C, method: str) -> T: """ - We allow CreateEntity for PUT, so we expect a type C. - - We PUT to the endpoint and return the Entity generated result + Internal logic to run POST vs. PUT """ entity = data.__class__ is_create = "create" in data.__class__.__name__.lower() @@ -262,15 +260,23 @@ def create_or_update(self, data: C) -> T: raise InvalidEntityException( f"PUT operations need a CreateEntity, not {entity}" ) - resp = self.client.put( - self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder) - ) + + fn = getattr(self.client, method) + resp = fn(self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder)) if not resp: raise EmptyPayloadException( f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.json()}" ) return entity_class(**resp) + def create_or_update(self, data: C) -> T: + """Run a PUT requesting via create request C""" + return self._create(data=data, method="put") + + def create(self, data: C) -> T: + """Run a POST requesting via create request C""" + return self._create(data=data, method="post") + def get_by_name( self, entity: Type[T], diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index 6cf70529ad07..3bb28585d580 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -50,6 +50,7 @@ CreateDataProductRequest, ) from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest +from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest from metadata.generated.schema.api.services.createDashboardService import ( @@ -113,6 +114,7 @@ from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.domains.dataProduct import DataProduct from metadata.generated.schema.entity.domains.domain import Domain +from metadata.generated.schema.entity.feed.suggestion import Suggestion from metadata.generated.schema.entity.policies.policy import Policy from metadata.generated.schema.entity.services.connections.testConnectionDefinition import ( TestConnectionDefinition, @@ -225,4 +227,7 @@ CreateDomainRequest.__name__: "/domains", DataProduct.__name__: "/dataProducts", CreateDataProductRequest.__name__: "/dataProducts", + # Suggestions + Suggestion.__name__: "/suggestions", + CreateSuggestionRequest.__name__: "/suggestions", } diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py index f5f7bf407e16..70bfcabe1b13 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_utils.py @@ -14,6 +14,7 @@ import traceback from typing import Optional, Union +from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.tests.testSuite import TestSuite from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -81,7 +82,8 @@ def generate_entity_link(dbt_test): manifest_node = dbt_test.get(DbtCommonEnum.MANIFEST_NODE.value) entity_link_list = [ entity_link.get_entity_link( - table_fqn=table_fqn, + Table, + fqn=table_fqn, column_name=manifest_node.column_name if hasattr(manifest_node, "column_name") else None, diff --git a/ingestion/src/metadata/utils/entity_link.py b/ingestion/src/metadata/utils/entity_link.py index 6ad0c2f0826b..4c34073c190c 100644 --- a/ingestion/src/metadata/utils/entity_link.py +++ b/ingestion/src/metadata/utils/entity_link.py @@ -13,17 +13,23 @@ Filter information has been taken from the ES indexes definitions """ -from typing import List, Optional +from functools import singledispatch +from typing import List, Optional, Type, TypeVar from antlr4.CommonTokenStream import CommonTokenStream from antlr4.error.ErrorStrategy import BailErrorStrategy from antlr4.InputStream import InputStream from antlr4.tree.Tree import ParseTreeWalker +from pydantic import BaseModel from requests.compat import unquote_plus from metadata.antlr.split_listener import EntityLinkSplitListener from metadata.generated.antlr.EntityLinkLexer import EntityLinkLexer from metadata.generated.antlr.EntityLinkParser import EntityLinkParser +from metadata.generated.schema.entity.data.table import Table +from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP + +T = TypeVar("T", bound=BaseModel) class EntityLinkBuildingException(Exception): @@ -86,16 +92,24 @@ def get_table_or_column_fqn(entity_link: str) -> str: ) -def get_entity_link(table_fqn: str, column_name: Optional[str]) -> str: +@singledispatch +def get_entity_link(entity_type: Type[T], fqn: str, **__) -> str: """From table fqn and column name get the entity_link Args: - table_fqn: table fqn - column_name: Optional param to generate entity link with column name + entity_type: Entity being built + fqn: Entity fqn """ + return f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}>" + + +@get_entity_link.register(Table) +def _(entity_type: Table, fqn: str, column_name: Optional[str]) -> str: + """From table fqn and column name get the entity_link""" + if column_name: - entity_link = f"<#E::table::" f"{table_fqn}" f"::columns::" f"{column_name}>" + entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}::columns::{column_name}>" else: - entity_link = f"<#E::table::" f"{table_fqn}>" + entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}>" return entity_link diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index 15e861f79cf0..cc558de4a229 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -18,20 +18,42 @@ from airflow import DAG from airflow.operators.bash import BashOperator +from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest +from metadata.generated.schema.api.data.createDatabaseSchema import ( + CreateDatabaseSchemaRequest, +) from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest +from metadata.generated.schema.api.data.createTable import CreateTableRequest +from metadata.generated.schema.api.services.createDatabaseService import ( + CreateDatabaseServiceRequest, +) from metadata.generated.schema.api.services.createPipelineService import ( CreatePipelineServiceRequest, ) +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.pipeline import Pipeline, Task +from metadata.generated.schema.entity.data.table import ( + Column, + ColumnName, + DataType, + Table, +) +from metadata.generated.schema.entity.services.connections.database.customDatabaseConnection import ( + CustomDatabaseConnection, + CustomDatabaseType, +) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( AuthProvider, OpenMetadataConnection, ) -from metadata.generated.schema.entity.services.connections.pipeline.airflowConnection import ( - AirflowConnection, +from metadata.generated.schema.entity.services.connections.pipeline.customPipelineConnection import ( + CustomPipelineConnection, + CustomPipelineType, ) -from metadata.generated.schema.entity.services.connections.pipeline.backendConnection import ( - BackendConnection, +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, + DatabaseServiceType, ) from metadata.generated.schema.entity.services.pipelineService import ( PipelineConnection, @@ -45,6 +67,7 @@ from metadata.ingestion.models.custom_pydantic import CustomSecretStr from metadata.ingestion.ometa.ometa_api import C, OpenMetadata, T from metadata.utils.dispatch import class_register +from src.metadata.generated.schema.entity.data.database import Database OM_JWT = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" @@ -58,7 +81,6 @@ def int_admin_ometa(url: str = "http://localhost:8585/api") -> OpenMetadata: ) metadata = OpenMetadata(server_config) assert metadata.health_check() - return metadata @@ -89,12 +111,21 @@ def _(name: EntityName) -> C: """Prepare a Create service request""" return CreatePipelineServiceRequest( name=name, - serviceType=PipelineServiceType.Airflow, + serviceType=PipelineServiceType.CustomPipeline, connection=PipelineConnection( - config=AirflowConnection( - hostPort="http://localhost:8080", - connection=BackendConnection(), - ), + config=CustomPipelineConnection(type=CustomPipelineType.CustomPipeline) + ), + ) + + +@create_service_registry.add(DatabaseService) +def _(name: EntityName) -> C: + """Prepare a Create service request""" + return CreateDatabaseServiceRequest( + name=name, + serviceType=DatabaseServiceType.CustomDatabase, + connection=DatabaseConnection( + config=CustomDatabaseConnection(type=CustomDatabaseType.CustomDatabase) ), ) @@ -132,6 +163,40 @@ def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: ) +@create_entity_registry.add(Database) +def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: + return CreateDatabaseRequest( + name=name, + service=reference, + ) + + +@create_entity_registry.add(DatabaseSchema) +def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: + return CreateDatabaseSchemaRequest( + name=name, + database=reference, + ) + + +@create_entity_registry.add(Table) +def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: + return CreateTableRequest( + name=name, + databaseSchema=reference, + columns=[ + Column(name=ColumnName(__root__="id"), dataType=DataType.BIGINT), + Column(name=ColumnName(__root__="name"), dataType=DataType.STRING), + Column(name=ColumnName(__root__="age"), dataType=DataType.INT), + Column( + name=ColumnName(__root__="address"), + dataType=DataType.VARCHAR, + dataLength=256, + ), + ], + ) + + def get_test_dag(name: str) -> DAG: """Get a DAG with the tasks created in the CreatePipelineRequest""" with DAG(name, start_date=datetime(2021, 1, 1)) as dag: diff --git a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py new file mode 100644 index 000000000000..0f4018a5e2cc --- /dev/null +++ b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py @@ -0,0 +1,136 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenMetadata high-level API Suggestion test +""" +from unittest import TestCase + +from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.feed.suggestion import SuggestionType +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.type.basic import EntityLink +from metadata.generated.schema.type.tagLabel import ( + LabelType, + State, + TagLabel, + TagSource, +) +from metadata.utils.entity_link import get_entity_link + +from ..integration_base import ( + generate_name, + get_create_entity, + get_create_service, + int_admin_ometa, +) + + +class OMetaSuggestionTest(TestCase): + """ + Run this integration test with the local API available + Install the ingestion package before running the tests + """ + + metadata = int_admin_ometa() + + service_name = generate_name() + db_name = generate_name() + schema_name = generate_name() + table_name = generate_name() + + @classmethod + def setUpClass(cls) -> None: + """ + Prepare ingredients: Pipeline Entity + """ + create_service = get_create_service( + entity=DatabaseService, name=cls.service_name + ) + cls.metadata.create_or_update(create_service) + + create_database = get_create_entity( + entity=Database, name=cls.schema_name, reference=cls.service_name.__root__ + ) + cls.database: Database = cls.metadata.create_or_update(create_database) + + create_schema = get_create_entity( + entity=DatabaseSchema, + name=cls.schema_name, + reference=cls.database.fullyQualifiedName.__root__, + ) + cls.schema: DatabaseSchema = cls.metadata.create_or_update(create_schema) + + create_table = get_create_entity( + entity=Table, + name=cls.schema_name, + reference=cls.schema.fullyQualifiedName.__root__, + ) + cls.table: Table = cls.metadata.create_or_update(create_table) + + @classmethod + def tearDownClass(cls) -> None: + """ + Clean up + """ + + service_id = str( + cls.metadata.get_by_name( + entity=DatabaseService, fqn=cls.service_name.__root__ + ).id.__root__ + ) + + cls.metadata.delete( + entity=DatabaseService, + entity_id=service_id, + recursive=True, + hard_delete=True, + ) + + def test_create_description_suggestion(self): + """We can create a suggestion""" + suggestion_request = CreateSuggestionRequest( + description="something", + type=SuggestionType.SuggestDescription, + entityLink=EntityLink( + __root__=get_entity_link( + Table, fqn=self.table.fullyQualifiedName.__root__ + ) + ), + ) + + # Suggestions only support POST (not PUT) + self.metadata.create(suggestion_request) + + def test_create_tag_suggestion(self): + """We can create a suggestion""" + suggestion_request = CreateSuggestionRequest( + tagLabels=[ + TagLabel( + tagFQN="PII.Sensitive", + labelType=LabelType.Automated, + state=State.Suggested.value, + source=TagSource.Classification, + ) + ], + type=SuggestionType.SuggestTagLabel, + entityLink=EntityLink( + __root__=get_entity_link( + Table, fqn=self.table.fullyQualifiedName.__root__ + ) + ), + ) + + # Suggestions only support POST (not PUT) + self.metadata.create(suggestion_request) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json b/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json index cb6ebfd171fb..271b7731d1da 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/feed/suggestion.json @@ -1,10 +1,9 @@ { "$id": "https://open-metadata.org/schema/entity/feed/thread.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Thread", - "description": "This schema defines the Thread entity. A Thread is a collection of posts made by the users. The first post that starts a thread is **about** a data asset **from** a user. Other users can respond to this post by creating new posts in the thread. Note that bot users can also interact with a thread. A post can contains links that mention Users or other Data Assets.", + "title": "Suggestion", + "description": "This schema defines the Suggestion entity. A suggestion can be applied to an asset to give the owner context about possible changes or improvements to descriptions, tags,...", "type": "object", - "definitions": { "suggestionType": { "javaType": "org.openmetadata.schema.type.SuggestionType", From 07f3a18d82adbad5cfa8c11dadcd57b7c6c0f8b0 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:28:02 +0530 Subject: [PATCH 03/49] Minor: Optimise Databricks Client (#14776) --- .../source/database/databricks/client.py | 86 +++++++++---------- .../source/database/databricks/lineage.py | 2 +- .../source/database/databricks/usage.py | 2 +- .../pipeline/databrickspipeline/metadata.py | 4 +- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/client.py b/ingestion/src/metadata/ingestion/source/database/databricks/client.py index 86a442717054..addbc926f5e5 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/client.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/client.py @@ -14,7 +14,7 @@ import json import traceback from datetime import timedelta -from typing import List +from typing import Iterable, List import requests @@ -31,6 +31,12 @@ QUERIES_PATH = "/sql/history/queries" +class DatabricksClientException(Exception): + """ + Class to throw auth and other databricks api exceptions. + """ + + class DatabricksClient: """ DatabricksClient creates a Databricks connection based on DatabricksCredentials. @@ -60,14 +66,33 @@ def test_query_api_access(self) -> None: if res.status_code != 200: raise APIError(res.json) + def _run_query_paginator(self, data, result, end_time, response): + while True: + if response: + next_page_token = response.get("next_page_token", None) + has_next_page = response.get("has_next_page", None) + if next_page_token: + data["page_token"] = next_page_token + if not has_next_page: + data = {} + break + else: + break + + if result[-1]["execution_end_time_ms"] <= end_time: + response = self.client.get( + self.base_query_url, + data=json.dumps(data), + headers=self.headers, + timeout=API_TIMEOUT, + ).json() + yield from response.get("res") or [] + def list_query_history(self, start_date=None, end_date=None) -> List[dict]: """ Method returns List the history of queries through SQL warehouses """ - query_details = [] try: - next_page_token = None - has_next_page = None data = {} daydiff = end_date - start_date @@ -98,36 +123,15 @@ def list_query_history(self, start_date=None, end_date=None) -> List[dict]: result = response.get("res") or [] data = {} - while True: - if result: - query_details.extend(result) - - next_page_token = response.get("next_page_token", None) - has_next_page = response.get("has_next_page", None) - if next_page_token: - data["page_token"] = next_page_token - - if not has_next_page: - data = {} - break - else: - break - - if result[-1]["execution_end_time_ms"] <= end_time: - response = self.client.get( - self.base_query_url, - data=json.dumps(data), - headers=self.headers, - timeout=API_TIMEOUT, - ).json() - result = response.get("res") + yield from result + yield from self._run_query_paginator( + data=data, result=result, end_time=end_time, response=response + ) or [] except Exception as exc: logger.debug(traceback.format_exc()) logger.error(exc) - return query_details - def is_query_valid(self, row) -> bool: query_text = row.get("query_text") return not ( @@ -137,18 +141,19 @@ def is_query_valid(self, row) -> bool: def list_jobs_test_connection(self) -> None: data = {"limit": 1, "expand_tasks": True, "offset": 0} - self.client.get( + response = self.client.get( self.jobs_list_url, data=json.dumps(data), headers=self.headers, timeout=API_TIMEOUT, - ).json() + ) + if response.status_code != 200: + raise DatabricksClientException(response.text) - def list_jobs(self) -> List[dict]: + def list_jobs(self) -> Iterable[dict]: """ Method returns List all the created jobs in a Databricks Workspace """ - job_list = [] try: data = {"limit": 25, "expand_tasks": True, "offset": 0} @@ -159,9 +164,9 @@ def list_jobs(self) -> List[dict]: timeout=API_TIMEOUT, ).json() - job_list.extend(response.get("jobs") or []) + yield from response.get("jobs") or [] - while response["has_more"]: + while response and response.get("has_more"): data["offset"] = len(response.get("jobs") or []) response = self.client.get( @@ -171,19 +176,16 @@ def list_jobs(self) -> List[dict]: timeout=API_TIMEOUT, ).json() - job_list.extend(response.get("jobs") or []) + yield from response.get("jobs") or [] except Exception as exc: logger.debug(traceback.format_exc()) logger.error(exc) - return job_list - def get_job_runs(self, job_id) -> List[dict]: """ Method returns List of all runs for a job by the specified job_id """ - job_runs = [] try: params = { "job_id": job_id, @@ -200,7 +202,7 @@ def get_job_runs(self, job_id) -> List[dict]: timeout=API_TIMEOUT, ).json() - job_runs.extend(response.get("runs") or []) + yield from response.get("runs") or [] while response["has_more"]: params.update({"start_time_to": response["runs"][-1]["start_time"]}) @@ -212,10 +214,8 @@ def get_job_runs(self, job_id) -> List[dict]: timeout=API_TIMEOUT, ).json() - job_runs.extend(response.get("runs" or [])) + yield from response.get("runs") or [] except Exception as exc: logger.debug(traceback.format_exc()) logger.error(exc) - - return job_runs diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py b/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py index 9c948213e42f..7795f6a262cc 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py @@ -35,7 +35,7 @@ def yield_table_query(self) -> Iterator[TableQuery]: start_date=self.start, end_date=self.end, ) - for row in data: + for row in data or []: try: if self.client.is_query_valid(row): yield TableQuery( diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/usage.py b/ingestion/src/metadata/ingestion/source/database/databricks/usage.py index 9199c79274b3..77f2cecd351e 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/usage.py @@ -39,7 +39,7 @@ def yield_table_queries(self) -> Iterable[TableQuery]: start_date=self.start, end_date=self.end, ) - for row in data: + for row in data or []: try: if self.client.is_query_valid(row): queries.append( diff --git a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py index a8910e877f8d..bda3964b32a3 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/metadata.py @@ -81,7 +81,7 @@ def create(cls, config_dict, metadata: OpenMetadata): return cls(config, metadata) def get_pipelines_list(self) -> Iterable[dict]: - for workflow in self.client.list_jobs(): + for workflow in self.client.list_jobs() or []: yield workflow def get_pipeline_name(self, pipeline_details: dict) -> str: @@ -195,7 +195,7 @@ def yield_pipeline_status(self, pipeline_details) -> Iterable[OMetaPipelineStatu for job_id in self.context.job_id_list: try: runs = self.client.get_job_runs(job_id=job_id) - for attempt in runs: + for attempt in runs or []: for task_run in attempt["tasks"]: task_status = [] task_status.append( From ffbb68200a91bbfdfcae68be89ca6978f7b01f3d Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 23 Jan 2024 07:02:39 +0100 Subject: [PATCH 04/49] MINOR - Fix SP topology context & Looker usage context (#14816) * MINOR - Fix SP topology context & Looker usage context * MINOR - Fix SP topology context & Looker usage context * Fix tests --- .../metadata/ingestion/api/topology_runner.py | 74 ++++++++-------- .../src/metadata/ingestion/models/topology.py | 4 + .../source/dashboard/looker/metadata.py | 30 +++++-- .../source/database/database_service.py | 1 + .../unit/topology/dashboard/test_looker.py | 84 ++++++++++--------- 5 files changed, 109 insertions(+), 84 deletions(-) diff --git a/ingestion/src/metadata/ingestion/api/topology_runner.py b/ingestion/src/metadata/ingestion/api/topology_runner.py index 70636b4294cf..e643becd991f 100644 --- a/ingestion/src/metadata/ingestion/api/topology_runner.py +++ b/ingestion/src/metadata/ingestion/api/topology_runner.py @@ -15,7 +15,7 @@ import traceback from collections import defaultdict from functools import singledispatchmethod -from typing import Any, Generic, Iterable, List, Type, TypeVar, Union +from typing import Any, Generic, Iterable, List, Type, TypeVar from pydantic import BaseModel @@ -267,20 +267,46 @@ def fqn_from_context(self, stage: NodeStage, entity_name: str) -> str: *context_names, entity_name ) - def update_context( - self, stage: NodeStage, context: Union[str, OMetaTagAndClassification] - ): + def update_context(self, stage: NodeStage, right: C): """ Append or update context - We'll store the entity_name in the topology context instead of the entity_fqn - and build the FQN on-the-fly wherever required. - This is mainly because we need the context in other places + We'll store the entity name or FQN in the topology context. + If we store the name, the FQN will be built in the source itself when needed. """ + + if stage.store_fqn: + new_context = self._build_new_context_fqn(right) + else: + new_context = model_str(right.name) + if stage.context and not stage.store_all_in_context: - self._replace_context(key=stage.context, value=context) + self._replace_context(key=stage.context, value=new_context) if stage.context and stage.store_all_in_context: - self._append_context(key=stage.context, value=context) + self._append_context(key=stage.context, value=new_context) + + @singledispatchmethod + def _build_new_context_fqn(self, right: C) -> str: + """Build context fqn string""" + raise NotImplementedError(f"Missing implementation for [{type(C)}]") + + @_build_new_context_fqn.register + def _(self, right: CreateStoredProcedureRequest) -> str: + """ + Implement FQN context building for Stored Procedures. + + We process the Stored Procedures lineage at the very end of the service. If we + just store the SP name, we lose the information of which db/schema the SP belongs to. + """ + + return fqn.build( + metadata=self.metadata, + entity_type=StoredProcedure, + service_name=self.context.database_service, + database_name=self.context.database, + schema_name=self.context.database_schema, + procedure_name=right.name.__root__, + ) def create_patch_request( self, original_entity: Entity, create_request: C @@ -379,7 +405,7 @@ def yield_and_update_context( "for the service connection." ) - self.update_context(stage=stage, context=entity_name) + self.update_context(stage=stage, right=right) @yield_and_update_context.register def _( @@ -395,7 +421,7 @@ def _( lineage has been properly drawn. We'll skip the process for now. """ yield entity_request - self.update_context(stage=stage, context=right.edge.fromEntity.name.__root__) + self.update_context(stage=stage, right=right.edge.fromEntity.name.__root__) @yield_and_update_context.register def _( @@ -408,7 +434,7 @@ def _( yield entity_request # We'll keep the tag fqn in the context and use if required - self.update_context(stage=stage, context=right) + self.update_context(stage=stage, right=right) @yield_and_update_context.register def _( @@ -421,29 +447,7 @@ def _( yield entity_request # We'll keep the tag fqn in the context and use if required - self.update_context(stage=stage, context=right) - - @yield_and_update_context.register - def _( - self, - right: CreateStoredProcedureRequest, - stage: NodeStage, - entity_request: Either[C], - ) -> Iterable[Either[Entity]]: - """Tag implementation for the context information""" - yield entity_request - - procedure_fqn = fqn.build( - metadata=self.metadata, - entity_type=StoredProcedure, - service_name=self.context.database_service, - database_name=self.context.database, - schema_name=self.context.database_schema, - procedure_name=right.name.__root__, - ) - - # We'll keep the proc fqn in the context and use if required - self.update_context(stage=stage, context=procedure_fqn) + self.update_context(stage=stage, right=right) def sink_request( self, stage: NodeStage, entity_request: Either[C] diff --git a/ingestion/src/metadata/ingestion/models/topology.py b/ingestion/src/metadata/ingestion/models/topology.py index 3dfec152a396..18aef03f3ae2 100644 --- a/ingestion/src/metadata/ingestion/models/topology.py +++ b/ingestion/src/metadata/ingestion/models/topology.py @@ -65,6 +65,10 @@ class Config: False, description="If we need to clean the values in the context for each produced element", ) + store_fqn: bool = Field( + False, + description="If true, store the entity FQN in the context instead of just the name", + ) # Used to compute the fingerprint cache_entities: bool = Field( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 1aa867fa290e..6bc74c12558c 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -47,9 +47,7 @@ ) from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.chart import Chart -from metadata.generated.schema.entity.data.dashboard import ( - Dashboard as MetadataDashboard, -) +from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.dashboardDataModel import ( DashboardDataModel, DataModelType, @@ -758,12 +756,12 @@ def yield_dashboard_lineage_details( if cached_explore: dashboard_fqn = fqn.build( self.metadata, - entity_type=MetadataDashboard, + entity_type=Dashboard, service_name=self.context.dashboard_service, dashboard_name=self.context.dashboard, ) dashboard_entity = self.metadata.get_by_name( - entity=MetadataDashboard, fqn=dashboard_fqn + entity=Dashboard, fqn=dashboard_fqn ) yield Either( right=AddLineageRequest( @@ -796,7 +794,7 @@ def build_lineage_request( self, source: str, db_service_name: str, - to_entity: Union[MetadataDashboard, DashboardDataModel], + to_entity: Union[Dashboard, DashboardDataModel], ) -> Optional[Either[AddLineageRequest]]: """ Once we have a list of origin data sources, check their components @@ -941,9 +939,23 @@ def yield_dashboard_usage( # pylint: disable=W0221 :return: UsageRequest, if not computed """ - dashboard: MetadataDashboard = self.context.dashboard + dashboard_name = self.context.dashboard try: + + dashboard_fqn = fqn.build( + metadata=self.metadata, + entity_type=Dashboard, + service_name=self.context.dashboard_service, + dashboard_name=dashboard_name, + ) + + dashboard: Dashboard = self.metadata.get_by_name( + entity=Dashboard, + fqn=dashboard_fqn, + fields=["usageSummary"], + ) + current_views = dashboard_details.view_count if not current_views: @@ -995,8 +1007,8 @@ def yield_dashboard_usage( # pylint: disable=W0221 except Exception as exc: yield Either( left=StackTraceError( - name=f"{dashboard.name} Usage", - error=f"Exception computing dashboard usage for {dashboard.fullyQualifiedName.__root__}: {exc}", + name=f"{dashboard_name} Usage", + error=f"Exception computing dashboard usage for {dashboard_name}: {exc}", stackTrace=traceback.format_exc(), ) ) diff --git a/ingestion/src/metadata/ingestion/source/database/database_service.py b/ingestion/src/metadata/ingestion/source/database/database_service.py index 9c57ff94c484..8d91ad2765b9 100644 --- a/ingestion/src/metadata/ingestion/source/database/database_service.py +++ b/ingestion/src/metadata/ingestion/source/database/database_service.py @@ -186,6 +186,7 @@ class DatabaseServiceTopology(ServiceTopology): processor="yield_stored_procedure", consumer=["database_service", "database", "database_schema"], store_all_in_context=True, + store_fqn=True, use_cache=True, ), ], diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index 70dab3451a41..cb4c2d3d50b2 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -397,26 +397,28 @@ def test_yield_dashboard_usage(self): Validate the logic for existing or new usage """ + self.looker.context.__dict__["dashboard"] = "dashboard_name" + MOCK_LOOKER_DASHBOARD.view_count = 10 + # Start checking dashboard without usage # and a view count - self.looker.context.__dict__["dashboard"] = Dashboard( + return_value = Dashboard( id=uuid.uuid4(), name="dashboard_name", fullyQualifiedName="dashboard_service.dashboard_name", service=EntityReference(id=uuid.uuid4(), type="dashboardService"), ) - MOCK_LOOKER_DASHBOARD.view_count = 10 - - self.assertEqual( - next(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD)).right, - DashboardUsage( - dashboard=self.looker.context.dashboard, - usage=UsageRequest(date=self.looker.today, count=10), - ), - ) + with patch.object(OpenMetadata, "get_by_name", return_value=return_value): + self.assertEqual( + next(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD)).right, + DashboardUsage( + dashboard=return_value, + usage=UsageRequest(date=self.looker.today, count=10), + ), + ) # Now check what happens if we already have some summary data for today - self.looker.context.__dict__["dashboard"] = Dashboard( + return_value = Dashboard( id=uuid.uuid4(), name="dashboard_name", fullyQualifiedName="dashboard_service.dashboard_name", @@ -425,14 +427,14 @@ def test_yield_dashboard_usage(self): dailyStats=UsageStats(count=10), date=self.looker.today ), ) - - # Nothing is returned - self.assertEqual( - len(list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))), 0 - ) + with patch.object(OpenMetadata, "get_by_name", return_value=return_value): + # Nothing is returned + self.assertEqual( + len(list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))), 0 + ) # But if we have usage for today but the count is 0, we'll return the details - self.looker.context.__dict__["dashboard"] = Dashboard( + return_value = Dashboard( id=uuid.uuid4(), name="dashboard_name", fullyQualifiedName="dashboard_service.dashboard_name", @@ -441,16 +443,17 @@ def test_yield_dashboard_usage(self): dailyStats=UsageStats(count=0), date=self.looker.today ), ) - self.assertEqual( - next(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD)).right, - DashboardUsage( - dashboard=self.looker.context.dashboard, - usage=UsageRequest(date=self.looker.today, count=10), - ), - ) + with patch.object(OpenMetadata, "get_by_name", return_value=return_value): + self.assertEqual( + next(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD)).right, + DashboardUsage( + dashboard=return_value, + usage=UsageRequest(date=self.looker.today, count=10), + ), + ) # But if we have usage for another day, then we do the difference - self.looker.context.__dict__["dashboard"] = Dashboard( + return_value = Dashboard( id=uuid.uuid4(), name="dashboard_name", fullyQualifiedName="dashboard_service.dashboard_name", @@ -460,17 +463,18 @@ def test_yield_dashboard_usage(self): date=datetime.strftime(datetime.now() - timedelta(1), "%Y-%m-%d"), ), ) - self.assertEqual( - next(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD)).right, - DashboardUsage( - dashboard=self.looker.context.dashboard, - usage=UsageRequest(date=self.looker.today, count=5), - ), - ) + with patch.object(OpenMetadata, "get_by_name", return_value=return_value): + self.assertEqual( + next(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD)).right, + DashboardUsage( + dashboard=return_value, + usage=UsageRequest(date=self.looker.today, count=5), + ), + ) # If the past usage is higher than what we have today, something weird is going on # we don't return usage but don't explode - self.looker.context.__dict__["dashboard"] = Dashboard( + return_value = Dashboard( id=uuid.uuid4(), name="dashboard_name", fullyQualifiedName="dashboard_service.dashboard_name", @@ -480,11 +484,11 @@ def test_yield_dashboard_usage(self): date=datetime.strftime(datetime.now() - timedelta(1), "%Y-%m-%d"), ), ) + with patch.object(OpenMetadata, "get_by_name", return_value=return_value): + self.assertEqual( + len(list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))), 1 + ) - self.assertEqual( - len(list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))), 1 - ) - - self.assertIsNotNone( - list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))[0].left - ) + self.assertIsNotNone( + list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))[0].left + ) From 00243ead50b05a90e45fc3feabb3275c3731fba5 Mon Sep 17 00:00:00 2001 From: Ayush Shah Date: Tue, 23 Jan 2024 11:58:23 +0530 Subject: [PATCH 05/49] Fixes #14598: Fix Tags / Labels ingestion on includeTags as False (#14782) --- .../source/database/bigquery/connection.py | 4 +-- .../source/database/bigquery/metadata.py | 28 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py index 2f5b1b1523a2..4c353cc5400d 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py @@ -113,8 +113,8 @@ def test_tags(): taxonomy_project_ids = [] if engine.url.host: taxonomy_project_ids.append(engine.url.host) - if service_connection.taxonomyProjectId: - taxonomy_project_ids.extend(service_connection.taxonomyProjectId) + if service_connection.taxonomyProjectID: + taxonomy_project_ids.extend(service_connection.taxonomyProjectID) if not taxonomy_project_ids: logger.info("'taxonomyProjectID' is not set, so skipping this test.") return None diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 8b7f803425b6..d55878b06530 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -89,11 +89,8 @@ from metadata.utils.helpers import get_start_and_end from metadata.utils.logger import ingestion_logger from metadata.utils.sqlalchemy_utils import is_complex_type -from metadata.utils.tag_utils import ( - get_ometa_tag_and_classification, - get_tag_label, - get_tag_labels, -) +from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_label +from metadata.utils.tag_utils import get_tag_labels as fetch_tag_labels_om _bigquery_table_types = { "BASE TABLE": TableType.Regular, @@ -296,6 +293,7 @@ def yield_tag( classification_name=key, tag_description="Bigquery Dataset Label", classification_description="", + include_tags=self.source_config.includeTags, ) # Fetching policy tags on the column level list_project_ids = [self.context.database] @@ -315,6 +313,7 @@ def yield_tag( classification_name=taxonomy.display_name, tag_description="Bigquery Policy Tag", classification_description="", + include_tags=self.source_config.includeTags, ) except Exception as exc: yield Either( @@ -370,16 +369,16 @@ def yield_database_schema( ) dataset_obj = self.client.get_dataset(schema_name) - if dataset_obj.labels: + if dataset_obj.labels and self.source_config.includeTags: database_schema_request_obj.tags = [] for label_classification, label_tag_name in dataset_obj.labels.items(): - database_schema_request_obj.tags.append( - get_tag_label( - metadata=self.metadata, - tag_name=label_tag_name, - classification_name=label_classification, - ) + tag_label = get_tag_label( + metadata=self.metadata, + tag_name=label_tag_name, + classification_name=label_classification, ) + if tag_label: + database_schema_request_obj.tags.append(tag_label) yield Either(right=database_schema_request_obj) def get_table_obj(self, table_name: str): @@ -388,7 +387,7 @@ def get_table_obj(self, table_name: str): bq_table_fqn = fqn._build(database, schema_name, table_name) return self.client.get_table(bq_table_fqn) - def yield_table_tag_details(self, table_name_and_type: Tuple[str, str]): + def yield_table_tags(self, table_name_and_type: Tuple[str, str]): table_name, _ = table_name_and_type table_obj = self.get_table_obj(table_name=table_name) if table_obj.labels: @@ -398,6 +397,7 @@ def yield_table_tag_details(self, table_name_and_type: Tuple[str, str]): classification_name=key, tag_description="Bigquery Table Label", classification_description="", + include_tags=self.source_config.includeTags, ) def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]: @@ -426,7 +426,7 @@ def get_column_tag_labels( is properly informed """ if column.get("policy_tags"): - return get_tag_labels( + return fetch_tag_labels_om( metadata=self.metadata, tags=[column["policy_tags"]], classification_name=column["taxonomy"], From 9bd97c02da1c68d58a0a4ddc7a6c25b1d93e279c Mon Sep 17 00:00:00 2001 From: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Date: Tue, 23 Jan 2024 12:38:35 +0530 Subject: [PATCH 06/49] fix(ui): password error message for char limits (#14808) * fix(ui): password error message for char limits * fix java side code --- .../main/java/org/openmetadata/service/util/PasswordUtil.java | 2 +- .../main/resources/json/schema/auth/passwordResetRequest.json | 4 ++-- .../main/resources/json/schema/auth/registrationRequest.json | 2 +- .../src/main/resources/ui/src/locale/languages/de-de.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/en-us.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/es-es.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/fr-fr.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/he-he.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/ja-jp.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/nl-nl.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/pt-br.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/ru-ru.json | 4 ++-- .../src/main/resources/ui/src/locale/languages/zh-cn.json | 4 ++-- 13 files changed, 24 insertions(+), 24 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/PasswordUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/PasswordUtil.java index 8f229efd4060..1143999077c9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/PasswordUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/PasswordUtil.java @@ -34,7 +34,7 @@ public class PasswordUtil { static { List rules = new ArrayList<>(); - // 8 and 16 characters + // 8 and 56 characters rules.add(new LengthRule(8, 56)); // No whitespace allowed rules.add(new WhitespaceRule()); diff --git a/openmetadata-spec/src/main/resources/json/schema/auth/passwordResetRequest.json b/openmetadata-spec/src/main/resources/json/schema/auth/passwordResetRequest.json index 8594d17610ca..1e8a7a0f0314 100644 --- a/openmetadata-spec/src/main/resources/json/schema/auth/passwordResetRequest.json +++ b/openmetadata-spec/src/main/resources/json/schema/auth/passwordResetRequest.json @@ -14,13 +14,13 @@ "description": "Password", "type": "string", "minLength": 8, - "maxLength": 16 + "maxLength": 56 }, "confirmPassword": { "description": "Confirm Password", "type": "string", "minLength": 8, - "maxLength": 16 + "maxLength": 56 }, "token": { "description": "Token", diff --git a/openmetadata-spec/src/main/resources/json/schema/auth/registrationRequest.json b/openmetadata-spec/src/main/resources/json/schema/auth/registrationRequest.json index b0efbb074f89..a64d33e44518 100644 --- a/openmetadata-spec/src/main/resources/json/schema/auth/registrationRequest.json +++ b/openmetadata-spec/src/main/resources/json/schema/auth/registrationRequest.json @@ -22,7 +22,7 @@ "description": "Login Password", "type": "string", "minLength": 8, - "maxLength": 16 + "maxLength": 56 } }, "required": [ diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 4a3e48786e7a..088e7dd9255a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Überwache und verstehe die Struktur deiner Tabellen mit dem Profiler.", "page-sub-header-for-teams": "Stelle deine gesamte Organisationsstruktur mit hierarchischen Teams dar.", "page-sub-header-for-users": "Stelle deine gesamte Organisationsstruktur mit hierarchischen Teams dar.", - "password-error-message": "Das Passwort muss mindestens 8 und maximal 16 Zeichen lang sein und mindestens einen Großbuchstaben (A-Z), einen Kleinbuchstaben (a-z), eine Zahl und ein Sonderzeichen (z. B. !, %, @ oder #) enthalten.", - "password-pattern-error": "Das Passwort muss mindestens 8 und maximal 16 Zeichen lang sein und mindestens einen Sonderbuchstaben, einen Großbuchstaben und einen Kleinbuchstaben enthalten.", + "password-error-message": "Das Passwort muss mindestens 8 und maximal 56 Zeichen lang sein und mindestens einen Großbuchstaben (A-Z), einen Kleinbuchstaben (a-z), eine Zahl und ein Sonderzeichen (z. B. !, %, @ oder #) enthalten.", + "password-pattern-error": "Das Passwort muss mindestens 8 und maximal 56 Zeichen lang sein und mindestens einen Sonderbuchstaben, einen Großbuchstaben und einen Kleinbuchstaben enthalten.", "path-of-the-dbt-files-stored": "Pfad zum Ordner, in dem die dbt-Dateien gespeichert sind", "permanently-delete-metadata": "Das dauerhafte Löschen dieses <0>{{entityName}} entfernt seine Metadaten dauerhaft aus OpenMetadata.", "permanently-delete-metadata-and-dependents": "Das dauerhafte Löschen dieses {{entityName}} entfernt seine Metadaten sowie die Metadaten von {{dependents}} dauerhaft aus OpenMetadata.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index d3e9fd1e7cc7..8105af1ff5d4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Monitor and understand your tables structure with the profiler.", "page-sub-header-for-teams": "Represent your entire organizational structure with hierarchical teams.", "page-sub-header-for-users": "Represent your entire organizational structure with hierarchical teams.", - "password-error-message": "Password must be a minimum of 8 and a maximum of 16 characters long and contain at least one uppercase character (A-Z), one lowercase character (a-z), one number, and one special character (such as !, %, @, or #)", - "password-pattern-error": "Password must be of minimum 8 and maximum 16 characters, with one special , one upper, one lower case character", + "password-error-message": "Password must be a minimum of 8 and a maximum of 56 characters long and contain at least one uppercase character (A-Z), one lowercase character (a-z), one number, and one special character (such as !, %, @, or #)", + "password-pattern-error": "Password must be of minimum 8 and maximum 56 characters, with one special , one upper, one lower case character", "path-of-the-dbt-files-stored": "Path of the folder where the dbt files are stored", "permanently-delete-metadata": "Permanently deleting this <0>{{entityName}} will remove its metadata from OpenMetadata permanently.", "permanently-delete-metadata-and-dependents": "Permanently deleting this {{entityName}} will remove its metadata, as well as the metadata of {{dependents}} from OpenMetadata permanently.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index e37eb078adb1..c4012d85c86d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Monitor and understand your tables structure with the profiler.", "page-sub-header-for-teams": "Represent your entire organizational structure with hierarchical teams.", "page-sub-header-for-users": "Represent your entire organizational structure with hierarchical teams.", - "password-error-message": "Password must be a minimum of 8 and a maximum of 16 characters long and contain at least one uppercase character (A-Z), one lowercase character (a-z), one number, and one special character (such as !, %, @, or #)", - "password-pattern-error": "La contraseña debe tener como mínimo 8 y como máximo 16 caracteres, con un caracter especial, una letra mayúscula, y una letra minúscula.", + "password-error-message": "Password must be a minimum of 8 and a maximum of 56 characters long and contain at least one uppercase character (A-Z), one lowercase character (a-z), one number, and one special character (such as !, %, @, or #)", + "password-pattern-error": "La contraseña debe tener como mínimo 8 y como máximo 56 caracteres, con un caracter especial, una letra mayúscula, y una letra minúscula.", "path-of-the-dbt-files-stored": "Ruta de la carpeta donde se almacenan los archivos dbt", "permanently-delete-metadata": "Al eliminar permanentemente este <0>{{entityName}}, se eliminaran sus metadatos de OpenMetadata permanentemente.", "permanently-delete-metadata-and-dependents": "Al eliminar permanentemente este {{entityName}}, se eliminaran sus metadatos, así como los metadatos de {{dependents}} de OpenMetadata permanentemente.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index 0605a9722963..654d18892fa6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Surveillez et comprenez la structure de vos tables avec le profilage.", "page-sub-header-for-teams": "Représentez toute la structure organisationnelle avec des équipes hiérarchiques.", "page-sub-header-for-users": "Représentez toute la structure organisationnelle avec des équipes hiérarchiques.", - "password-error-message": "Le mot de passe doit comporter au moins 8 caractères et au plus 16 caractères et doit contenir au moins une lettre majuscule (A-Z), une lettre minuscule (a-z), un chiffre et un caractère spécial (tel que !, %, @ ou #).", - "password-pattern-error": "Le mot de passe doit comporter au moins 8 caractères et au plus 16 caractères, avec un caractère spécial, une lettre majuscule et une lettre minuscule.", + "password-error-message": "Le mot de passe doit comporter au moins 8 caractères et au plus 56 caractères et doit contenir au moins une lettre majuscule (A-Z), une lettre minuscule (a-z), un chiffre et un caractère spécial (tel que !, %, @ ou #).", + "password-pattern-error": "Le mot de passe doit comporter au moins 8 caractères et au plus 56 caractères, avec un caractère spécial, une lettre majuscule et une lettre minuscule.", "path-of-the-dbt-files-stored": "Chemin du dossier où sont situés les fichiers dbt.", "permanently-delete-metadata": "La suppression permanente de cette <0>{{entityName}} supprimera ses métadonnées de façon permanente d'OpenMetadata.", "permanently-delete-metadata-and-dependents": "La suppression permanente de cette {{entityName}} supprimera ses métadonnées ainsi que les métadonnées de {{dependents}} de façon permanente d'OpenMetadata.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json index 32dcf1d881f0..be695f9dea47 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "נטרו, נתחו והבינו את מבנה הטבלאות שלכם באמצועות הפרופיילר.", "page-sub-header-for-teams": "ייצג את מבנה הארגון באמצעות היררכיה של צוותים", "page-sub-header-for-users": "ייצג את מבנה הארגון באמצעות היררכיה של צוותים", - "password-error-message": "יש להזין סיסמה באורך של 8 עד 16 תווים ולכלול לפחות אות גדולה אחת (A-Z), אות קטנה אחת (a-z), מספר אחד ותו מיוחד (כמו !, %, @, או #)", - "password-pattern-error": "יש להזין סיסמה באורך של 8 עד 16 תווים, עם תו מיוחד אחד, אות גדולה אחת ואות קטנה אחת", + "password-error-message": "יש להזין סיסמה באורך של 8 עד 15 תווים ולכלול לפחות אות גדולה אחת (A-Z), אות קטנה אחת (a-z), מספר אחד ותו מיוחד (כמו !, %, @, או #)", + "password-pattern-error": "יש להזין סיסמה באורך של 8 עד 56 תווים, עם תו מיוחד אחד, אות גדולה אחת ואות קטנה אחת", "path-of-the-dbt-files-stored": "נתיב לתיקייה בה נשמרים קבצי ה-dbt", "permanently-delete-metadata": "מחיקה של {{entityName}} תסיר את המטה-דאטה שלו מ-OpenMetadata לצמיתות.", "permanently-delete-metadata-and-dependents": "מחיקה של {{entityName}} תסיר את המטה-דאטה שלו, כמו גם את המטה-דאטה של {{dependents}} מ-OpenMetadata לצמיתות.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index c5538a03a5ca..47099c0f9a50 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Monitor and understand your tables structure with the profiler.", "page-sub-header-for-teams": "Represent your entire organizational structure with hierarchical teams.", "page-sub-header-for-users": "Represent your entire organizational structure with hierarchical teams.", - "password-error-message": "Password must be a minimum of 8 and a maximum of 16 characters long and contain at least one uppercase character (A-Z), one lowercase character (a-z), one number, and one special character (such as !, %, @, or #)", - "password-pattern-error": "パスワードは8～16の文字列で、1つの特殊文字、大文字1つ、小文字1つが含まれる必要があります。", + "password-error-message": "Password must be a minimum of 8 and a maximum of 66 characters long and contain at least one uppercase character (A-Z), one lowercase character (a-z), one number, and one special character (such as !, %, @, or #)", + "password-pattern-error": "パスワードは8～56の文字列で、1つの特殊文字、大文字1つ、小文字1つが含まれる必要があります。", "path-of-the-dbt-files-stored": "Path of the folder where the dbt files are stored", "permanently-delete-metadata": "Permanently deleting this <0>{{entityName}} will remove its metadata from OpenMetadata permanently.", "permanently-delete-metadata-and-dependents": "Permanently deleting this {{entityName}} will remove its metadata, as well as the metadata of {{dependents}} from OpenMetadata permanently.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json index 3029189151f2..bd7ddc5122ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Volg en begrijp de structuur van je tabellen met de profiler.", "page-sub-header-for-teams": "Vertegenwoordig je hele organisatiestructuur met hiërarchische teams.", "page-sub-header-for-users": "Vertegenwoordig je hele organisatiestructuur met hiërarchische teams.", - "password-error-message": "Het wachtwoord moet minimaal 8 en maximaal 16 tekens lang zijn en ten minste één hoofdletter (A-Z), één kleine letter (a-z), één cijfer en één speciaal teken (zoals !, %, @ of #) bevatten", - "password-pattern-error": "Wachtwoord moet minimaal 8 en maximaal 16 tekens lang zijn, met één speciaal teken, één hoofdletter, één kleine letter", + "password-error-message": "Het wachtwoord moet minimaal 8 en maximaal 56 tekens lang zijn en ten minste één hoofdletter (A-Z), één kleine letter (a-z), één cijfer en één speciaal teken (zoals !, %, @ of #) bevatten", + "password-pattern-error": "Wachtwoord moet minimaal 8 en maximaal 56 tekens lang zijn, met één speciaal teken, één hoofdletter, één kleine letter", "path-of-the-dbt-files-stored": "Pad naar de map waarin de dbt-bestanden zijn opgeslagen", "permanently-delete-metadata": "Het permanent verwijderen van deze <0>{{entityName}} verwijdert de metadata ervan permanent uit OpenMetadata.", "permanently-delete-metadata-and-dependents": "Het permanent verwijderen van deze {{entityName}} verwijdert de metadata ervan, evenals de metadata van {{dependents}} permanent uit OpenMetadata.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index 06f40864483e..77323d795b6b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Monitore e compreenda a estrutura de suas tabelas com o examinador.", "page-sub-header-for-teams": "Represente toda a estrutura organizacional com equipes hierárquicas.", "page-sub-header-for-users": "Represente toda a estrutura organizacional com equipes hierárquicas.", - "password-error-message": "A senha deve ter no mínimo 8 e no máximo 16 caracteres e conter pelo menos uma letra maiúscula (A-Z), uma letra minúscula (a-z), um número e um caractere especial (como !, %, @ ou #)", - "password-pattern-error": "A senha deve ter no mínimo 8 e no máximo 16 caracteres, com pelo menos um caractere especial, uma letra maiúscula e uma letra minúscula", + "password-error-message": "A senha deve ter no mínimo 8 e no máximo 56 caracteres e conter pelo menos uma letra maiúscula (A-Z), uma letra minúscula (a-z), um número e um caractere especial (como !, %, @ ou #)", + "password-pattern-error": "A senha deve ter no mínimo 8 e no máximo 56 caracteres, com pelo menos um caractere especial, uma letra maiúscula e uma letra minúscula", "path-of-the-dbt-files-stored": "Caminho da pasta onde os arquivos dbt são armazenados", "permanently-delete-metadata": "Excluir permanentemente este(a) <0>{{entityName}} removerá seus metadados do OpenMetadata permanentemente.", "permanently-delete-metadata-and-dependents": "Excluir permanentemente este(a) {{entityName}} removerá seus metadados, bem como os metadados de {{dependents}} do OpenMetadata permanentemente.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index b9aacc54a5e6..6b81e846f271 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "Отслеживайте и анализируйте структуру ваших таблиц с помощью профилировщика.", "page-sub-header-for-teams": "Представьте всю свою организационную структуру с помощью иерархических команд.", "page-sub-header-for-users": "Представьте всю свою организационную структуру с помощью иерархических команд.", - "password-error-message": "Пароль должен содержать не менее 8 и не более 16 символов и содержать как минимум один символ верхнего регистра (A-Z), один символ нижнего регистра (az), одну цифру и один специальный символ (например, !, %, @ или #). )", - "password-pattern-error": "Пароль должен состоять минимум из 8 и максимум из 16 символов, включая один специальный, один верхний и один нижний регистр.", + "password-error-message": "Пароль должен содержать не менее 8 и не более 56 символов и содержать как минимум один символ верхнего регистра (A-Z), один символ нижнего регистра (az), одну цифру и один специальный символ (например, !, %, @ или #). )", + "password-pattern-error": "Пароль должен состоять минимум из 8 и максимум из 56 символов, включая один специальный, один верхний и один нижний регистр.", "path-of-the-dbt-files-stored": "Путь к папке, в которой хранятся файлы dbt", "permanently-delete-metadata": "При окончательном удалении этого объекта <0>{{entityName}} его метаданные будут навсегда удалены из OpenMetadata.", "permanently-delete-metadata-and-dependents": "Безвозвратное удаление этого {{entityName}} удалит его метаданные, а также метаданные {{dependers}} из OpenMetadata навсегда.", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index aa25d3eaf46a..61bdb7fd2696 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -1543,8 +1543,8 @@ "page-sub-header-for-table-profile": "通过数据分析工具了解和跟踪您的数据表结构", "page-sub-header-for-teams": "将组织机构的架构通过团队进行分层分级", "page-sub-header-for-users": "将组织机构的架构通过团队进行分层分级", - "password-error-message": "密码必须为8到16个字符，至少包括一个大写字母（A-Z）、一个小写字母（a-z），和一个特殊字符（例如：!, %, @, or #）", - "password-pattern-error": "密码必须为8到16个字符，至少包括一个特殊字符、一个大写字母、一个小写字母", + "password-error-message": "密码必须为8到56个字符，至少包括一个大写字母（A-Z）、一个小写字母（a-z），和一个特殊字符（例如：!, %, @, or #）", + "password-pattern-error": "密码必须为8到56个字符，至少包括一个特殊字符、一个大写字母、一个小写字母", "path-of-the-dbt-files-stored": "存储 dbt 文件的文件夹路径", "permanently-delete-metadata": "永久删除此<0>{{entityName}}将永久从 OpenMetadata 中删除其元数据", "permanently-delete-metadata-and-dependents": "永久删除此{{entityName}}将永久从 OpenMetadata 中删除其元数据以及{{dependents}}的元数据", From 993dc56ffb5bef6cd4225c198ecd06f282a0f911 Mon Sep 17 00:00:00 2001 From: kwgdaig <18678754+kwgdaig@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:36:36 +0900 Subject: [PATCH 07/49] Fixes #13556: Support for Salesforce table description ingestion (#14733) * ISSUE-13556: Add suport for Salesforce table description ingestion * ISSUE-13556: Remove unnecessary blank line * ISSUE-13556: Fix to get description for each table --------- Co-authored-by: Teddy --- .../source/database/salesforce/metadata.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py index 7a44d3af6291..ca8de3860cb3 100644 --- a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py @@ -151,6 +151,7 @@ def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]: :return: tables or views, depending on config """ schema_name = self.context.database_schema + try: if self.service_connection.sobjectName: table_name = self.standardize_table_name( @@ -191,6 +192,30 @@ def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]: ) ) + def get_table_description(self, table_name: str) -> Optional[str]: + """ + Method to get the table description for salesforce with Tooling API + """ + try: + result = self.client.toolingexecute( + f"query/?q=SELECT+Description+FROM+EntityDefinition+WHERE+QualifiedApiName='{table_name}'" + ) + return result["records"][0]["Description"] + except KeyError as err: + logger.warning( + f"Unable to get required key from Tooling API response for table [{table_name}]: {err}" + ) + except IndexError as err: + logger.warning( + f"Unable to get row for table [{table_name}] from EntityDefinition: {err}" + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Unable to get description with Tooling API for table [{table_name}]: {exc}" + ) + return None + def yield_table( self, table_name_and_type: Tuple[str, str] ) -> Iterable[Either[CreateTableRequest]]: @@ -209,6 +234,7 @@ def yield_table( table_request = CreateTableRequest( name=table_name, tableType=table_type, + description=self.get_table_description(table_name), columns=columns, tableConstraints=table_constraints, databaseSchema=fqn.build( From d8555fc4c36784badc83251de146a905891684e2 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 23 Jan 2024 09:24:19 +0100 Subject: [PATCH 08/49] MINOR - Better handling of Ingestion Pipeline Status (#14792) * MINOR - Better handling of Ingestion Pipeline Status * format * format --- .../src/metadata/workflow/application.py | 2 +- .../workflow/workflow_status_mixin.py | 49 +++++++++++-------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 44aa6898181f..09ebd2461a5d 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -145,7 +145,7 @@ def raise_from_status_internal(self, raise_warnings=False): and self.calculate_success() < SUCCESS_THRESHOLD_VALUE ): raise WorkflowExecutionError( - f"{self.source.name} reported errors: {Summary.from_step(self.source)}" + f"{self.runner.name} reported errors: {Summary.from_step(self.runner)}" ) if raise_warnings and self.runner.get_status().warnings: diff --git a/ingestion/src/metadata/workflow/workflow_status_mixin.py b/ingestion/src/metadata/workflow/workflow_status_mixin.py index 1d0fa16a7911..c6d7e278e52d 100644 --- a/ingestion/src/metadata/workflow/workflow_status_mixin.py +++ b/ingestion/src/metadata/workflow/workflow_status_mixin.py @@ -11,6 +11,7 @@ """ Add methods to the workflows for updating the IngestionPipeline status """ +import traceback import uuid from datetime import datetime from typing import Optional, Tuple @@ -29,8 +30,10 @@ OpenMetadataWorkflowConfig, ) from metadata.ingestion.api.step import Step, Summary -from metadata.ingestion.api.steps import Source from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.logger import ometa_logger + +logger = ometa_logger() SUCCESS_THRESHOLD_VALUE = 90 @@ -49,8 +52,6 @@ class WorkflowStatusMixin: _start_ts: int ingestion_pipeline: Optional[IngestionPipeline] - # All workflows require a source as a first step - source: Source # All workflows execute a series of steps, aside from the source steps: Tuple[Step] @@ -86,24 +87,30 @@ def set_ingestion_pipeline_status( Method to set the pipeline status of current ingestion pipeline """ - # if we don't have a related Ingestion Pipeline FQN, no status is set. - if self.config.ingestionPipelineFQN and self.ingestion_pipeline: - pipeline_status = self.metadata.get_pipeline_status( - self.config.ingestionPipelineFQN, self.run_id - ) - if not pipeline_status: - # We need to crete the status - pipeline_status = self._new_pipeline_status(state) - else: - # if workflow is ended then update the end date in status - pipeline_status.endDate = datetime.now().timestamp() * 1000 - pipeline_status.pipelineState = state - - pipeline_status.status = ( - ingestion_status if ingestion_status else pipeline_status.status - ) - self.metadata.create_or_update_pipeline_status( - self.config.ingestionPipelineFQN, pipeline_status + try: + # if we don't have a related Ingestion Pipeline FQN, no status is set. + if self.config.ingestionPipelineFQN and self.ingestion_pipeline: + pipeline_status = self.metadata.get_pipeline_status( + self.ingestion_pipeline.fullyQualifiedName.__root__, self.run_id + ) + if not pipeline_status: + # We need to crete the status + pipeline_status = self._new_pipeline_status(state) + else: + # if workflow is ended then update the end date in status + pipeline_status.endDate = datetime.now().timestamp() * 1000 + pipeline_status.pipelineState = state + + pipeline_status.status = ( + ingestion_status if ingestion_status else pipeline_status.status + ) + self.metadata.create_or_update_pipeline_status( + self.ingestion_pipeline.fullyQualifiedName.__root__, pipeline_status + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.error( + f"Unhandled error trying to update Ingestion Pipeline status [{err}]" ) def raise_from_status(self, raise_warnings=False): From cc00da66fd05bfe5ed9c3c493148f71e7a8e3976 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Tue, 23 Jan 2024 14:25:24 +0530 Subject: [PATCH 09/49] MINOR: Added table validation for cost analysis data (#14793) * Added validation for cost analysis source * centralized life cycle logic --- .../source/database/bigquery/metadata.py | 34 +----------------- .../source/database/life_cycle_query_mixin.py | 36 ++++++++++++++++++- .../source/database/redshift/metadata.py | 34 +----------------- .../source/database/snowflake/metadata.py | 34 +----------------- 4 files changed, 38 insertions(+), 100 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index d55878b06530..a81a87e48d54 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -34,7 +34,6 @@ from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode from metadata.generated.schema.entity.data.table import ( IntervalType, - Table, TablePartition, TableType, ) @@ -54,7 +53,6 @@ from metadata.generated.schema.type.tagLabel import TagLabel from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException -from metadata.ingestion.models.life_cycle import OMetaLifeCycleData from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.connections import get_test_connection_fn @@ -220,6 +218,7 @@ def __init__(self, config, metadata): # list of all project IDs. Subsequently, after the invokation, # we proceed to test the connections for each of these project IDs self.project_ids = self.set_project_id() + self.life_cycle_query = BIGQUERY_LIFE_CYCLE_QUERY self.test_connection = self._test_connection self.test_connection() @@ -542,37 +541,6 @@ def close(self): if os.path.exists(temp_file_path): os.remove(temp_file_path) - def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]: - """ - Get the life cycle data of the table - """ - try: - table_fqn = fqn.build( - self.metadata, - entity_type=Table, - service_name=self.context.database_service, - database_name=self.context.database, - schema_name=self.context.database_schema, - table_name=self.context.table, - skip_es_search=True, - ) - table = self.metadata.get_by_name(entity=Table, fqn=table_fqn) - yield from self.get_life_cycle_data( - entity=table, - query=BIGQUERY_LIFE_CYCLE_QUERY.format( - database_name=table.database.name, - schema_name=table.databaseSchema.name, - ), - ) - except Exception as exc: - yield Either( - left=StackTraceError( - name="lifeCycle", - error=f"Error Processing life cycle data: {exc}", - stackTrace=traceback.format_exc(), - ) - ) - def _get_source_url( self, database_name: Optional[str] = None, diff --git a/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py b/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py index 91ee3cea1121..1361939f55f7 100644 --- a/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py +++ b/ingestion/src/metadata/ingestion/source/database/life_cycle_query_mixin.py @@ -15,11 +15,12 @@ from collections import defaultdict from datetime import datetime from functools import lru_cache -from typing import Dict, List, Optional +from typing import Dict, Iterable, List, Optional from pydantic import BaseModel, Field from sqlalchemy.engine import Engine +from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -32,6 +33,7 @@ from metadata.ingestion.models.life_cycle import OMetaLifeCycleData from metadata.ingestion.models.topology import TopologyContext from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils import fqn from metadata.utils.logger import ingestion_logger from metadata.utils.time_utils import convert_timestamp_to_milliseconds @@ -118,3 +120,35 @@ def get_life_cycle_data(self, entity: Entity, query: str): stackTrace=traceback.format_exc(), ) ) + + def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]: + """ + Get the life cycle data of the table + """ + try: + table_fqn = fqn.build( + self.metadata, + entity_type=Table, + service_name=self.context.database_service, + database_name=self.context.database, + schema_name=self.context.database_schema, + table_name=self.context.table, + skip_es_search=True, + ) + table = self.metadata.get_by_name(entity=Table, fqn=table_fqn) + if table: + yield from self.get_life_cycle_data( + entity=table, + query=self.life_cycle_query.format( + database_name=table.database.name, + schema_name=table.databaseSchema.name, + ), + ) + except Exception as exc: + yield Either( + left=StackTraceError( + name="lifeCycle", + error=f"Error Processing life cycle data: {exc}", + stackTrace=traceback.format_exc(), + ) + ) diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py index 46e726258f88..e8b158486d77 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py @@ -33,7 +33,6 @@ from metadata.generated.schema.entity.data.table import ( ConstraintType, IntervalType, - Table, TableConstraint, TablePartition, TableType, @@ -50,7 +49,6 @@ from metadata.generated.schema.type.basic import EntityName from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException -from metadata.ingestion.models.life_cycle import OMetaLifeCycleData from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.common_db_source import ( CommonDbSourceService, @@ -123,6 +121,7 @@ class RedshiftSource( def __init__(self, config, metadata): super().__init__(config, metadata) self.partition_details = {} + self.life_cycle_query = REDSHIFT_LIFE_CYCLE_QUERY @classmethod def create(cls, config_dict, metadata: OpenMetadata): @@ -313,34 +312,3 @@ def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]] ) return queries_dict - - def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]: - """ - Get the life cycle data of the table - """ - try: - table_fqn = fqn.build( - self.metadata, - entity_type=Table, - service_name=self.context.database_service, - database_name=self.context.database, - schema_name=self.context.database_schema, - table_name=self.context.table, - skip_es_search=True, - ) - table = self.metadata.get_by_name(entity=Table, fqn=table_fqn) - yield from self.get_life_cycle_data( - entity=table, - query=REDSHIFT_LIFE_CYCLE_QUERY.format( - database_name=table.database.name, - schema_name=table.databaseSchema.name, - ), - ) - except Exception as exc: - yield Either( - left=StackTraceError( - name="lifeCycle", - error=f"Error Processing life cycle data: {exc}", - stackTrace=traceback.format_exc(), - ) - ) diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py index db31303a221f..7ea4a6132d84 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py @@ -29,7 +29,6 @@ from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode from metadata.generated.schema.entity.data.table import ( IntervalType, - Table, TablePartition, TableType, ) @@ -45,7 +44,6 @@ from metadata.generated.schema.type.basic import EntityName, SourceUrl from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException -from metadata.ingestion.models.life_cycle import OMetaLifeCycleData from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type @@ -145,6 +143,7 @@ def __init__(self, config, metadata): self._account: Optional[str] = None self._org_name: Optional[str] = None + self.life_cycle_query = SNOWFLAKE_LIFE_CYCLE_QUERY @classmethod def create(cls, config_dict, metadata: OpenMetadata): @@ -476,37 +475,6 @@ def get_source_url( logger.error(f"Unable to get source url: {exc}") return None - def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]: - """ - Get the life cycle data of the table - """ - try: - table_fqn = fqn.build( - self.metadata, - entity_type=Table, - service_name=self.context.database_service, - database_name=self.context.database, - schema_name=self.context.database_schema, - table_name=self.context.table, - skip_es_search=True, - ) - table = self.metadata.get_by_name(entity=Table, fqn=table_fqn) - yield from self.get_life_cycle_data( - entity=table, - query=SNOWFLAKE_LIFE_CYCLE_QUERY.format( - database_name=table.database.name, - schema_name=table.databaseSchema.name, - ), - ) - except Exception as exc: - yield Either( - left=StackTraceError( - name="lifeCycle", - error=f"Error Processing life cycle data: {exc}", - stackTrace=traceback.format_exc(), - ) - ) - def query_view_names_and_types( self, schema_name: str ) -> Iterable[TableNameAndType]: From 97440f78d4b785081939fe923e6932a443cfbf12 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Tue, 23 Jan 2024 15:30:05 +0530 Subject: [PATCH 10/49] CYPRESS: simplify side navigation click in cypress (#14818) * simplify side navigation click in cypress * make sidbar item uses common enum * fix cypress failure of outside import --- .../common/CustomizeLandingPageUtils.js | 3 +- .../ui/cypress/common/DomainUtils.js | 5 +-- .../common/Entities/ServiceBaseClass.ts | 3 +- .../ui/cypress/common/Entities/UserClass.ts | 9 ++--- .../ui/cypress/common/GlossaryUtils.js | 11 ++----- .../resources/ui/cypress/common/TagUtils.js | 9 ++--- .../ui/cypress/common/Utils/CustomProperty.ts | 6 ++-- .../ui/cypress/common/Utils/Services.ts | 4 +-- .../ui/cypress/common/Utils/Users.ts | 3 +- .../ui/cypress/common/advancedSearch.js | 3 +- .../resources/ui/cypress/common/common.js | 11 ++++--- .../ui/cypress/common/serviceUtils.js | 3 +- .../ui/cypress/constants/Entity.interface.ts | 16 +++++++++ .../ui/cypress/constants/sidebar.constant.js | 33 +++++++++++++++++++ .../e2e/Features/IncidentManager.spec.js | 20 +++++------ .../cypress/e2e/Features/SchemaSearch.spec.js | 3 +- .../e2e/Features/TeamsHierarchy.spec.js | 3 +- .../e2e/Flow/AddRoleAndAssignToUser.spec.js | 3 +- .../Flow/AdvancedSearchQuickFilters.spec.js | 7 ++-- .../ui/cypress/e2e/Flow/Collect.spec.js | 30 +++++------------ .../ui/cypress/e2e/Flow/PersonaFlow.spec.js | 3 +- .../ui/cypress/e2e/Pages/Alerts.spec.js | 3 +- .../ui/cypress/e2e/Pages/Bots.spec.js | 3 +- .../e2e/Pages/CustomLogoConfig.spec.js | 3 +- .../e2e/Pages/Customproperties.spec.js | 11 ++++--- .../ui/cypress/e2e/Pages/DataInsight.spec.js | 15 +++++---- .../e2e/Pages/DataInsightAlert.spec.js | 4 +-- .../e2e/Pages/DataInsightSettings.spec.js | 3 +- .../e2e/Pages/DataQualityAndProfiler.spec.js | 30 +++-------------- .../ui/cypress/e2e/Pages/Domains.spec.js | 12 +++---- .../ui/cypress/e2e/Pages/Glossary.spec.js | 10 ++---- .../cypress/e2e/Pages/LoginConfiguration.ts | 5 +-- .../ui/cypress/e2e/Pages/Policies.spec.js | 3 +- .../ui/cypress/e2e/Pages/Roles.spec.js | 3 +- .../e2e/Pages/SearchIndexDetails.spec.js | 3 +- .../ui/cypress/e2e/Pages/Service.spec.js | 3 +- .../ui/cypress/e2e/Pages/Teams.spec.js | 3 +- .../ui/cypress/e2e/Pages/Users.spec.ts | 13 ++------ .../ui/cypress/e2e/Pages/redirections.spec.js | 1 - .../e2e/Service/redshiftWithDBT.spec.js | 7 ++-- .../resources/ui/cypress/support/commands.js | 28 ++++++++++------ .../ui/src/constants/LeftSidebar.constants.ts | 25 +++++++------- .../resources/ui/src/enums/sidebar.enum.ts | 28 ++++++++++++++++ 43 files changed, 225 insertions(+), 179 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.js create mode 100644 openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js index 17609e5d33cb..283d641eb425 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/CustomizeLandingPageUtils.js @@ -13,6 +13,7 @@ // eslint-disable-next-line spaced-comment /// +import { SidebarItem } from '../constants/Entity.interface'; import { interceptURL, toastNotification, @@ -47,7 +48,7 @@ export const navigateToCustomizeLandingPage = ({ }) => { interceptURL('GET', '/api/v1/teams/name/*', 'settingsPage'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@settingsPage', 200); cy.get('[data-testid="settings-left-panel"]').should('be.visible'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js index 3891927c7a6e..2e00aaa6dab9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js @@ -17,6 +17,7 @@ import { NAME_VALIDATION_ERROR, SEARCH_ENTITY_TABLE, } from '../constants/constants'; +import { SidebarItem } from '../constants/Entity.interface'; import { descriptionBox, interceptURL, @@ -145,7 +146,7 @@ export const updateAssets = (domainObj) => { cy.get('[data-testid="domain-link"]').should('contain', domainObj.name); - cy.sidebarClick('app-bar-item-domain'); + cy.sidebarClick(SidebarItem.DOMAIN); goToAssetsTab(domainObj); @@ -179,7 +180,7 @@ export const removeAssets = (domainObj) => { cy.get('[data-testid="remove-owner"]').click(); verifyResponseStatusCode('@patchDomain', 200); - cy.sidebarClick('app-bar-item-domain'); + cy.sidebarClick(SidebarItem.DOMAIN); goToAssetsTab(domainObj); cy.contains('Adding a new Asset is easy, just give it a spin!').should( diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts index 6941ff1d6d72..cf61ca3a8a70 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/ServiceBaseClass.ts @@ -14,6 +14,7 @@ import { INVALID_NAMES, NAME_VALIDATION_ERROR, } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; import { interceptURL, replaceAllSpacialCharWith_, @@ -354,7 +355,7 @@ class ServiceBaseClass { verifyResponseStatusCode('@updateEntity', 200); // re-run ingestion flow - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); // Services page cy.get('.ant-menu-title-content').contains(this.category).click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/UserClass.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/UserClass.ts index 124241ff28cf..49d6155f219e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/UserClass.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Entities/UserClass.ts @@ -16,6 +16,7 @@ import { interceptURL, verifyResponseStatusCode, } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; import { VISIT_SERVICE_PAGE_DETAILS } from '../../constants/service.constants'; import { permanentDeleteUser, @@ -31,7 +32,7 @@ class UsersTestClass { } visitUserListPage() { - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL('GET', '/api/v1/users?*', 'getUsers'); cy.get('[data-testid="settings-left-panel"]').contains('Users').click(); } @@ -76,14 +77,14 @@ class UsersTestClass { } checkStewardServicesPermissions() { - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); Object.values(VISIT_SERVICE_PAGE_DETAILS).forEach((service) => { - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get(`[data-menu-id*="${service.settingsMenuId}"]`).click(); cy.get('[data-testid="add-service-button"] > span').should('not.exist'); }); - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); cy.get('[data-testid="tables-tab"]').click(); cy.get( '.ant-drawer-title > [data-testid="entity-link"] > .ant-typography' diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js index 2b570dbf0b96..bf823c527b47 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/GlossaryUtils.js @@ -14,6 +14,7 @@ /// import { DELETE_TERM } from '../constants/constants'; +import { SidebarItem } from '../constants/Entity.interface'; import { interceptURL, toastNotification, @@ -23,15 +24,7 @@ import { export const visitGlossaryPage = () => { interceptURL('GET', '/api/v1/glossaries?fields=*', 'getGlossaries'); - cy.sidebarHover(); - cy.get('[data-testid="governance"]').click({ - animationDistanceThreshold: 20, - waitForAnimations: true, - }); - - // Applying force true as the hover over tooltip - - cy.sidebarClick('app-bar-item-glossary'); + cy.sidebarClick(SidebarItem.GLOSSARY); verifyResponseStatusCode('@getGlossaries', 200); }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.js index 446dadf8a0a7..62429c83bb95 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/TagUtils.js @@ -17,6 +17,7 @@ import { NAME_VALIDATION_ERROR, TAG_INVALID_NAMES, } from '../constants/constants'; +import { SidebarItem } from '../constants/Entity.interface'; import { interceptURL, verifyResponseStatusCode } from './common'; export const submitForm = () => { @@ -59,13 +60,7 @@ export const validateForm = () => { export const visitClassificationPage = () => { interceptURL('GET', '/api/v1/tags*', 'getTags'); - cy.sidebarHover(); - cy.get('[data-testid="governance"]').click({ - animationDistanceThreshold: 20, - waitForAnimations: true, - }); - - cy.sidebarClick('app-bar-item-tags'); + cy.sidebarClick(SidebarItem.TAGS); verifyResponseStatusCode('@getTags', 200); }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts index 07a2324c783b..baffc00b6f53 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/CustomProperty.ts @@ -11,7 +11,7 @@ * limitations under the License. */ -import { EntityType } from '../../constants/Entity.interface'; +import { EntityType, SidebarItem } from '../../constants/Entity.interface'; import { interceptURL, verifyResponseStatusCode } from '../common'; export enum CustomPropertyType { @@ -41,7 +41,7 @@ export const createCustomPropertyForEntity = ({ }) => { interceptURL('GET', '/api/v1/teams/name/*', 'settingsPage'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@settingsPage', 200); @@ -81,7 +81,7 @@ export const deleteCustomPropertyForEntity = ({ property: CustomProperty; type: EntityType; }) => { - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL('GET', `/api/v1/metadata/types/name/*`, 'getEntity'); interceptURL('PATCH', `/api/v1/metadata/types/*`, 'patchEntity'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts index 76ca9eb6f9ec..22c408aaf4ef 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Services.ts @@ -11,7 +11,7 @@ * limitations under the License. */ import { DELETE_TERM } from '../../constants/constants'; -import { EntityType } from '../../constants/Entity.interface'; +import { EntityType, SidebarItem } from '../../constants/Entity.interface'; import { interceptURL, toastNotification, @@ -38,7 +38,7 @@ export const goToServiceListingPage = (services: Services) => { 'getSettingsPage' ); // Click on settings page - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getSettingsPage', 200); // Services page interceptURL('GET', '/api/v1/services/*', 'getServiceList'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Users.ts b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Users.ts index e460ab511842..bfd11699a129 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Users.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/Utils/Users.ts @@ -14,6 +14,7 @@ import { customFormatDateTime, getEpochMillisForFutureDays, } from '../../../src/utils/date-time/DateTimeUtils'; +import { SidebarItem } from '../../constants/Entity.interface'; import { descriptionBox, interceptURL, @@ -170,7 +171,7 @@ export const permanentDeleteUser = (username: string) => { cy.get('[data-testid="search-error-placeholder"]').should('be.exist'); }; export const visitUserListPage = () => { - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL('GET', '/api/v1/users?*', 'getUsers'); cy.get('[data-testid="settings-left-panel"]').contains('Users').click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js b/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js index 632b8ea31dcd..df19359e75c6 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js @@ -12,6 +12,7 @@ */ import { SEARCH_ENTITY_TABLE } from '../constants/constants'; +import { SidebarItem } from '../constants/Entity.interface'; import { DATABASE_DETAILS, DATABASE_SERVICE_DETAILS, @@ -229,7 +230,7 @@ export const searchForField = (condition, fieldid, searchCriteria, index) => { export const goToAdvanceSearch = () => { // Navigate to explore page - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); cy.get('[data-testid="advance-search-button"]').click(); cy.get('[data-testid="reset-btn"]').click(); }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js index bdeddb18221e..01046536ba25 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js @@ -30,6 +30,7 @@ import { NAME_VALIDATION_ERROR, SEARCH_INDEX, } from '../constants/constants'; +import { SidebarItem } from '../constants/Entity.interface'; export const descriptionBox = '.toastui-editor-md-container > .toastui-editor > .ProseMirror'; @@ -425,7 +426,7 @@ export const deleteCreatedService = ( 'getSettingsPage' ); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getSettingsPage', 200); // Services page @@ -511,7 +512,7 @@ export const goToAddNewServicePage = (service_type) => { 'getSettingsPage' ); // Click on settings page - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getSettingsPage', 200); // Services page @@ -967,7 +968,7 @@ export const updateDescriptionForIngestedTables = ( verifyResponseStatusCode('@updateEntity', 200); // re-run ingestion flow - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); // Services page cy.get('.ant-menu-title-content').contains(type).should('be.visible').click(); @@ -1141,7 +1142,7 @@ export const visitServiceDetailsPage = ( ); interceptURL('GET', '/api/v1/teams/name/*', 'getOrganization'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getOrganization', 200); @@ -1173,7 +1174,7 @@ export const visitServiceDetailsPage = ( export const visitDataModelPage = (dataModelFQN, dataModelName) => { interceptURL('GET', '/api/v1/teams/name/*', 'getOrganization'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getOrganization', 200); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.js index d5e673ccd559..9ed9a2a14b06 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/serviceUtils.js @@ -10,6 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { SidebarItem } from '../constants/Entity.interface'; import { interceptURL, verifyResponseStatusCode } from './common'; export const searchServiceFromSettingPage = (service) => { @@ -24,7 +25,7 @@ export const searchServiceFromSettingPage = (service) => { }; export const visitServiceDetailsPage = (service, verifyHeader = true) => { - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); // Services page interceptURL('GET', '/api/v1/services/*', 'getServices'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts b/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts index 80baa51afacc..fd48d3cf5b96 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/Entity.interface.ts @@ -94,3 +94,19 @@ export const SEARCH_INDEX: Record< [EntityType.Glossary]: 'glossary_search_index', [EntityType.Domain]: 'domain_search_index', } as const; + +export enum SidebarItem { + EXPLORE = 'explore', + OBSERVABILITY = 'observability', + DATA_QUALITY = 'data-quality', + INCIDENT_MANAGER = 'incident-manager', + OBSERVABILITY_ALERT = 'observability-alert', + DATA_INSIGHT = 'data-insight', + DOMAIN = 'domain', + GOVERNANCE = 'governance', + GLOSSARY = 'glossary', + TAGS = 'tags', + INSIGHTS = 'insights', + SETTINGS = 'settings', + LOGOUT = 'logout', +} diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.js new file mode 100644 index 000000000000..36867a49f69c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/sidebar.constant.js @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { SidebarItem } from './Entity.interface'; + +export const SIDEBAR_LIST_ITEMS = { + [SidebarItem.DATA_QUALITY]: [ + SidebarItem.OBSERVABILITY, + SidebarItem.DATA_QUALITY, + ], + [SidebarItem.INCIDENT_MANAGER]: [ + SidebarItem.OBSERVABILITY, + SidebarItem.INCIDENT_MANAGER, + ], + [SidebarItem.OBSERVABILITY_ALERT]: [ + SidebarItem.OBSERVABILITY, + SidebarItem.OBSERVABILITY_ALERT, + ], + [SidebarItem.GLOSSARY]: [SidebarItem.GOVERNANCE, SidebarItem.GLOSSARY], + [SidebarItem.TAGS]: [SidebarItem.GOVERNANCE, SidebarItem.TAGS], + + // Profile Dropdown + 'user-name': ['dropdown-profile', 'user-name'], +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js index 3ab72acc62a4..12f1f8f47c9f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/IncidentManager.spec.js @@ -19,6 +19,7 @@ import { } from '../../common/common'; import { createEntityTableViaREST } from '../../common/Utils/Entity'; import { DATA_ASSETS, NEW_TABLE_TEST_CASE } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; const TABLE_NAME = DATABASE_SERVICE.entity.name; @@ -148,9 +149,7 @@ describe('Incident Manager', () => { }); it('Assign incident to user', () => { - cy.sidebarHover(); - cy.get("[data-testid='observability'").click(); - cy.sidebarClick('app-bar-item-incident-manager'); + cy.sidebarClick(SidebarItem.INCIDENT_MANAGER); cy.get(`[data-testid="test-case-${NEW_TABLE_TEST_CASE.name}"]`).should( 'be.visible' ); @@ -187,9 +186,9 @@ describe('Incident Manager', () => { 'getTestCase' ); interceptURL('GET', '/api/v1/feed?entityLink=*&type=Task', 'getTaskFeed'); - cy.sidebarHover(); - cy.get("[data-testid='observability'").click(); - cy.sidebarClick('app-bar-item-incident-manager'); + + cy.sidebarClick(SidebarItem.INCIDENT_MANAGER); + cy.get(`[data-testid="test-case-${NEW_TABLE_TEST_CASE.name}"]`).click(); verifyResponseStatusCode('@getTestCase', 200); cy.get('[data-testid="incident"]').click(); @@ -226,9 +225,7 @@ describe('Incident Manager', () => { 'getTestCase' ); interceptURL('GET', '/api/v1/feed?entityLink=*&type=Task', 'getTaskFeed'); - cy.sidebarHover(); - cy.get("[data-testid='observability'").click(); - cy.sidebarClick('app-bar-item-incident-manager'); + cy.sidebarClick(SidebarItem.INCIDENT_MANAGER); cy.get(`[data-testid="test-case-${NEW_TABLE_TEST_CASE.name}"]`).click(); verifyResponseStatusCode('@getTestCase', 200); cy.get('[data-testid="incident"]').click(); @@ -338,9 +335,8 @@ describe('Incident Manager', () => { '/api/v1/dataQuality/testCases/testCaseIncidentStatus?latest=true&startTs=*&endTs=*&limit=*', 'getIncidentList' ); - cy.sidebarHover(); - cy.get("[data-testid='observability'").click(); - cy.sidebarClick('app-bar-item-incident-manager'); + cy.sidebarClick(SidebarItem.INCIDENT_MANAGER); + verifyResponseStatusCode('@getIncidentList', 200); cy.get(`[data-testid="test-case-${testName}"]`).should('be.visible'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js index 301498abba25..03e847311fbb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/SchemaSearch.spec.js @@ -15,6 +15,7 @@ import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { searchServiceFromSettingPage } from '../../common/serviceUtils'; +import { SidebarItem } from '../../constants/Entity.interface'; const schemaNames = ['sales', 'admin', 'anonymous', 'dip', 'gsmadmin_internal']; let serviceId; @@ -87,7 +88,7 @@ describe('Schema search', () => { 'getSettingsPage' ); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getSettingsPage', 200); // Services page diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js index 3703f675030b..0883357659ae 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Features/TeamsHierarchy.spec.js @@ -17,6 +17,7 @@ import { uuid, verifyResponseStatusCode, } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; const buTeamName = `bu-${uuid()}`; const divTeamName = `div-${uuid()}`; @@ -40,7 +41,7 @@ describe('Add nested teams and test TeamsSelectable', () => { interceptURL('GET', '/api/v1/teams/name/*', 'getOrganization'); interceptURL('GET', '/api/v1/permissions/team/name/*', 'getPermissions'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getOrganization', 200); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js index 78a5836b299c..f042c43175f4 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AddRoleAndAssignToUser.spec.js @@ -18,6 +18,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { BASE_URL } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const roleName = `Role-test-${uuid()}`; const userName = `usercttest${uuid()}`; @@ -28,7 +29,7 @@ describe('Test Add role and assign it to the user', () => { cy.login(); interceptURL('GET', '*api/v1/roles*', 'getRoles'); interceptURL('GET', '/api/v1/users?*', 'usersPage'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); }); it('Create role', () => { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js index 264a149afa1b..ae8b80b3933d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js @@ -16,6 +16,7 @@ import { searchAndClickOnOption } from '../../common/advancedSearchQuickFilters' import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { QUICK_FILTERS_BY_ASSETS } from '../../constants/advancedSearchQuickFilters.constants'; import { SEARCH_ENTITY_TABLE } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const ownerName = 'Aaron Johnson'; describe(`Advanced search quick filters should work properly for assets`, () => { @@ -35,7 +36,7 @@ describe(`Advanced search quick filters should work properly for assets`, () => it(`should show the quick filters for respective assets`, () => { // Navigate to explore page - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); QUICK_FILTERS_BY_ASSETS.map((asset) => { cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); @@ -52,7 +53,7 @@ describe(`Advanced search quick filters should work properly for assets`, () => const asset = QUICK_FILTERS_BY_ASSETS[0]; // Navigate to explore page - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); asset.filters @@ -77,7 +78,7 @@ describe(`Advanced search quick filters should work properly for assets`, () => }); const testIsNullAndIsNotNullFilters = (operatorTitle, queryFilter, alias) => { - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); const asset = QUICK_FILTERS_BY_ASSETS[0]; cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); cy.get('[data-testid="advance-search-button"]').click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.js index 7157e583818e..eece92fe2bf7 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Collect.spec.js @@ -12,40 +12,37 @@ */ import { interceptURL } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; describe('Collect end point should work properly', () => { const PAGES = { setting: { name: 'Settings', - mainMenuId: `[data-testid="app-bar-item-settings"]`, + menuId: SidebarItem.SETTINGS, }, explore: { name: 'Explore', - mainMenuId: `[data-testid="app-bar-item-explore"]`, + menuId: SidebarItem.EXPLORE, }, dataQuality: { name: 'Quality', - mainMenuId: `[data-testid="observability"]`, - subMenu: `[data-testid="app-bar-item-data-quality"]`, + menuId: SidebarItem.DATA_QUALITY, }, incidentManager: { name: 'Incident Manager', - mainMenuId: `[data-testid="observability"]`, - subMenu: `[data-testid="app-bar-item-incident-manager"]`, + menuId: SidebarItem.INCIDENT_MANAGER, }, insight: { name: 'Insights', - mainMenuId: `[data-testid="app-bar-item-data-insight"]`, + menuId: SidebarItem.DATA_INSIGHT, }, glossary: { name: 'Glossary', - mainMenuId: `[data-testid="governance"]`, - subMenu: `[data-testid="app-bar-item-glossary"]`, + menuId: SidebarItem.GLOSSARY, }, tag: { name: 'Tags', - mainMenuId: `[data-testid="governance"]`, - subMenu: `[data-testid="app-bar-item-tags"]`, + menuId: SidebarItem.TAGS, }, }; @@ -66,16 +63,7 @@ describe('Collect end point should work properly', () => { Object.values(PAGES).map((page) => { it(`Visit ${page.name} page should trigger collect API`, () => { - cy.sidebarHover(); - cy.get(page.mainMenuId) - .should('be.visible') - .click({ animationDistanceThreshold: 10 }); - if (page.subMenu) { - // adding manual wait to open dropdown in UI - cy.wait(500); - cy.get(page.subMenu).should('be.visible').click({ force: true }); - } - cy.sidebarHoverOutside(); + cy.sidebarClick(page.menuId); assertCollectEndPoint(); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js index dd6a754c3635..d67f76bae9c3 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/PersonaFlow.spec.js @@ -20,6 +20,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { DELETE_TERM } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; import { PERSONA_DETAILS, USER_DETAILS } from '../../constants/EntityConstant'; const updatePersonaDisplayName = (displayName) => { @@ -78,7 +79,7 @@ describe('Persona operations', () => { cy.login(); interceptURL('GET', '/api/v1/teams/name/*', 'settingsPage'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@settingsPage', 200); cy.get('[data-testid="settings-left-panel"]').should('be.visible'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Alerts.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Alerts.spec.js index ea2da021d7db..4833a61c967c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Alerts.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Alerts.spec.js @@ -22,6 +22,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { DELETE_TERM, DESTINATION, TEST_CASE } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const alertForAllAssets = `Alert-ct-test-${uuid()}`; const description = 'This is alert description'; @@ -62,7 +63,7 @@ describe.skip('Alerts page should work properly', () => { interceptURL('POST', '/api/v1/events/subscriptions', 'createAlert'); interceptURL('GET', `/api/v1/search/query?q=*`, 'getSearchResult'); cy.login(); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL('GET', '/api/v1/events/subscriptions?*', 'alertsPage'); cy.get('[data-testid="global-setting-left-panel"]') .contains('Alerts') diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.js index 67be0b148f07..c610f306cf7c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Bots.spec.js @@ -21,6 +21,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { DELETE_TERM } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const botName = `Bot-ct-test-${uuid()}`; const botEmail = `${botName}@mail.com`; @@ -70,7 +71,7 @@ const revokeToken = () => { describe('Bots Page should work properly', () => { beforeEach(() => { cy.login(); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL( 'GET', 'api/v1/bots?limit=*&include=non-deleted', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js index c4cf70856cc4..05cf1106d8c3 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/CustomLogoConfig.spec.js @@ -14,6 +14,7 @@ /// import { interceptURL, verifyResponseStatusCode } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; const config = { logo: 'https://custom-logo.png', @@ -26,7 +27,7 @@ describe('Custom Logo Config', () => { beforeEach(() => { cy.login(); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL( 'GET', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js index e9027a96796e..cdb85db4ca31 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Customproperties.spec.js @@ -19,12 +19,13 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { ENTITIES, uuid } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; describe('Custom Properties should work properly', () => { beforeEach(() => { cy.login(); interceptURL('GET', '/api/v1/teams/name/*', 'settingsPage'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@settingsPage', 200); cy.get('[data-testid="settings-left-panel"]').should('be.visible'); }); @@ -57,7 +58,7 @@ describe('Custom Properties should work properly', () => { ); // Navigating back to custom properties page - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get(`[data-menu-id*="customAttributes.${entity.name}"]`) .scrollIntoView() .click(); @@ -128,7 +129,7 @@ describe('Custom Properties should work properly', () => { ); // Navigating back to custom properties page - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); // Selecting the entity cy.get(`[data-menu-id*="customAttributes.${entity.name}"]`) .scrollIntoView() @@ -202,7 +203,7 @@ describe('Custom Properties should work properly', () => { ); // Navigating back to custom properties page - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get(`[data-menu-id*="customAttributes.${entity.name}"]`) .scrollIntoView() .should('be.visible') @@ -273,7 +274,7 @@ describe('Custom Properties should work properly', () => { ); // Navigating to explore page - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); interceptURL( 'GET', `/api/v1/metadata/types/name/glossaryTerm*`, diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js index a69ed0ef230d..ee1e3844c8b2 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsight.spec.js @@ -23,6 +23,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { checkDataInsightSuccessStatus } from '../../common/DataInsightUtils'; +import { SidebarItem } from '../../constants/Entity.interface'; const KPI_DATA = [ { @@ -93,13 +94,13 @@ describe('Data Insight feature', () => { }); it('Initial setup', () => { - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); deleteKpiRequest(); }); it('Create description and owner KPI', () => { - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); cy.get('[data-menu-id*="kpi"]').click(); KPI_DATA.map((data) => { @@ -144,7 +145,7 @@ describe('Data Insight feature', () => { }); it('Verifying Data assets tab', () => { - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); cy.get('[data-testid="search-dropdown-Team"]').should('be.visible'); cy.get('[data-testid="search-dropdown-Tier"]').should('be.visible'); @@ -170,7 +171,7 @@ describe('Data Insight feature', () => { }); it('Verifying App analytics tab', () => { - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); cy.get('[data-menu-id*="app-analytics"]').click(); verifyResponseStatusCode('@dataInsightsChart', 200); @@ -192,7 +193,7 @@ describe('Data Insight feature', () => { }); it('Verifying KPI tab', () => { - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); cy.get('[data-menu-id*="kpi"]').click(); verifyResponseStatusCode('@dataInsightsChart', 200); @@ -210,7 +211,7 @@ describe('Data Insight feature', () => { it('Update KPI', () => { interceptURL('GET', '/api/v1/kpi/name/*', 'fetchKpiByName'); interceptURL('PATCH', '/api/v1/kpi/*', 'updateKpi'); - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); cy.get('[data-menu-id*="kpi"]').click(); verifyResponseStatusCode('@dataInsightsChart', 200); @@ -233,7 +234,7 @@ describe('Data Insight feature', () => { '/api/v1/kpi/*?hardDelete=true&recursive=false', 'deleteKpi' ); - cy.sidebarClick('app-bar-item-data-insight'); + cy.sidebarClick(SidebarItem.DATA_INSIGHT); verifyResponseStatusCode('@dataInsightsChart', 200); cy.get('[data-menu-id*="kpi"]').click(); verifyResponseStatusCode('@dataInsightsChart', 200); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.js index 4cfb70fbd8a3..49eaede3e47b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightAlert.spec.js @@ -16,6 +16,7 @@ import { toastNotification, verifyResponseStatusCode, } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; const dataInsightReport = { triggerType: 'Scheduled', @@ -30,8 +31,7 @@ const dataInsightReport = { describe.skip('Data Insight Alert', () => { beforeEach(() => { cy.login(); - cy.sidebarClick('app-bar-item-settings'); - + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL( 'GET', 'api/v1/events/subscriptions/name/DataInsightReport?include=all', diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js index 51c050b6e042..14560c175530 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataInsightSettings.spec.js @@ -12,13 +12,14 @@ */ import { interceptURL, verifyResponseStatusCode } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; describe('Data Insight settings page should work properly', () => { beforeEach(() => { cy.login(); interceptURL('GET', '/api/v1/teams/name/*', 'settingsPage'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@settingsPage', 200); cy.get('[data-testid="settings-left-panel"]').should('be.visible'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js index f8707ec42588..8713258b117b 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/DataQualityAndProfiler.spec.js @@ -43,6 +43,7 @@ import { SERVICE_TYPE, TEAM_ENTITY, } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; import { DATABASE_SERVICE } from '../../constants/EntityConstant'; import { SERVICE_CATEGORIES } from '../../constants/service.constants'; @@ -114,13 +115,7 @@ const visitTestSuiteDetailsPage = (testSuiteName) => { ); interceptURL('GET', '/api/v1/dataQuality/testCases?fields=*', 'testCase'); - cy.sidebarHover(); - - cy.get('[data-testid="observability"]').click(); - - cy.sidebarClick('app-bar-item-data-quality'); - - cy.sidebarHoverOutside(); + cy.sidebarClick(SidebarItem.DATA_QUALITY); cy.get('[data-testid="by-test-suites"]').click(); verifyResponseStatusCode('@testSuite', 200); @@ -214,11 +209,8 @@ describe('Data Quality and Profiler should work properly', () => { goToProfilerTab(); cy.get('[data-testid="no-profiler-placeholder"]').should('be.visible'); - cy.clickOnLogo(); - - cy.sidebarClick('app-bar-item-settings'); - + cy.sidebarClick(SidebarItem.SETTINGS); cy.get('[data-menu-id*="services.databases"]').should('be.visible').click(); cy.intercept('/api/v1/services/ingestionPipelines?*').as('ingestionData'); interceptURL( @@ -549,13 +541,7 @@ describe('Data Quality and Profiler should work properly', () => { 'getTestCase' ); - cy.sidebarHover(); - - cy.get('[data-testid="observability"]').click(); - - cy.sidebarClick('app-bar-item-data-quality'); - - cy.sidebarHoverOutside(); + cy.sidebarClick(SidebarItem.DATA_QUALITY); cy.get('[data-testid="by-test-suites"]').click(); verifyResponseStatusCode('@testSuite', 200); @@ -865,13 +851,7 @@ describe('Data Quality and Profiler should work properly', () => { it('Update displayName of test case', () => { interceptURL('GET', '/api/v1/dataQuality/testCases?*', 'getTestCase'); - cy.sidebarHover(); - - cy.get('[data-testid="observability"]').click(); - - cy.sidebarClick('app-bar-item-data-quality'); - - cy.sidebarHoverOutside(); + cy.sidebarClick(SidebarItem.DATA_QUALITY); cy.get('[data-testid="by-test-cases"]').click(); verifyResponseStatusCode('@getTestCase', 200); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js index f040f52e1dce..bf699b3ab0b9 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js @@ -29,12 +29,12 @@ import { verifyDomain, } from '../../common/DomainUtils'; import { DOMAIN_1, DOMAIN_2, DOMAIN_3 } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; describe('Domain page should work properly', () => { beforeEach(() => { cy.login(); - - cy.sidebarClick('app-bar-item-domain'); + cy.sidebarClick(SidebarItem.DOMAIN); }); it('Create new domain flow should work properly', () => { @@ -59,14 +59,14 @@ describe('Domain page should work properly', () => { it('Create new data product should work properly', () => { DOMAIN_1.dataProducts.forEach((dataProduct) => { createDataProducts(dataProduct, DOMAIN_1); - cy.sidebarClick('app-bar-item-domain'); + cy.sidebarClick(SidebarItem.DOMAIN); }); }); it('Add data product assets using asset selection modal should work properly', () => { DOMAIN_2.dataProducts.forEach((dp) => { createDataProducts(dp, DOMAIN_2); - cy.sidebarClick('app-bar-item-domain'); + cy.sidebarClick(SidebarItem.DOMAIN); }); addAssetsToDataProduct(DOMAIN_2.dataProducts[0], DOMAIN_2); @@ -75,7 +75,7 @@ describe('Domain page should work properly', () => { it('Add data product assets using asset selection modal with separate domain and dp having space', () => { DOMAIN_3.dataProducts.forEach((dp) => { createDataProducts(dp, DOMAIN_3); - cy.sidebarClick('app-bar-item-domain'); + cy.sidebarClick(SidebarItem.DOMAIN); }); addAssetsToDataProduct(DOMAIN_3.dataProducts[0], DOMAIN_3); @@ -96,7 +96,7 @@ describe('Domain page should work properly', () => { 'tableSearchQuery' ); - cy.sidebarClick('app-bar-item-explore'); + cy.sidebarClick(SidebarItem.EXPLORE); verifyResponseStatusCode('@tableSearchQuery', 200); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js index b55f1a3bf3f0..4b1e2d1ef2c0 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js @@ -44,6 +44,7 @@ import { NEW_GLOSSARY_TERMS, SEARCH_ENTITY_TABLE, } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const userName = `test_dataconsumer${uuid()}`; @@ -584,14 +585,7 @@ const voteGlossary = (isGlossary) => { const goToGlossaryPage = () => { interceptURL('GET', '/api/v1/glossaryTerms*', 'getGlossaryTerms'); interceptURL('GET', '/api/v1/glossaries?fields=*', 'fetchGlossaries'); - - cy.sidebarHover(); - cy.get('[data-testid="governance"]').click({ - animationDistanceThreshold: 20, - waitForAnimations: true, - }); - - cy.sidebarClick('app-bar-item-glossary', 'governance'); + cy.sidebarClick(SidebarItem.GLOSSARY); }; const approveGlossaryTermWorkflow = ({ glossary, glossaryTerm }) => { diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts index 2987752e8bb4..40b2d5a76f60 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/LoginConfiguration.ts @@ -1,3 +1,5 @@ +import { SidebarItem } from '../../constants/Entity.interface'; + /* * Copyright 2023 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +15,7 @@ describe('template spec', () => { beforeEach(() => { cy.login(); - - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get('[data-testid="settings-left-panel"]') .contains('Login Configuration') diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js index e82ad1d0dbbd..da636c402553 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Policies.spec.js @@ -18,6 +18,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { BASE_URL } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const roles = { dataConsumer: 'Data Consumer', @@ -100,7 +101,7 @@ describe('Policy page should work properly', () => { cy.login(); cy.intercept('GET', '*api/v1/policies*').as('getPolicies'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get('[data-testid="settings-left-panel"]') .contains('Policies') diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js index 05873e4dab79..ba6abdb88b1c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Roles.spec.js @@ -18,6 +18,7 @@ import { verifyResponseStatusCode, } from '../../common/common'; import { BASE_URL } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; const roles = { dataConsumer: 'Data Consumer', @@ -61,7 +62,7 @@ describe('Roles page should work properly', () => { interceptURL('GET', '*api/v1/roles*', 'getRoles'); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get('[data-testid="settings-left-panel"]') .contains('Roles') diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js index dcb59105de9d..ee17b5d664ee 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/SearchIndexDetails.spec.js @@ -24,6 +24,7 @@ import { visitEntityDetailsPage, } from '../../common/common'; import { BASE_URL, uuid } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; import { SEARCH_INDEX_DETAILS_FOR_DETAILS_PAGE_TEST, SEARCH_INDEX_DISPLAY_NAME, @@ -190,7 +191,7 @@ describe('Prerequisite for data steward role tests', () => { // Assign data steward role to the created user - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); interceptURL('GET', `/api/v1/users?*`, 'getUsersList'); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js index a2dc3fabf0c5..799d0a5d458f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Service.spec.js @@ -18,6 +18,7 @@ import { } from '../../common/common'; import { searchServiceFromSettingPage } from '../../common/serviceUtils'; import { service } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; describe('Services page should work properly', () => { beforeEach(() => { @@ -39,7 +40,7 @@ describe('Services page should work properly', () => { cy.login(); // redirecting to services page - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); cy.get('[data-testid="settings-left-panel"]') .contains('Database') diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js index 2a7fc2ea6490..a8a74b67f676 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Teams.spec.js @@ -23,6 +23,7 @@ import { uuid, verifyResponseStatusCode, } from '../../common/common'; +import { SidebarItem } from '../../constants/Entity.interface'; const updatedDescription = 'This is updated description'; @@ -53,7 +54,7 @@ describe('Teams flow should work properly', () => { interceptURL('GET', `/api/v1/permissions/team/name/*`, 'permissions'); cy.login(); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); // Clicking on teams cy.get('[data-testid="settings-left-panel"]').contains('Teams').click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Users.spec.ts b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Users.spec.ts index d63c9b5b8e2b..fdfdcc6c758d 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Users.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Users.spec.ts @@ -32,7 +32,7 @@ import { ID, uuid, } from '../../constants/constants'; -import { EntityType } from '../../constants/Entity.interface'; +import { EntityType, SidebarItem } from '../../constants/Entity.interface'; import { NAVBAR_DETAILS } from '../../constants/redirections.constants'; const entity = new UsersTestClass(); @@ -110,16 +110,7 @@ describe('User with different Roles', () => { cy.url().should('eq', `${BASE_URL}/my-data`); // Check CRUD for Glossary - cy.sidebarHover(); - - cy.get(glossary.testid) - .should('be.visible') - .click({ animationDistanceThreshold: 10, waitForAnimations: true }); - if (glossary.subMenu) { - cy.get(glossary.subMenu).should('be.visible').click({ force: true }); - } - cy.clickOutside(); - + cy.sidebarClick(SidebarItem.GLOSSARY); cy.clickOnLogo(); // Check CRUD for Tags diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/redirections.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/redirections.spec.js index 732a4fd93271..a831b8d42e6f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/redirections.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/redirections.spec.js @@ -34,7 +34,6 @@ describe('Redirection link should work properly', () => { if (navbar.subMenu) { cy.get(navbar.subMenu).should('be.visible').click({ force: true }); } - // cy.get('body').click(); validateURL(navbar.url); cy.clickOnLogo(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/redshiftWithDBT.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/redshiftWithDBT.spec.js index cba926a82420..566dc62b5a5e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/redshiftWithDBT.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/redshiftWithDBT.spec.js @@ -30,6 +30,7 @@ import { HTTP_CONFIG_SOURCE, SERVICE_TYPE, } from '../../constants/constants'; +import { SidebarItem } from '../../constants/Entity.interface'; import { REDSHIFT } from '../../constants/service.constants'; const dbtEntityFqn = `${REDSHIFT.serviceName}.${Cypress.env( @@ -106,7 +107,7 @@ describe('RedShift Ingestion', () => { '/api/v1/services/ingestionPipelines/*/pipelineStatus?startTs=*&endTs=*', 'pipelineStatus' ); - cy.sidebarClick('app-bar-item-settings'); + cy.sidebarClick(SidebarItem.SETTINGS); verifyResponseStatusCode('@getSettingsPage', 200); // Services page interceptURL('GET', '/api/v1/services/*', 'getServices'); @@ -223,10 +224,8 @@ describe('RedShift Ingestion', () => { `/api/v1/tags?*parent=${DBT.classification}*`, 'getTagList' ); - cy.sidebarHover(); - cy.get('[data-testid="governance"]').click(); - cy.sidebarClick('app-bar-item-tags'); + cy.sidebarClick(SidebarItem.TAGS); verifyResponseStatusCode('@fetchClassifications', 200); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/support/commands.js b/openmetadata-ui/src/main/resources/ui/cypress/support/commands.js index a520d101769d..bb954f30e82f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/support/commands.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/support/commands.js @@ -38,6 +38,7 @@ import { interceptURL, verifyResponseStatusCode } from '../common/common'; import { BASE_URL, LOGIN } from '../constants/constants'; +import { SIDEBAR_LIST_ITEMS } from '../constants/sidebar.constant'; Cypress.Commands.add('loginByGoogleApi', () => { cy.log('Logging in to Google'); @@ -164,17 +165,24 @@ Cypress.Commands.add('logout', () => { Cypress.session.clearAllSavedSessions(); }); -// This command is used to click on the sidebar item -// id: data-testid of the sidebar item -// parentId: data-testid of the parent sidebar item to close after click if present -Cypress.Commands.add('sidebarClick', (id, parentId) => { - cy.get(`[data-testid="${id}"]`).click({ - animationDistanceThreshold: 20, - waitForAnimations: true, - }); +/* + This command is used to click on the sidebar item + id: data-testid of the sidebar item to be clicked + */ +Cypress.Commands.add('sidebarClick', (id) => { + const items = SIDEBAR_LIST_ITEMS[id]; + if (items) { + cy.sidebarHover(); + cy.get(`[data-testid="${items[0]}"]`).click({ + animationDistanceThreshold: 20, + waitForAnimations: true, + }); + + cy.get(`[data-testid="app-bar-item-${items[1]}"]`).click(); - if (parentId) { - cy.get(`[data-testid="${parentId}"]`).click(); + cy.get(`[data-testid="${items[0]}"]`).click(); + } else { + cy.get(`[data-testid="app-bar-item-${id}"]`).click(); } cy.sidebarHoverOutside(); diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts index 57335ba54b9a..0a1ef9b54466 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/LeftSidebar.constants.ts @@ -25,6 +25,7 @@ import { ReactComponent as SettingsIcon } from '../assets/svg/ic-settings-v1.svg import { ReactComponent as InsightsIcon } from '../assets/svg/lampcharge.svg'; import { ReactComponent as LogoutIcon } from '../assets/svg/logout.svg'; +import { SidebarItem } from '../enums/sidebar.enum'; import { getDataInsightPathWithFqn } from '../utils/DataInsightUtils'; import { ROUTES } from './constants'; @@ -34,34 +35,34 @@ export const SIDEBAR_LIST = [ label: i18next.t('label.explore'), redirect_url: '/explore/tables', icon: ExploreIcon, - dataTestId: 'app-bar-item-explore', + dataTestId: `app-bar-item-${SidebarItem.EXPLORE}`, }, { key: ROUTES.OBSERVABILITY, label: i18next.t('label.observability'), icon: ObservabilityIcon, - dataTestId: 'observability', + dataTestId: SidebarItem.OBSERVABILITY, children: [ { key: ROUTES.DATA_QUALITY, label: i18next.t('label.data-quality'), redirect_url: ROUTES.DATA_QUALITY, icon: DataQualityIcon, - dataTestId: 'app-bar-item-data-quality', + dataTestId: `app-bar-item-${SidebarItem.DATA_QUALITY}`, }, { key: ROUTES.INCIDENT_MANAGER, label: i18next.t('label.incident-manager'), redirect_url: ROUTES.INCIDENT_MANAGER, icon: IncidentMangerIcon, - dataTestId: 'app-bar-item-incident-manager', + dataTestId: `app-bar-item-${SidebarItem.INCIDENT_MANAGER}`, isBeta: true, }, { key: ROUTES.OBSERVABILITY, label: i18next.t('label.alert-plural'), icon: AlertIcon, - dataTestId: 'app-bar-item-observability-alert', + dataTestId: `app-bar-item-${SidebarItem.OBSERVABILITY_ALERT}`, }, ], }, @@ -70,34 +71,34 @@ export const SIDEBAR_LIST = [ label: i18next.t('label.insight-plural'), redirect_url: getDataInsightPathWithFqn(), icon: InsightsIcon, - dataTestId: 'app-bar-item-data-insight', + dataTestId: `app-bar-item-${SidebarItem.DATA_INSIGHT}`, }, { key: ROUTES.DOMAIN, label: i18next.t('label.domain-plural'), redirect_url: ROUTES.DOMAIN, icon: DomainsIcon, - dataTestId: 'app-bar-item-domain', + dataTestId: `app-bar-item-${SidebarItem.DOMAIN}`, }, { key: 'governance', label: i18next.t('label.govern'), icon: GovernIcon, - dataTestId: 'governance', + dataTestId: SidebarItem.GOVERNANCE, children: [ { key: ROUTES.GLOSSARY, label: i18next.t('label.glossary'), redirect_url: ROUTES.GLOSSARY, icon: GlossaryIcon, - dataTestId: 'app-bar-item-glossary', + dataTestId: `app-bar-item-${SidebarItem.GLOSSARY}`, }, { key: ROUTES.TAGS, label: i18next.t('label.classification'), redirect_url: ROUTES.TAGS, icon: ClassificationIcon, - dataTestId: 'app-bar-item-tags', + dataTestId: `app-bar-item-${SidebarItem.TAGS}`, }, ], }, @@ -108,12 +109,12 @@ export const SETTING_ITEM = { label: i18next.t('label.setting-plural'), redirect_url: ROUTES.SETTINGS, icon: SettingsIcon, - dataTestId: 'app-bar-item-settings', + dataTestId: `app-bar-item-${SidebarItem.SETTINGS}`, }; export const LOGOUT_ITEM = { key: 'logout', label: i18next.t('label.logout'), icon: LogoutIcon, - dataTestId: 'app-bar-item-logout', + dataTestId: `app-bar-item-${SidebarItem.LOGOUT}`, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts new file mode 100644 index 000000000000..501bce18b13a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/enums/sidebar.enum.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export enum SidebarItem { + EXPLORE = 'explore', + OBSERVABILITY = 'observability', + DATA_QUALITY = 'data-quality', + INCIDENT_MANAGER = 'incident-manager', + OBSERVABILITY_ALERT = 'observability-alert', + DATA_INSIGHT = 'data-insight', + DOMAIN = 'domain', + GOVERNANCE = 'governance', + GLOSSARY = 'glossary', + TAGS = 'tags', + INSIGHTS = 'insights', + SETTINGS = 'settings', + LOGOUT = 'logout', +} From 8d61948f3dd382cc4767344c403ce7dadf963c62 Mon Sep 17 00:00:00 2001 From: Abhishek Porwal <80886271+Abhishek332@users.noreply.github.com> Date: Tue, 23 Jan 2024 17:45:38 +0530 Subject: [PATCH 11/49] fix(#14326): tier dropdown is not working in advance search (#14780) * improvement in advance search based on custom property * fix a reading undefined property issue * wip: advance search based on tier * some code cleanup and improvement * some fixes * fix: ui flicker when advanceSearched is apply and refresh the page * some cleanup * no need to call customproperty api call, if entity not suppport customProperties * minor change * fix: autocomplete not working in tier search option in advance search modal * added unit test for advance search provider component * some cleanup * added testcase for open modal * added testcase for resetAllFilters method * removed unwanted code * added e2e test for testing tier advance search * fix: e2e search flow for single field * fix: string field not working after giving listValues in TierSearch * fix: group query e2e test fix * used asyncFetch way to get the tierOptions synchronously * some cleanup * remove unwanted lines * some cleanup * fix: selected option show option value instead of option title --- .../ui/cypress/common/advancedSearch.js | 94 ++++++++----- .../Flow/AdvancedSearchQuickFilters.spec.js | 11 +- .../ui/cypress/e2e/Flow/SearchFlow.spec.js | 47 +++++-- .../Explore/AdvanceSearchModal.component.tsx | 59 +-------- .../AdvanceSearchProvider.component.tsx | 73 ++++++++--- .../AdvanceSearchProvider.test.tsx | 124 ++++++++++++++++++ .../ExploreV1/ExploreV1.component.tsx | 4 +- .../src/constants/AdvancedSearch.constants.ts | 45 +++++-- .../ui/src/utils/AdvancedSearchUtils.tsx | 23 +++- 9 files changed, 338 insertions(+), 142 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.test.tsx diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js b/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js index df19359e75c6..c93ac8954364 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/advancedSearch.js @@ -138,16 +138,15 @@ export const FIELDS = { searchCriteriaSecondGroup: 'PersonalData.SpecialCategory', responseValueSecondGroup: '"tagFQN":"PersonalData.SpecialCategory"', }, - // skipping tier for now, as it is not working, BE need to fix it - - // Tiers: { - // name: 'Tier', - // testid: '[title="Tier"]', - // searchCriteriaFirstGroup: 'Tier.Tier1', - // responseValueFirstGroup: '"tagFQN":"Tier.Tier1"', - // searchCriteriaSecondGroup: 'Tier.Tier2', - // responseValueSecondGroup: '"tagFQN":"Tier.Tier2"', - // }, + Tiers: { + name: 'Tier', + testid: '[title="Tier"]', + searchCriteriaFirstGroup: 'Tier.Tier1', + responseValueFirstGroup: '"tagFQN":"Tier.Tier1"', + searchCriteriaSecondGroup: 'Tier.Tier2', + responseValueSecondGroup: '"tagFQN":"Tier.Tier2"', + isLocalSearch: true, + }, Service: { name: 'Service', testid: '[title="Service"]', @@ -193,12 +192,21 @@ export const OPERATOR = { }, }; -export const searchForField = (condition, fieldid, searchCriteria, index) => { - interceptURL('GET', '/api/v1/search/aggregate?*', 'suggestApi'); +export const searchForField = ( + condition, + fieldId, + searchCriteria, + index, + isLocalSearch = false +) => { + if (!isLocalSearch) { + interceptURL('GET', '/api/v1/search/aggregate?*', 'suggestApi'); + } + // Click on field dropdown cy.get('.rule--field > .ant-select > .ant-select-selector').eq(index).click(); // Select owner fields - cy.get(`${fieldid}`).eq(index).click(); + cy.get(`${fieldId}`).eq(index).click(); // Select the condition cy.get('.rule--operator > .ant-select > .ant-select-selector') .eq(index) @@ -219,8 +227,16 @@ export const searchForField = (condition, fieldid, searchCriteria, index) => { cy.get('.widget--widget > .ant-select > .ant-select-selector') .eq(index) .type(searchCriteria); + + // checking filter is working + cy.get( + `.ant-select-item-option-active[title="${searchCriteria}"]` + ).should('be.visible'); + // select value from dropdown - verifyResponseStatusCode('@suggestApi', 200); + if (!isLocalSearch) { + verifyResponseStatusCode('@suggestApi', 200); + } cy.get(`.ant-select-dropdown [title = '${searchCriteria}']`) .trigger('mouseover') .trigger('click'); @@ -240,12 +256,13 @@ export const checkmustPaths = ( field, searchCriteria, index, - responseSearch + responseSearch, + isLocalSearch ) => { goToAdvanceSearch(); // Search with advance search - searchForField(condition, field, searchCriteria, index); + searchForField(condition, field, searchCriteria, index, isLocalSearch); interceptURL( 'GET', @@ -271,12 +288,13 @@ export const checkmust_notPaths = ( field, searchCriteria, index, - responseSearch + responseSearch, + isLocalSearch ) => { goToAdvanceSearch(); // Search with advance search - searchForField(condition, field, searchCriteria, index); + searchForField(condition, field, searchCriteria, index, isLocalSearch); interceptURL( 'GET', `/api/v1/search/query?q=&index=*&from=0&size=10&deleted=false&query_filter=*must_not*${encodeURI( @@ -417,16 +435,17 @@ export const addTag = ({ tag, term, serviceName, entity }) => { export const checkAddGroupWithOperator = ( condition_1, condition_2, - fieldid, + fieldId, searchCriteria_1, searchCriteria_2, index_1, index_2, - operatorindex, + operatorIndex, filter_1, filter_2, response_1, - response_2 + response_2, + isLocalSearch = false ) => { goToAdvanceSearch(); // Click on field dropdown @@ -435,7 +454,7 @@ export const checkAddGroupWithOperator = ( .should('be.visible') .click(); // Select owner fields - cy.get(fieldid).eq(0).should('be.visible').click(); + cy.get(fieldId).eq(0).should('be.visible').click(); // Select the condition cy.get('.rule--operator > .ant-select > .ant-select-selector') .eq(index_1) @@ -456,13 +475,17 @@ export const checkAddGroupWithOperator = ( .should('be.visible') .type(searchCriteria_1); } else { - interceptURL('GET', '/api/v1/search/aggregate?*', 'suggestApi'); + if (!isLocalSearch) { + interceptURL('GET', '/api/v1/search/aggregate?*', 'suggestApi'); + } cy.get('.widget--widget > .ant-select > .ant-select-selector') .eq(index_1) .should('be.visible') .type(searchCriteria_1); - verifyResponseStatusCode('@suggestApi', 200); + if (!isLocalSearch) { + verifyResponseStatusCode('@suggestApi', 200); + } cy.get('.ant-select-dropdown') .not('.ant-select-dropdown-hidden') .find(`[title="${searchCriteria_1}"]`) @@ -484,13 +507,13 @@ export const checkAddGroupWithOperator = ( // Select the AND/OR condition cy.get( - `.group--conjunctions > .ant-btn-group > :nth-child(${operatorindex})` + `.group--conjunctions > .ant-btn-group > :nth-child(${operatorIndex})` ).click(); // Click on field dropdown cy.get('.rule--field').eq(index_2).should('be.visible').click(); - cy.get(fieldid).eq(2).should('be.visible').click(); + cy.get(fieldId).eq(2).should('be.visible').click(); // Select the condition cy.get('.rule--operator').eq(index_2).should('be.visible').click(); @@ -509,12 +532,17 @@ export const checkAddGroupWithOperator = ( .should('be.visible') .type(searchCriteria_2); } else { - interceptURL('GET', '/api/v1/search/aggregate?*', 'suggestApi'); + if (!isLocalSearch) { + interceptURL('GET', '/api/v1/search/aggregate?*', 'suggestApi'); + } cy.get('.widget--widget > .ant-select > .ant-select-selector') .eq(index_2) .should('be.visible') .type(searchCriteria_2); - verifyResponseStatusCode('@suggestApi', 200); + + if (!isLocalSearch) { + verifyResponseStatusCode('@suggestApi', 200); + } cy.get('.ant-select-dropdown') .not('.ant-select-dropdown-hidden') @@ -547,12 +575,12 @@ export const checkAddGroupWithOperator = ( export const checkAddRuleWithOperator = ( condition_1, condition_2, - fieldid, + fieldId, searchCriteria_1, searchCriteria_2, index_1, index_2, - operatorindex, + operatorIndex, filter_1, filter_2, response_1, @@ -562,7 +590,7 @@ export const checkAddRuleWithOperator = ( // Click on field dropdown cy.get('.rule--field').eq(index_1).should('be.visible').click(); // Select owner fields - cy.get(fieldid).eq(0).should('be.visible').click(); + cy.get(fieldId).eq(0).should('be.visible').click(); // Select the condition cy.get('.rule--operator').eq(index_1).should('be.visible').click(); @@ -604,13 +632,13 @@ export const checkAddRuleWithOperator = ( // Select the AND/OR condition cy.get( - `.group--conjunctions > .ant-btn-group > :nth-child(${operatorindex})` + `.group--conjunctions > .ant-btn-group > :nth-child(${operatorIndex})` ).click(); // Click on field dropdown cy.get('.rule--field').eq(index_2).should('be.visible').click(); - cy.get(fieldid).eq(2).should('be.visible').click(); + cy.get(fieldId).eq(2).should('be.visible').click(); // Select the condition cy.get('.rule--operator').eq(index_2).should('be.visible').click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js index ae8b80b3933d..bf4c614cf419 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/AdvancedSearchQuickFilters.spec.js @@ -11,7 +11,11 @@ * limitations under the License. */ -import { addOwner, removeOwner } from '../../common/advancedSearch'; +import { + addOwner, + goToAdvanceSearch, + removeOwner, +} from '../../common/advancedSearch'; import { searchAndClickOnOption } from '../../common/advancedSearchQuickFilters'; import { interceptURL, verifyResponseStatusCode } from '../../common/common'; import { QUICK_FILTERS_BY_ASSETS } from '../../constants/advancedSearchQuickFilters.constants'; @@ -78,10 +82,7 @@ describe(`Advanced search quick filters should work properly for assets`, () => }); const testIsNullAndIsNotNullFilters = (operatorTitle, queryFilter, alias) => { - cy.sidebarClick(SidebarItem.EXPLORE); - const asset = QUICK_FILTERS_BY_ASSETS[0]; - cy.get(`[data-testid="${asset.tab}"]`).scrollIntoView().click(); - cy.get('[data-testid="advance-search-button"]').click(); + goToAdvanceSearch(); // Check Is Null or Is Not Null cy.get('.rule--operator > .ant-select > .ant-select-selector').eq(0).click(); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js index 1b2afdf8a58d..c91099d9bf8f 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/SearchFlow.spec.js @@ -68,9 +68,12 @@ describe('Advance search', () => { checkmustPaths( condition.name, field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), 0, - field.responseValueFirstGroup + field.responseValueFirstGroup, + field.isLocalSearch ); }); @@ -78,9 +81,12 @@ describe('Advance search', () => { checkmust_notPaths( condition.name, field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), 0, - field.responseValueFirstGroup + field.responseValueFirstGroup, + field.isLocalSearch ); }); }); @@ -108,15 +114,20 @@ describe('Advance search', () => { CONDITIONS_MUST.equalTo.name, CONDITIONS_MUST_NOT.notEqualTo.name, field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), - Cypress._.toLower(field.searchCriteriaSecondGroup), + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), 0, 1, operator.index, CONDITIONS_MUST.equalTo.filter, CONDITIONS_MUST_NOT.notEqualTo.filter, field.responseValueFirstGroup, - val + val, + field.isLocalSearch ); }); }); @@ -131,15 +142,20 @@ describe('Advance search', () => { CONDITIONS_MUST.anyIn.name, CONDITIONS_MUST_NOT.notIn.name, field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), - Cypress._.toLower(field.searchCriteriaSecondGroup), + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), 0, 1, operator.index, CONDITIONS_MUST.anyIn.filter, CONDITIONS_MUST_NOT.notIn.filter, field.responseValueFirstGroup, - val + val, + field.isLocalSearch ); }); }); @@ -152,15 +168,20 @@ describe('Advance search', () => { CONDITIONS_MUST.contains.name, CONDITIONS_MUST_NOT.notContains.name, field.testid, - Cypress._.toLower(field.searchCriteriaFirstGroup), - Cypress._.toLower(field.searchCriteriaSecondGroup), + field.isLocalSearch + ? field.searchCriteriaFirstGroup + : Cypress._.toLower(field.searchCriteriaFirstGroup), + field.isLocalSearch + ? field.searchCriteriaSecondGroup + : Cypress._.toLower(field.searchCriteriaSecondGroup), 0, 1, operator.index, CONDITIONS_MUST.contains.filter, CONDITIONS_MUST_NOT.notContains.filter, field.responseValueFirstGroup, - val + val, + field.isLocalSearch ); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchModal.component.tsx index 523ccd8c34c9..a01f27b8655c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchModal.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchModal.component.tsx @@ -12,18 +12,9 @@ */ import { Button, Modal, Space, Typography } from 'antd'; -import { cloneDeep } from 'lodash'; -import React, { FunctionComponent, useEffect } from 'react'; -import { - Builder, - FieldGroup, - Query, - ValueSource, -} from 'react-awesome-query-builder'; +import React, { FunctionComponent } from 'react'; +import { Builder, Query } from 'react-awesome-query-builder'; import { useTranslation } from 'react-i18next'; -import { getTypeByFQN } from '../../rest/metadataTypeAPI'; -import { EntitiesSupportedCustomProperties } from '../../utils/CustomProperties/CustomProperty.utils'; -import { getEntityTypeFromSearchIndex } from '../../utils/SearchUtils'; import './advanced-search-modal.less'; import { useAdvanceSearch } from './AdvanceSearchProvider/AdvanceSearchProvider.component'; @@ -39,51 +30,7 @@ export const AdvancedSearchModal: FunctionComponent = ({ onCancel, }: Props) => { const { t } = useTranslation(); - const { - config, - treeInternal, - onTreeUpdate, - onReset, - onUpdateConfig, - searchIndex, - } = useAdvanceSearch(); - - const updatedConfig = cloneDeep(config); - - async function getCustomAttributesSubfields() { - try { - const entityType = getEntityTypeFromSearchIndex(searchIndex); - if (!entityType) { - return; - } - const res = await getTypeByFQN(entityType); - const customAttributes = res.customProperties; - - const subfields: Record< - string, - { type: string; valueSources: ValueSource[] } - > = {}; - - if (customAttributes) { - customAttributes.forEach((attr) => { - subfields[attr.name] = { - type: 'text', - valueSources: ['value'], - }; - }); - } - (updatedConfig.fields.extension as FieldGroup).subfields = subfields; - onUpdateConfig(updatedConfig); - } catch (error) { - // Error - } - } - - useEffect(() => { - if (visible && EntitiesSupportedCustomProperties.includes(searchIndex)) { - getCustomAttributesSubfields(); - } - }, [visible, searchIndex]); + const { config, treeInternal, onTreeUpdate, onReset } = useAdvanceSearch(); return ( ( export const AdvanceSearchProvider = ({ children, }: AdvanceSearchProviderProps) => { + const tierOptions = useMemo(getTierOptions, []); + const tabsInfo = searchClassBase.getTabsInfo(); const location = useLocation(); const history = useHistory(); @@ -68,7 +71,10 @@ export const AdvanceSearchProvider = ({ return tabInfo[0] as ExploreSearchIndex; }, [tab]); - const [config, setConfig] = useState(getQbConfigs(searchIndex)); + const [config, setConfig] = useState( + getQbConfigs(searchIndex, tierOptions) + ); + const [initialised, setInitialised] = useState(false); const defaultTree = useMemo( () => QbUtils.checkTree(QbUtils.loadTree(emptyJsonTree), config), @@ -117,7 +123,7 @@ export const AdvanceSearchProvider = ({ ); useEffect(() => { - setConfig(getQbConfigs(searchIndex)); + setConfig(getQbConfigs(searchIndex, tierOptions)); }, [searchIndex]); const handleChange = useCallback( @@ -150,7 +156,7 @@ export const AdvanceSearchProvider = ({ setTreeInternal(QbUtils.checkTree(QbUtils.loadTree(emptyJsonTree), config)); setQueryFilter(undefined); setSQLQuery(''); - }, []); + }, [config]); const handleConfigUpdate = (updatedConfig: Config) => { setConfig(updatedConfig); @@ -171,20 +177,24 @@ export const AdvanceSearchProvider = ({ }, [history, location.pathname]); async function getCustomAttributesSubfields() { - const updatedConfig = cloneDeep(config); + const subfields: Record< + string, + { type: string; valueSources: ValueSource[] } + > = {}; + try { + if (!EntitiesSupportedCustomProperties.includes(searchIndex)) { + return subfields; + } + const entityType = getEntityTypeFromSearchIndex(searchIndex); if (!entityType) { - return; + return subfields; } + const res = await getTypeByFQN(entityType); const customAttributes = res.customProperties; - const subfields: Record< - string, - { type: string; valueSources: ValueSource[] } - > = {}; - if (customAttributes) { customAttributes.forEach((attr) => { subfields[attr.name] = { @@ -193,30 +203,55 @@ export const AdvanceSearchProvider = ({ }; }); } - (updatedConfig.fields.extension as FieldGroup).subfields = subfields; - return updatedConfig; + return subfields; } catch (error) { // Error - return updatedConfig; + return subfields; } } + const loadData = async () => { + const actualConfig = getQbConfigs(searchIndex, tierOptions); + + const extensionSubField = await getCustomAttributesSubfields(); + + if (!isEmpty(extensionSubField)) { + (actualConfig.fields.extension as FieldGroup).subfields = + extensionSubField; + } + + setConfig(actualConfig); + setInitialised(true); + }; + const loadTree = useCallback( async (treeObj: JsonTree) => { - const updatedConfig = (await getCustomAttributesSubfields()) ?? config; + const updatedConfig = config; const tree = QbUtils.checkTree(QbUtils.loadTree(treeObj), updatedConfig); + setTreeInternal(tree); const qFilter = { query: elasticSearchFormat(tree, updatedConfig), }; + if (isEqual(qFilter, queryFilter)) { + return; + } + setQueryFilter(qFilter); setSQLQuery(QbUtils.sqlFormat(tree, updatedConfig) ?? ''); }, - [config] + [config, queryFilter] ); useEffect(() => { + loadData(); + }, [searchIndex]); + + useEffect(() => { + if (!initialised) { + return; + } if (jsonTree) { loadTree(jsonTree); } else { @@ -224,7 +259,7 @@ export const AdvanceSearchProvider = ({ } setLoading(false); - }, [jsonTree]); + }, [jsonTree, initialised]); const handleSubmit = useCallback(() => { const qFilter = { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.test.tsx new file mode 100644 index 000000000000..5b40f65fb660 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.test.tsx @@ -0,0 +1,124 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import React from 'react'; +import { + AdvanceSearchProvider, + useAdvanceSearch, +} from './AdvanceSearchProvider.component'; + +jest.mock('../../../rest/metadataTypeAPI', () => ({ + getTypeByFQN: jest.fn().mockResolvedValue({}), +})); + +jest.mock('../../../rest/tagAPI', () => ({ + getTags: jest.fn().mockResolvedValue({}), +})); + +jest.mock('../AdvanceSearchModal.component', () => ({ + AdvancedSearchModal: jest + .fn() + .mockImplementation(({ visible, onSubmit, onCancel }) => ( + <> + {visible ? ( +

AdvanceSearchModal Open

+ ) : ( +

AdvanceSearchModal Close

+ )} + + + + )), +})); + +jest.mock('../../Loader/Loader', () => + jest.fn().mockReturnValue(
Loader
) +); + +const mockPush = jest.fn(); + +jest.mock('react-router-dom', () => ({ + useLocation: jest.fn().mockReturnValue({ + search: 'queryFilter={"some":"value"}', + }), + useParams: jest.fn().mockReturnValue({ + tab: 'tabValue', + }), + useHistory: jest.fn().mockImplementation(() => ({ + push: mockPush, + })), +})); + +const Children = () => { + const { toggleModal, onResetAllFilters } = useAdvanceSearch(); + + return ( + <> + + + + ); +}; + +const mockWithAdvanceSearch = + (Component: React.FC) => + (props: JSX.IntrinsicAttributes & { children?: React.ReactNode }) => { + return ( + + + + ); + }; + +const ComponentWithProvider = mockWithAdvanceSearch(Children); + +describe('AdvanceSearchProvider component', () => { + it('should render the AdvanceSearchModal as close by default', () => { + render(); + + expect(screen.getByText('AdvanceSearchModal Close')).toBeInTheDocument(); + }); + + it('should call mockPush after submit advance search form', async () => { + render(); + + userEvent.click(screen.getByText('Apply Advance Search')); + + expect(mockPush).toHaveBeenCalled(); + }); + + it('should open the AdvanceSearchModal on call of toggleModal with true', async () => { + await act(async () => { + render(); + }); + + expect(screen.getByText('AdvanceSearchModal Close')).toBeInTheDocument(); + + userEvent.click(screen.getByText('Open AdvanceSearch Modal')); + + expect(screen.getByText('AdvanceSearchModal Open')).toBeInTheDocument(); + }); + + it('onResetAllFilters call mockPush should be called', async () => { + await act(async () => { + render(); + }); + + userEvent.click(screen.getByText('Reset All Filters')); + + expect(mockPush).toHaveBeenCalled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreV1.component.tsx index 8d531745feb9..5db22a90142b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreV1.component.tsx @@ -219,7 +219,7 @@ const ExploreV1: React.FC = ({ handleSummaryPanelDisplay( highlightEntityNameAndDescription( firstEntity._source, - firstEntity.highlight + firstEntity?.highlight ) ); } else { @@ -356,7 +356,7 @@ const ExploreV1: React.FC = ({ handleClosePanel={handleClosePanel} highlights={omit( { - ...firstEntity.highlight, // highlights of firstEntity that we get from the query api + ...firstEntity?.highlight, // highlights of firstEntity that we get from the query api 'tag.name': ( selectedQuickFilters?.find( (filterOption) => filterOption.key === TAG_FQN_KEY diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts index eb62d5e33e5c..7889ee982c66 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts @@ -13,9 +13,12 @@ import { t } from 'i18next'; import { + AsyncFetchListValues, + AsyncFetchListValuesResult, BasicConfig, Fields, JsonTree, + ListItem, SelectFieldSettings, Utils as QbUtils, } from 'react-awesome-query-builder'; @@ -304,6 +307,23 @@ export const autocomplete: (args: { }; }; +export const autoCompleteTier: ( + tierOptions: Promise +) => SelectFieldSettings['asyncFetch'] = (tierOptions) => { + return async (search) => { + const resolvedTierOptions = (await tierOptions) as ListItem[]; + + return { + values: !search + ? resolvedTierOptions + : resolvedTierOptions.filter((tier) => + tier.title?.toLowerCase()?.includes(search.toLowerCase()) + ), + hasMore: false, + } as AsyncFetchListValuesResult; + }; +}; + const mainWidgetProps = { fullWidth: true, valueLabel: t('label.criteria') + ':', @@ -313,7 +333,8 @@ const mainWidgetProps = { * Common fields that exit for all searchable entities */ const getCommonQueryBuilderFields = ( - entitySearchIndex: SearchIndex = SearchIndex.TABLE + entitySearchIndex: SearchIndex = SearchIndex.TABLE, + tierOptions: Promise = Promise.resolve([]) ) => { const commonQueryBuilderFields: Fields = { deleted: { @@ -356,10 +377,7 @@ const getCommonQueryBuilderFields = ( type: 'select', mainWidgetProps, fieldSettings: { - asyncFetch: autocomplete({ - searchIndex: entitySearchIndex ?? [SearchIndex.TAG], - entityField: EntityFields.TIER, - }), + asyncFetch: autoCompleteTier(tierOptions), useAsyncSearch: true, }, }, @@ -522,15 +540,16 @@ const getInitialConfigWithoutFields = () => { /** * Builds search index specific configuration for the query builder */ -export const getQbConfigs: (searchIndex: SearchIndex) => BasicConfig = ( - searchIndex -) => { +export const getQbConfigs: ( + searchIndex: SearchIndex, + tierOptions: Promise +) => BasicConfig = (searchIndex, tierOptions) => { switch (searchIndex) { case SearchIndex.MLMODEL: return { ...getInitialConfigWithoutFields(), fields: { - ...getCommonQueryBuilderFields(SearchIndex.MLMODEL), + ...getCommonQueryBuilderFields(SearchIndex.MLMODEL, tierOptions), ...getServiceQueryBuilderFields(SearchIndex.MLMODEL), }, }; @@ -539,7 +558,7 @@ export const getQbConfigs: (searchIndex: SearchIndex) => BasicConfig = ( return { ...getInitialConfigWithoutFields(), fields: { - ...getCommonQueryBuilderFields(SearchIndex.PIPELINE), + ...getCommonQueryBuilderFields(SearchIndex.PIPELINE, tierOptions), ...getServiceQueryBuilderFields(SearchIndex.PIPELINE), }, }; @@ -548,7 +567,7 @@ export const getQbConfigs: (searchIndex: SearchIndex) => BasicConfig = ( return { ...getInitialConfigWithoutFields(), fields: { - ...getCommonQueryBuilderFields(SearchIndex.DASHBOARD), + ...getCommonQueryBuilderFields(SearchIndex.DASHBOARD, tierOptions), ...getServiceQueryBuilderFields(SearchIndex.DASHBOARD), }, }; @@ -557,7 +576,7 @@ export const getQbConfigs: (searchIndex: SearchIndex) => BasicConfig = ( return { ...getInitialConfigWithoutFields(), fields: { - ...getCommonQueryBuilderFields(SearchIndex.TABLE), + ...getCommonQueryBuilderFields(SearchIndex.TABLE, tierOptions), ...getServiceQueryBuilderFields(SearchIndex.TABLE), ...tableQueryBuilderFields, }, @@ -567,7 +586,7 @@ export const getQbConfigs: (searchIndex: SearchIndex) => BasicConfig = ( return { ...getInitialConfigWithoutFields(), fields: { - ...getCommonQueryBuilderFields(SearchIndex.TOPIC), + ...getCommonQueryBuilderFields(SearchIndex.TOPIC, tierOptions), ...getServiceQueryBuilderFields(SearchIndex.TOPIC), }, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx index 0a1f350cbca6..093cf940648c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx @@ -16,7 +16,10 @@ import { Button, Checkbox, MenuProps, Space, Typography } from 'antd'; import i18next from 'i18next'; import { isArray, isEmpty } from 'lodash'; import React from 'react'; -import { RenderSettings } from 'react-awesome-query-builder'; +import { + AsyncFetchListValues, + RenderSettings, +} from 'react-awesome-query-builder'; import ProfilePicture from '../components/common/ProfilePicture/ProfilePicture'; import { AssetsOfEntity } from '../components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface'; import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface'; @@ -38,6 +41,7 @@ import { TableSearchSource, TopicSearchSource, } from '../interface/search.interface'; +import { getTags } from '../rest/tagAPI'; import { getCountBadge } from '../utils/CommonUtils'; import { getEntityName } from './EntityUtils'; import searchClassBase from './SearchClassBase'; @@ -393,3 +397,20 @@ export const getOptionsFromAggregationBucket = (buckets: Bucket[]) => { count: option.doc_count ?? 0, })); }; + +export const getTierOptions: () => Promise = async () => { + try { + const { data: tiers } = await getTags({ + parent: 'Tier', + }); + + const tierFields = tiers.map((tier) => ({ + title: tier.fullyQualifiedName, // tier.name, + value: tier.fullyQualifiedName, + })); + + return tierFields; + } catch (error) { + return []; + } +}; From 98a14482471cf7519ebe4173453e10b585a9a32e Mon Sep 17 00:00:00 2001 From: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Date: Tue, 23 Jan 2024 18:00:30 +0530 Subject: [PATCH 12/49] fix(minor): update skip icon for executions (#14809) --- .../src/main/resources/ui/src/assets/svg/skipped-badge.svg | 5 ++++- .../src/main/resources/ui/src/utils/PipelineDetailsUtils.ts | 2 +- .../src/main/resources/ui/src/utils/executionUtils.tsx | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/skipped-badge.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/skipped-badge.svg index c1e3d0ee6484..838a367e35bd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/skipped-badge.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/skipped-badge.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/PipelineDetailsUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/PipelineDetailsUtils.ts index 20c5c2407631..f46bceaedede 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/PipelineDetailsUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/PipelineDetailsUtils.ts @@ -37,7 +37,7 @@ export const getStatusBadgeIcon = (status?: StatusType) => { return Icons.FAIL_BADGE; case StatusType.Pending: - return Icons.PENDING_BADGE; + return Icons.SKIPPED_BADGE; case StatusType.Skipped: return Icons.SKIPPED_BADGE; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/executionUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/executionUtils.tsx index ae5130c3c64c..b58df1cfafe0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/executionUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/executionUtils.tsx @@ -13,7 +13,7 @@ import { Col, Row, Space, Tooltip } from 'antd'; import { DataNode } from 'antd/lib/tree'; -import { groupBy, isUndefined, map, toLower, uniqueId } from 'lodash'; +import { groupBy, isUndefined, map, toLower } from 'lodash'; import React, { ReactNode } from 'react'; import { MenuOptions } from '../constants/execution.constants'; import { @@ -188,12 +188,12 @@ export const getTreeData = ( const viewElements = map(viewData, (value, key) => ({ key, value: ( - +
{value.map((status) => ( From a6f6fff6dc03f56854ead0aef1d785dd1c845788 Mon Sep 17 00:00:00 2001 From: Carlo Q Date: Tue, 23 Jan 2024 04:32:23 -0800 Subject: [PATCH 13/49] Fixes #14803: ignore capitalization when confirming deletes (#14804) * ignore case when confirming deletes * Test confirmation of deletes works when case differs Added test case for 'delete' as the confirmation text. --- .../DeleteWidget/DeleteWidgetModal.test.tsx | 18 ++++++++++++++++++ .../common/DeleteWidget/DeleteWidgetModal.tsx | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.test.tsx index ea3402f9c2e8..5a4b09c35cfb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.test.tsx @@ -80,6 +80,24 @@ describe('Test DeleteWidgetV1 Component', () => { }); }); + it('Delete click should work properly regardless of capitalization', async () => { + await act(async () => { + render(); + + const inputBox = await screen.findByTestId('confirmation-text-input'); + const confirmButton = await screen.findByTestId('confirm-button'); + const hardDelete = await screen.findByTestId('hard-delete'); + + userEvent.click(hardDelete); + + userEvent.type(inputBox, 'delete'); + + expect(confirmButton).not.toBeDisabled(); + + userEvent.click(confirmButton); + }); + }); + it('Discard click should work properly', async () => { await act(async () => { render(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx index 14907f68a1fe..d7f3e97ff6e3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx @@ -124,7 +124,7 @@ const DeleteWidgetModal = ({ const isDeleteTextPresent = useMemo(() => { return ( - deleteConfirmationText === DELETE_CONFIRMATION_TEXT && + deleteConfirmationText.toLowerCase() === DELETE_CONFIRMATION_TEXT.toLowerCase() && (deletionType === DeleteType.SOFT_DELETE || deletionType === DeleteType.HARD_DELETE) ); From 6fa3eb4f30dba7d7c998bf46eaf0da21529701cb Mon Sep 17 00:00:00 2001 From: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com> Date: Tue, 23 Jan 2024 18:23:23 +0530 Subject: [PATCH 14/49] minor(config): update openmetadata-ui code reviewers (#14823) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5d9b58e271bd..fcaa86f28b4a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,7 @@ # Individual scopes # Review from UI owners for changes around UI code -/openmetadata-ui/src/main/resources/ui/ @harshach @karanh37 @chirag-madlani +/openmetadata-ui/src/main/resources/ui/ @harshach @karanh37 @chirag-madlani @Sachin-chaurasiya # Review from Backend owners for changes around Backend code /openmetadata-service/ @open-metadata/backend From 0b80504f04ff505dc0415d5a562d60af00c580ad Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Tue, 23 Jan 2024 09:58:17 -0800 Subject: [PATCH 15/49] Add Tests --- .../native/1.3.0/mysql/schemaChanges.sql | 1 + .../native/1.3.0/postgres/schemaChanges.sql | 1 + .../service/jdbi3/CollectionDAO.java | 11 ++-- .../service/jdbi3/SuggestionRepository.java | 4 +- .../resources/feeds/SuggestionsResource.java | 3 + .../feeds/SuggestionResourceTest.java | 56 ++++++++++++++++++- 6 files changed, 70 insertions(+), 6 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql index 25697b97178c..343a4ad27cb5 100644 --- a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql @@ -171,6 +171,7 @@ DELETE FROM consumers_dlq; CREATE TABLE IF NOT EXISTS suggestions ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL COLLATE ascii_bin, entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.type') NOT NULL, json JSON NOT NULL, diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index efc8aefe2d86..34bdc7d99804 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -183,6 +183,7 @@ DELETE FROM consumers_dlq; CREATE TABLE IF NOT EXISTS suggestions ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fqnHash VARCHAR(256) NOT NULL, entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.type') NOT NULL, json JSON NOT NULL, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index d49719945c25..7ccd9c95bf34 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4320,12 +4320,12 @@ int listCount( interface SuggestionDAO { @ConnectionAwareSqlUpdate( - value = "INSERT INTO suggestions(json) VALUES (:json)", + value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json)", connectionType = MYSQL) @ConnectionAwareSqlUpdate( - value = "INSERT INTO suggestions(json) VALUES (:json :: jsonb)", + value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json :: jsonb)", connectionType = POSTGRES) - void insert(@Bind("json") String json); + void insert(@BindFQN("fqnHash") String fullyQualifiedName, @Bind("json") String json); @ConnectionAwareSqlUpdate( value = "UPDATE suggestions SET json = :json where id = :id", @@ -4347,7 +4347,10 @@ interface SuggestionDAO { @SqlUpdate("DELETE FROM suggestions WHERE id = :id") void delete(@BindUUID("id") UUID id); - @SqlQuery("SELECT json FROM thread_entity ORDER BY createdAt DESC LIMIT :limit") + @SqlUpdate("DELETE FROM suggestions WHERE fqnHash = :fqnHash") + void delete(@BindUUID("fqnHash") String fullyQualifiedName); + + @SqlQuery("SELECT json FROM suggestions ORDER BY createdAt DESC LIMIT :limit") List list(@Bind("limit") int limit, @Define("condition") String condition); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java index 621b6e985490..05e9ab5c5a64 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -66,7 +66,9 @@ public Suggestion update(Suggestion suggestion, String userName) { @Transaction public void store(Suggestion suggestion) { // Insert a new Suggestion - dao.suggestionDAO().insert(JsonUtils.pojoToJson(suggestion)); + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + dao.suggestionDAO().insert(entityLink.getEntityFQN(), JsonUtils.pojoToJson(suggestion)); } @Transaction diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index e4a1e0692db0..0c286d61dab7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -316,6 +316,9 @@ private Suggestion getSuggestion(SecurityContext securityContext, CreateSuggesti } private void validate(CreateSuggestion suggestion) { + if (suggestion.getEntityLink() == null) { + throw new WebApplicationException("Suggestion's entityLink cannot be null."); + } MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); Entity.getEntityReferenceByName( diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java index e4ea1172247c..affd5bcfb99d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java @@ -1,8 +1,14 @@ package org.openmetadata.service.resources.feeds; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.resources.EntityResourceTest.C1; import static org.openmetadata.service.security.SecurityUtil.authHeaders; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; +import static org.openmetadata.service.util.TestUtils.NON_EXISTENT_ENTITY; +import static org.openmetadata.service.util.TestUtils.assertResponse; +import static org.openmetadata.service.util.TestUtils.assertResponseContains; import java.io.IOException; import java.net.URISyntaxException; @@ -20,6 +26,7 @@ import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.feed.CreateSuggestion; +import org.openmetadata.schema.api.feed.CreateThread; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Suggestion; @@ -104,7 +111,54 @@ public void setup(TestInfo test) throws IOException, URISyntaxException { } @Test - void post_validThreadAndList_200(TestInfo test) throws IOException { + void post_suggestionWithoutEntityLink_4xx() { + // Create thread without addressed to entity in the request + CreateSuggestion create = create().withEntityLink(null); + assertResponse( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, "Suggestion's entityLink cannot be null."); + } + + @Test + void post_suggestionWithInvalidAbout_4xx() { + // Create Suggestion without addressed to entity in the request + CreateSuggestion create = create().withEntityLink("<>"); // Invalid EntityLink + + String failureReason = "[entityLink must match \"(?U)^<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#%]+>$\"]"; + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + + create.withEntityLink("<#E::>"); // Invalid EntityLink - missing entityType and entityId + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + + create.withEntityLink("<#E::table::>"); // Invalid EntityLink - missing entityId + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + + create.withEntityLink("<#E::table::tableName"); // Invalid EntityLink - missing closing bracket ">" + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); + } + + @Test + void post_suggestionWithoutDescriptionOrTags_4xx() { + CreateSuggestion create = create().withDescription(null); + assertResponseContains( + () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, "Suggestion's description cannot be empty"); + } + + @Test + void post_feedWithNonExistentEntity_404() { + CreateSuggestion create = create().withEntityLink("<#E::table::invalidTableName>"); + assertResponse( + () -> createSuggestion(create, USER_AUTH_HEADERS), + NOT_FOUND, + entityNotFound(Entity.TABLE, "invalidTableName")); + } + + + @Test + void post_validSuggestionAndList_200(TestInfo test) throws IOException { CreateSuggestion create = create(); Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); From 91889370d68fab0d44dfd2600332fa76ae7d11d1 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Tue, 23 Jan 2024 23:27:59 -0800 Subject: [PATCH 16/49] Add list/accept/reject apis --- .../native/1.3.0/mysql/schemaChanges.sql | 4 +- .../exception/CatalogExceptionMessage.java | 5 + .../service/jdbi3/CollectionDAO.java | 60 +++- .../service/jdbi3/EntityRepository.java | 7 +- .../service/jdbi3/SuggestionFilter.java | 52 ++++ .../service/jdbi3/SuggestionRepository.java | 138 +++++++-- .../service/jdbi3/TableRepository.java | 22 ++ .../resources/feeds/SuggestionsResource.java | 122 ++++++-- .../feeds/SuggestionResourceTest.java | 284 +++++++++++++++++- .../sdk/exception/SuggestionException.java | 40 +++ 10 files changed, 669 insertions(+), 65 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java create mode 100644 openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java diff --git a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql index 343a4ad27cb5..33466c0cb025 100644 --- a/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/mysql/schemaChanges.sql @@ -173,10 +173,10 @@ CREATE TABLE IF NOT EXISTS suggestions ( id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, fqnHash VARCHAR(256) NOT NULL COLLATE ascii_bin, entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, - suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.type') NOT NULL, + suggestionType VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json ->> '$.type')) NOT NULL, json JSON NOT NULL, updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, - status VARCHAR(256) GENERATED ALWAYS AS (json -> '$.status') NOT NULL, + status VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json -> '$.status')) NOT NULL, PRIMARY KEY (id) ); \ No newline at end of file diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index c6f69e00af9e..8939e652ab7b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -208,6 +208,11 @@ public static String taskOperationNotAllowed(String user, String operations) { "Principal: CatalogPrincipal{name='%s'} operations %s not allowed", user, operations); } + public static String suggestionOperationNotAllowed(String user, String operations) { + return String.format( + "Principal: CatalogPrincipal{name='%s'} operations %s not allowed", user, operations); + } + public static String entityIsNotEmpty(String entityType) { return String.format("%s is not empty", entityType); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 7ccd9c95bf34..6e5452aff23d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -3528,8 +3528,7 @@ default List listAfterTsOrder(ListFilter filter, int limit, Inte } default int countOfTestCases(List testCaseIds) { - return countOfTestCases( - getTableName(), testCaseIds.stream().map(Object::toString).collect(Collectors.toList())); + return countOfTestCases(getTableName(), testCaseIds.stream().map(Object::toString).toList()); } @SqlQuery("SELECT count(*) FROM WHERE id IN ()") @@ -4319,6 +4318,10 @@ int listCount( } interface SuggestionDAO { + default String getTableName() { + return "suggestions"; + } + @ConnectionAwareSqlUpdate( value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json)", connectionType = MYSQL) @@ -4338,19 +4341,56 @@ interface SuggestionDAO { @SqlQuery("SELECT json FROM suggestions WHERE id = :id") String findById(@BindUUID("id") UUID id); - @SqlQuery("SELECT json FROM suggestions ORDER BY createdAt DESC") - List list(); - - @SqlQuery("SELECT count(id) FROM suggestions ") - int listCount(@Define("condition") String condition); - @SqlUpdate("DELETE FROM suggestions WHERE id = :id") void delete(@BindUUID("id") UUID id); @SqlUpdate("DELETE FROM suggestions WHERE fqnHash = :fqnHash") - void delete(@BindUUID("fqnHash") String fullyQualifiedName); + void deleteByFQN(@BindUUID("fqnHash") String fullyQualifiedName); - @SqlQuery("SELECT json FROM suggestions ORDER BY createdAt DESC LIMIT :limit") + @SqlQuery("SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit") List list(@Bind("limit") int limit, @Define("condition") String condition); + + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM suggestions ", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM suggestions ", + connectionType = POSTGRES) + int listCount( + @Define("mysqlCond") String mysqlCond, @Define("postgresCond") String postgresCond); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT updatedAt, json FROM suggestions " + + "ORDER BY updatedAt DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY updatedAt", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT updatedAt, json FROM suggestions " + + "ORDER BY updatedAt DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY updatedAt", + connectionType = POSTGRES) + List listBefore( + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("before") String before); + + @ConnectionAwareSqlQuery( + value = "SELECT json FROM suggestions ORDER BY updatedAt LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM suggestions ORDER BY updatedAt LIMIT :limit", + connectionType = POSTGRES) + List listAfter( + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("after") String after); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 0cdd0fd9335d..2ec5bf3efbf0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1678,7 +1678,7 @@ protected List getExperts(T entity) { } public final EntityReference getOwner(EntityReference ref) { - return !supportsOwner ? null : Entity.getEntityReferenceById(ref.getType(), ref.getId(), ALL); + return !supportsOwner ? null : getFromEntityRef(ref.getId(), Relationship.OWNS, null, false); } public final void inheritDomain(T entity, Fields fields, EntityInterface parent) { @@ -1934,6 +1934,11 @@ public SuggestionRepository.SuggestionWorkflow getSuggestionWorkflow(Suggestion return new SuggestionRepository.SuggestionWorkflow(suggestion); } + public EntityInterface applySuggestion( + EntityInterface entity, String childFQN, Suggestion suggestion) { + return entity; + } + public final void validateTaskThread(ThreadContext threadContext) { ThreadType threadType = threadContext.getThread().getType(); if (threadType != ThreadType.Task) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java new file mode 100644 index 000000000000..3bfd327433c2 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionFilter.java @@ -0,0 +1,52 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.util.RestUtil.decodeCursor; + +import java.util.UUID; +import lombok.Builder; +import lombok.Getter; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.service.util.FullyQualifiedName; + +@Getter +@Builder +public class SuggestionFilter { + private SuggestionType suggestionType; + private SuggestionStatus suggestionStatus; + private UUID createdBy; + private String entityFQN; + private SuggestionRepository.PaginationType paginationType; + private String before; + private String after; + + public String getCondition(boolean includePagination) { + StringBuilder condition = new StringBuilder(); + condition.append("WHERE TRUE "); + if (suggestionType != null) { + condition.append(String.format(" AND type = '%s' ", suggestionType.value())); + } + if (suggestionStatus != null) { + condition.append(String.format(" AND status = '%s' ", suggestionStatus.value())); + } + if (entityFQN != null) { + condition.append( + String.format(" AND fqnHash = '%s' ", FullyQualifiedName.buildHash(entityFQN))); + } + if (createdBy != null) { + condition.append( + String.format( + " AND id in (select toId from entity_relationship where fromId = '%s') ", createdBy)); + } + if (paginationType != null && includePagination) { + String paginationCondition = + paginationType == SuggestionRepository.PaginationType.BEFORE + ? String.format(" AND updatedAt > %s ", Long.parseLong(decodeCursor(before))) + : String.format( + " AND updatedAt < %s ", + after != null ? Long.parseLong(decodeCursor(after)) : Long.MAX_VALUE); + condition.append(paginationCondition); + } + return condition.toString(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java index 05e9ab5c5a64..a4dfcdf193e6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -1,15 +1,17 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.schema.type.EventType.ENTITY_DELETED; import static org.openmetadata.schema.type.EventType.SUGGESTION_ACCEPTED; +import static org.openmetadata.schema.type.EventType.SUGGESTION_DELETED; import static org.openmetadata.schema.type.EventType.SUGGESTION_REJECTED; import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.schema.type.Relationship.CREATED; import static org.openmetadata.schema.type.Relationship.IS_ABOUT; +import static org.openmetadata.service.Entity.TEAM; import static org.openmetadata.service.Entity.USER; import static org.openmetadata.service.jdbi3.UserRepository.TEAMS_FIELD; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -23,25 +25,34 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.SuggestionStatus; import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.ResourceRegistry; import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.SuggestionsResource; import org.openmetadata.service.security.AuthorizationException; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.ResultList; @Slf4j @Repository public class SuggestionRepository { private final CollectionDAO dao; + public enum PaginationType { + BEFORE, + AFTER + } + public SuggestionRepository() { this.dao = Entity.getCollectionDAO(); Entity.setSuggestionRepository(this); @@ -75,7 +86,7 @@ public void store(Suggestion suggestion) { public void storeRelationships(Suggestion suggestion) { MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); - // Add relationship User -- created --> Thread relationship + // Add relationship User -- created --> Suggestion relationship dao.relationshipDAO() .insert( suggestion.getCreatedBy().getId(), @@ -84,7 +95,7 @@ public void storeRelationships(Suggestion suggestion) { Entity.SUGGESTION, CREATED.ordinal()); - // Add field relationship for data asset - Thread -- isAbout ---> entity/entityField + // Add field relationship for data asset - Suggestion -- entityLink ---> entity/entityField dao.fieldRelationshipDAO() .insert( suggestion.getId().toString(), // from FQN @@ -106,7 +117,15 @@ public RestUtil.DeleteResponse deleteSuggestion( Suggestion suggestion, String deletedByUser) { deleteSuggestionInternal(suggestion.getId()); LOG.debug("{} deleted suggestion with id {}", deletedByUser, suggestion.getId()); - return new RestUtil.DeleteResponse<>(suggestion, ENTITY_DELETED); + return new RestUtil.DeleteResponse<>(suggestion, SUGGESTION_DELETED); + } + + @Transaction + public RestUtil.DeleteResponse deleteSuggestionsForAnEntity( + EntityInterface entity, String deletedByUser) { + deleteSuggestionInternalForAnEntity(entity); + LOG.debug("{} deleted suggestions for the entity id {}", deletedByUser, entity.getId()); + return new RestUtil.DeleteResponse<>(entity, SUGGESTION_DELETED); } @Transaction @@ -121,23 +140,44 @@ public void deleteSuggestionInternal(UUID id) { dao.suggestionDAO().delete(id); } + @Transaction + public void deleteSuggestionInternalForAnEntity(EntityInterface entity) { + // Delete all the field relationships to other entities + dao.fieldRelationshipDAO().deleteAllByPrefix(entity.getId().toString()); + + // Finally, delete the suggestion + dao.suggestionDAO().deleteByFQN(entity.getFullyQualifiedName()); + } + @Getter public static class SuggestionWorkflow { protected final Suggestion suggestion; + protected final MessageParser.EntityLink entityLink; SuggestionWorkflow(Suggestion suggestion) { this.suggestion = suggestion; + this.entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); } - public EntityInterface acceptSuggestions(EntityInterface entityInterface) { - if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { - entityInterface.setTags(suggestion.getTagLabels()); - return entityInterface; - } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { - entityInterface.setDescription(suggestion.getDescription()); + public EntityInterface acceptSuggestions( + EntityRepository repository, EntityInterface entityInterface) { + if (entityLink.getFieldName() != null) { + entityInterface = + repository.applySuggestion( + entityInterface, entityLink.getFullyQualifiedFieldValue(), suggestion); return entityInterface; } else { - throw new WebApplicationException("Invalid suggestion Type"); + if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { + List tags = new ArrayList<>(entityInterface.getTags()); + tags.addAll(suggestion.getTagLabels()); + entityInterface.setTags(tags); + return entityInterface; + } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { + entityInterface.setDescription(suggestion.getDescription()); + return entityInterface; + } else { + throw new WebApplicationException("Invalid suggestion Type"); + } } } } @@ -158,10 +198,10 @@ protected void acceptSuggestion(Suggestion suggestion, String user) { entityLink, suggestion.getType() == SuggestionType.SuggestTagLabel ? "tags" : "", ALL); String origJson = JsonUtils.pojoToJson(entity); SuggestionWorkflow suggestionWorkflow = getSuggestionWorkflow(suggestion); - EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestions(entity); + EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); + EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestions(repository, entity); String updatedEntityJson = JsonUtils.pojoToJson(updatedEntity); JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); - EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); repository.patch(null, entity.getId(), user, patch); suggestion.setStatus(SuggestionStatus.Accepted); update(suggestion, user); @@ -182,17 +222,25 @@ public void checkPermissionsForAcceptOrRejectSuggestion( MessageParser.EntityLink about = MessageParser.EntityLink.parse(suggestion.getEntityLink()); EntityReference aboutRef = EntityUtil.validateEntityLink(about); EntityReference ownerRef = Entity.getOwner(aboutRef); - User owner = - Entity.getEntityByName(USER, ownerRef.getFullyQualifiedName(), TEAMS_FIELD, NON_DELETED); + List ownerTeamNames = new ArrayList<>(); + try { + User owner = + Entity.getEntityByName(USER, ownerRef.getFullyQualifiedName(), TEAMS_FIELD, NON_DELETED); + ownerTeamNames = + owner.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + } catch (EntityNotFoundException e) { + Team owner = Entity.getEntityByName(TEAM, ownerRef.getFullyQualifiedName(), "", NON_DELETED); + ownerTeamNames.add(owner.getFullyQualifiedName()); + } + List userTeamNames = user.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); - List ownerTeamNames = - owner.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + if (Boolean.FALSE.equals(user.getIsAdmin()) - && !owner.getName().equals(userName) + && !ownerRef.getName().equals(userName) && Collections.disjoint(userTeamNames, ownerTeamNames)) { throw new AuthorizationException( - CatalogExceptionMessage.taskOperationNotAllowed(userName, status.value())); + CatalogExceptionMessage.suggestionOperationNotAllowed(userName, status.value())); } } @@ -202,4 +250,56 @@ public SuggestionWorkflow getSuggestionWorkflow(Suggestion suggestion) { EntityRepository repository = Entity.getEntityRepository(entityLink.getEntityType()); return repository.getSuggestionWorkflow(suggestion); } + + public int listCount(SuggestionFilter filter) { + String mySqlCondition = filter.getCondition(false); + String postgresCondition = filter.getCondition(false); + return dao.suggestionDAO().listCount(mySqlCondition, postgresCondition); + } + + public ResultList listBefore(SuggestionFilter filter, int limit, String before) { + int total = listCount(filter); + String mySqlCondition = filter.getCondition(true); + String postgresCondition = filter.getCondition(true); + List jsons = + dao.suggestionDAO().listBefore(mySqlCondition, postgresCondition, limit, before); + List suggestions = getSuggestionList(jsons); + String beforeCursor = null; + String afterCursor; + if (suggestions.size() > limit) { + suggestions.remove(0); + beforeCursor = suggestions.get(0).getUpdatedAt().toString(); + } + afterCursor = + !suggestions.isEmpty() + ? suggestions.get(suggestions.size() - 1).getUpdatedAt().toString() + : null; + return new ResultList<>(suggestions, beforeCursor, afterCursor, total); + } + + public ResultList listAfter(SuggestionFilter filter, int limit, String after) { + int total = listCount(filter); + String mySqlCondition = filter.getCondition(true); + String postgresCondition = filter.getCondition(true); + List jsons = + dao.suggestionDAO().listAfter(mySqlCondition, postgresCondition, limit, after); + List suggestions = getSuggestionList(jsons); + String beforeCursor; + String afterCursor = null; + beforeCursor = after == null ? null : suggestions.get(0).getUpdatedAt().toString(); + if (suggestions.size() > limit) { + suggestions.remove(limit); + afterCursor = suggestions.get(limit - 1).getUpdatedAt().toString(); + } + return new ResultList<>(suggestions, beforeCursor, afterCursor, total); + } + + private List getSuggestionList(List jsons) { + List suggestions = new ArrayList<>(); + for (String json : jsons) { + Suggestion suggestion = JsonUtils.readValue(json, Suggestion.class); + suggestions.add(suggestion); + } + return suggestions; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 56a30143bf82..4f5493d1c716 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -45,6 +45,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.ws.rs.WebApplicationException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; @@ -58,6 +59,7 @@ import org.openmetadata.schema.api.feed.ResolveTask; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.tests.CustomMetric; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.Column; @@ -70,6 +72,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.JoinedWith; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.SuggestionType; import org.openmetadata.schema.type.SystemProfile; import org.openmetadata.schema.type.TableConstraint; import org.openmetadata.schema.type.TableData; @@ -737,6 +740,25 @@ public TaskWorkflow getTaskWorkflow(ThreadContext threadContext) { return super.getTaskWorkflow(threadContext); } + @Override + public Table applySuggestion(EntityInterface entity, String columnFQN, Suggestion suggestion) { + Table table = Entity.getEntity(TABLE, entity.getId(), "columns,tags", ALL); + for (Column col : table.getColumns()) { + if (col.getFullyQualifiedName().equals(columnFQN)) { + if (suggestion.getType().equals(SuggestionType.SuggestTagLabel)) { + List tags = new ArrayList<>(col.getTags()); + tags.addAll(suggestion.getTagLabels()); + col.setTags(tags); + } else if (suggestion.getType().equals(SuggestionType.SuggestDescription)) { + col.setDescription(suggestion.getDescription()); + } else { + throw new WebApplicationException("Invalid suggestion Type"); + } + } + } + return table; + } + @Override public String exportToCsv(String name, String user) throws IOException { // Validate table diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index 0c286d61dab7..f0d9a4aa269f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -40,21 +40,22 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.feed.CreateSuggestion; import org.openmetadata.schema.entity.feed.Suggestion; -import org.openmetadata.schema.entity.feed.Thread; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.SuggestionStatus; import org.openmetadata.schema.type.SuggestionType; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.sdk.exception.SuggestionException; import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.SuggestionFilter; import org.openmetadata.service.jdbi3.SuggestionRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.tags.TagLabelUtil; @@ -78,6 +79,7 @@ public class SuggestionsResource { public static final String COLLECTION_PATH = "/v1/suggestions/"; private final SuggestionRepository dao; private final Authorizer authorizer; + private final String INVALID_SUGGESTION_REQUEST = "INVALID_SUGGESTION_REQUEST"; public static void addHref(UriInfo uriInfo, List suggestions) { if (uriInfo != null) { @@ -116,7 +118,7 @@ public static class SuggestionList extends ResultList { mediaType = "application/json", schema = @Schema(implementation = SuggestionList.class))) }) - public ResultList list( + public ResultList list( @Context UriInfo uriInfo, @Parameter( description = @@ -136,13 +138,9 @@ public ResultList list( schema = @Schema(type = "string")) @QueryParam("after") String after, - @Parameter( - description = - "Filter threads by entity link of entity about which this thread is created", - schema = - @Schema(type = "string", example = "")) - @QueryParam("entityLink") - String entityLink, + @Parameter(description = "Filter suggestions by entityFQN", schema = @Schema(type = "string")) + @QueryParam("entityFQN") + String entityFQN, @Parameter( description = "Filter threads by user id or bot id. This filter requires a 'filterType' query param.", @@ -152,11 +150,30 @@ public ResultList list( @Parameter( description = "Filter threads by whether they are accepted or rejected. By default status is OPEN.") - @DefaultValue("OPEN") + @DefaultValue("Open") @QueryParam("status") String status) { RestUtil.validateCursors(before, after); - return null; + SuggestionFilter filter = + SuggestionFilter.builder() + .suggestionStatus(SuggestionStatus.valueOf(status)) + .entityFQN(entityFQN) + .createdBy(userId) + .paginationType( + before != null + ? SuggestionRepository.PaginationType.BEFORE + : SuggestionRepository.PaginationType.AFTER) + .before(before) + .after(after) + .build(); + ResultList suggestions; + if (before != null) { + suggestions = dao.listAfter(filter, limitParam, after); + } else { + suggestions = dao.listBefore(filter, limitParam, before); + } + addHref(uriInfo, suggestions.getData()); + return suggestions; } @GET @@ -209,11 +226,40 @@ public Response acceptSuggestion( UUID id) { Suggestion suggestion = dao.get(id); dao.checkPermissionsForAcceptOrRejectSuggestion( - suggestion, suggestion.getStatus(), securityContext); + suggestion, SuggestionStatus.Accepted, securityContext); return dao.acceptSuggestion(uriInfo, suggestion, securityContext.getUserPrincipal().getName()) .toResponse(); } + @PUT + @Path("/{id}/reject") + @Operation( + operationId = "rejectSuggestion", + summary = "Reject a Suggestion", + description = "Close a Suggestion without making any changes to the entity.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Suggestion.", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Suggestion.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response rejectSuggestion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the suggestion", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + Suggestion suggestion = dao.get(id); + dao.checkPermissionsForAcceptOrRejectSuggestion( + suggestion, SuggestionStatus.Rejected, securityContext); + return dao.rejectSuggestion(uriInfo, suggestion, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + @PUT @Path("/{id}") @Operation( @@ -299,6 +345,37 @@ public Response deleteSuggestion( .toResponse(); } + @DELETE + @Path("/{entityType}/name/{entityFQN}") + @Operation( + operationId = "deleteSuggestions", + summary = "Delete a Suggestions by entityFQN", + description = "Delete an existing Suggestions and all its relationships.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "404", description = "thread with {threadId} is not found"), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response deleteSuggestions( + @Context SecurityContext securityContext, + @Parameter(description = "entity type", schema = @Schema(type = "string")) + @PathParam("entityType") + String entityType, + @Parameter(description = "fullyQualifiedName of entity", schema = @Schema(type = "string")) + @PathParam("entityFQN") + String entityFQN) { + // validate and get the thread + EntityInterface entity = + Entity.getEntityByName(entityType, entityFQN, "owner", Include.NON_DELETED); + // delete thread only if the admin/bot/author tries to delete it + OperationContext operationContext = + new OperationContext(Entity.SUGGESTION, MetadataOperation.DELETE); + ResourceContextInterface resourceContext = new PostResourceContext(entity.getOwner().getName()); + authorizer.authorize(securityContext, operationContext, resourceContext); + return dao.deleteSuggestionsForAnEntity(entity, securityContext.getUserPrincipal().getName()) + .toResponse(); + } + private Suggestion getSuggestion(SecurityContext securityContext, CreateSuggestion create) { validate(create); return new Suggestion() @@ -317,26 +394,37 @@ private Suggestion getSuggestion(SecurityContext securityContext, CreateSuggesti private void validate(CreateSuggestion suggestion) { if (suggestion.getEntityLink() == null) { - throw new WebApplicationException("Suggestion's entityLink cannot be null."); + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's entityLink cannot be null."); } MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); Entity.getEntityReferenceByName( entityLink.getEntityType(), entityLink.getEntityFQN(), Include.NON_DELETED); + if (suggestion.getType() == SuggestionType.SuggestDescription) { if (suggestion.getDescription() == null || suggestion.getDescription().isEmpty()) { - throw new WebApplicationException("Suggestion's description cannot be empty."); + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's description cannot be empty."); } } else if (suggestion.getType() == SuggestionType.SuggestTagLabel) { if (suggestion.getTagLabels().isEmpty()) { - throw new WebApplicationException("Suggestion's tag label's cannot be empty."); + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's tag label's cannot be empty."); } else { for (TagLabel label : listOrEmpty(suggestion.getTagLabels())) { TagLabelUtil.applyTagCommonFields(label); } } } else { - throw new WebApplicationException("Invalid Suggestion Type."); + throw new SuggestionException( + Response.Status.BAD_REQUEST, INVALID_SUGGESTION_REQUEST, "Invalid Suggestion Type."); } } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java index affd5bcfb99d..9e011b6070ed 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java @@ -1,32 +1,42 @@ package org.openmetadata.service.resources.feeds; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.resources.EntityResourceTest.C1; +import static org.openmetadata.service.resources.EntityResourceTest.C2; +import static org.openmetadata.service.resources.EntityResourceTest.PERSONAL_DATA_TAG_LABEL; +import static org.openmetadata.service.resources.EntityResourceTest.PII_SENSITIVE_TAG_LABEL; import static org.openmetadata.service.security.SecurityUtil.authHeaders; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; -import static org.openmetadata.service.util.TestUtils.NON_EXISTENT_ENTITY; import static org.openmetadata.service.util.TestUtils.assertResponse; import static org.openmetadata.service.util.TestUtils.assertResponseContains; import java.io.IOException; import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.UUID; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.feed.CreateSuggestion; -import org.openmetadata.schema.api.feed.CreateThread; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Suggestion; @@ -35,12 +45,16 @@ import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.SuggestionStatus; import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; +import org.openmetadata.service.security.CatalogOpenIdAuthorizationRequestFilter; import org.openmetadata.service.util.TestUtils; @Slf4j @@ -50,8 +64,9 @@ public class SuggestionResourceTest extends OpenMetadataApplicationTest { public static Table TABLE; public static Table TABLE2; public static String TABLE_LINK; - public static String TABLE_COLUMN_LINK; - public static String TABLE_DESCRIPTION_LINK; + public static String TABLE2_LINK; + public static String TABLE_COLUMN1_LINK; + public static String TABLE_COLUMN2_LINK; public static List COLUMNS; public static User USER; public static String USER_LINK; @@ -96,11 +111,11 @@ public void setup(TestInfo test) throws IOException, URISyntaxException { Collections.singletonList( new Column().withName("column1").withDataType(ColumnDataType.BIGINT)); TABLE_LINK = String.format("<#E::table::%s>", TABLE.getFullyQualifiedName()); - TABLE_COLUMN_LINK = - String.format( - "<#E::table::%s::columns::" + C1 + "::description>", TABLE.getFullyQualifiedName()); - TABLE_DESCRIPTION_LINK = - String.format("<#E::table::%s::description>", TABLE.getFullyQualifiedName()); + TABLE2_LINK = String.format("<#E::table::%s>", TABLE2.getFullyQualifiedName()); + TABLE_COLUMN1_LINK = + String.format("<#E::table::%s::columns::" + C1 + ">", TABLE.getFullyQualifiedName()); + TABLE_COLUMN2_LINK = + String.format("<#E::table::%s::columns::" + C2 + ">", TABLE.getFullyQualifiedName()); USER = TableResourceTest.USER1; USER_LINK = String.format("<#E::user::%s>", USER.getFullyQualifiedName()); @@ -115,7 +130,9 @@ void post_suggestionWithoutEntityLink_4xx() { // Create thread without addressed to entity in the request CreateSuggestion create = create().withEntityLink(null); assertResponse( - () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, "Suggestion's entityLink cannot be null."); + () -> createSuggestion(create, USER_AUTH_HEADERS), + BAD_REQUEST, + "Suggestion's entityLink cannot be null."); } @Test @@ -123,7 +140,8 @@ void post_suggestionWithInvalidAbout_4xx() { // Create Suggestion without addressed to entity in the request CreateSuggestion create = create().withEntityLink("<>"); // Invalid EntityLink - String failureReason = "[entityLink must match \"(?U)^<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#%]+>$\"]"; + String failureReason = + "[entityLink must match \"(?U)^<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#%]+>$\"]"; assertResponseContains( () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); @@ -135,7 +153,8 @@ void post_suggestionWithInvalidAbout_4xx() { assertResponseContains( () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); - create.withEntityLink("<#E::table::tableName"); // Invalid EntityLink - missing closing bracket ">" + create.withEntityLink( + "<#E::table::tableName"); // Invalid EntityLink - missing closing bracket ">" assertResponseContains( () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, failureReason); } @@ -144,7 +163,9 @@ void post_suggestionWithInvalidAbout_4xx() { void post_suggestionWithoutDescriptionOrTags_4xx() { CreateSuggestion create = create().withDescription(null); assertResponseContains( - () -> createSuggestion(create, USER_AUTH_HEADERS), BAD_REQUEST, "Suggestion's description cannot be empty"); + () -> createSuggestion(create, USER_AUTH_HEADERS), + BAD_REQUEST, + "Suggestion's description cannot be empty"); } @Test @@ -156,12 +177,139 @@ void post_feedWithNonExistentEntity_404() { entityNotFound(Entity.TABLE, "invalidTableName")); } - @Test void post_validSuggestionAndList_200(TestInfo test) throws IOException { CreateSuggestion create = create(); Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + create = create().withEntityLink(TABLE_LINK); + int suggestionCount = 1; + for (int i = 0; i < 10; i++) { + createAndCheck(create, USER_AUTH_HEADERS); + // List all the threads and make sure the number of threads increased by 1 + assertEquals( + ++suggestionCount, + listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + } + create = create().withEntityLink(TABLE_COLUMN1_LINK); + createAndCheck(create, USER2_AUTH_HEADERS); + create = create().withEntityLink(TABLE_COLUMN2_LINK); + createAndCheck(create, USER2_AUTH_HEADERS); + assertEquals( + suggestionCount + 2, + listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + create = create().withEntityLink(TABLE2_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + assertEquals( + suggestionCount + 2, + listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + assertEquals( + 1, + listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + assertEquals( + 2, + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + USER2.getId(), + null, + null, + null, + null) + .getPaging() + .getTotal()); + + /* deleteSuggestions("table", TABLE.getFullyQualifiedName(), USER_AUTH_HEADERS); + assertEquals( + 0, + listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS).getPaging().getTotal()); + deleteSuggestions("table", TABLE2.getFullyQualifiedName(), USER_AUTH_HEADERS); + assertEquals( + 0, + listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS).getPaging().getTotal());*/ + } + + @Test + @Order(1) + void put_acceptSuggestion_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + acceptSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + TableResourceTest tableResourceTest = new TableResourceTest(); + Table table = tableResourceTest.getEntity(TABLE.getId(), "", USER_AUTH_HEADERS); + assertEquals(suggestion.getDescription(), table.getDescription()); + suggestion = getSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + assertEquals(SuggestionStatus.Accepted, suggestion.getStatus()); + create = createTagSuggestion(); + Suggestion suggestion1 = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + assertResponse( + () -> acceptSuggestion(suggestion1.getId(), USER2_AUTH_HEADERS), + FORBIDDEN, + CatalogExceptionMessage.taskOperationNotAllowed(USER2.getName(), "Accepted")); + + acceptSuggestion(suggestion1.getId(), USER_AUTH_HEADERS); + table = tableResourceTest.getEntity(TABLE.getId(), "tags", USER_AUTH_HEADERS); + List expectedTags = new ArrayList<>(table.getTags()); + expectedTags.addAll(suggestion1.getTagLabels()); + validateAppliedTags(expectedTags, table.getTags()); + + create = createTagSuggestion().withEntityLink(TABLE_COLUMN1_LINK); + Suggestion suggestion2 = createSuggestion(create, USER_AUTH_HEADERS); + acceptSuggestion(suggestion2.getId(), USER_AUTH_HEADERS); + table = tableResourceTest.getEntity(TABLE.getId(), "columns,tags", USER_AUTH_HEADERS); + Column column = null; + for (Column col : table.getColumns()) { + if (col.getName().equals(C1)) { + column = col; + } + } + if (column != null) { + expectedTags = new ArrayList<>(column.getTags()); + expectedTags.addAll(suggestion2.getTagLabels()); + validateAppliedTags(expectedTags, column.getTags()); + } + } + + @Test + @Order(2) + void put_rejectSuggestion_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + assertEquals( + 1, + listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + rejectSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + suggestion = getSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + assertEquals(SuggestionStatus.Rejected, suggestion.getStatus()); + CreateSuggestion create1 = create().withEntityLink(TABLE2_LINK); + final Suggestion suggestion1 = createSuggestion(create1, USER2_AUTH_HEADERS); + Assertions.assertEquals(create1.getEntityLink(), suggestion1.getEntityLink()); + assertEquals( + 1, + listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); + assertResponse( + () -> rejectSuggestion(suggestion1.getId(), USER_AUTH_HEADERS), + FORBIDDEN, + CatalogExceptionMessage.taskOperationNotAllowed(USER.getName(), "Rejected")); + rejectSuggestion(suggestion1.getId(), USER2_AUTH_HEADERS); + Suggestion suggestion2 = getSuggestion(suggestion1.getId(), USER2_AUTH_HEADERS); + assertEquals(SuggestionStatus.Rejected, suggestion2.getStatus()); } public Suggestion createSuggestion(CreateSuggestion create, Map authHeaders) @@ -170,10 +318,114 @@ public Suggestion createSuggestion(CreateSuggestion create, Map } public CreateSuggestion create() { - String entityLink = String.format("<#E::%s::%s>", Entity.TABLE, TABLE.getFullyQualifiedName()); return new CreateSuggestion() .withDescription("Update description") .withType(SuggestionType.SuggestDescription) - .withEntityLink(entityLink); + .withEntityLink(TABLE_LINK); + } + + public CreateSuggestion createTagSuggestion() { + return new CreateSuggestion() + .withTagLabels(List.of(PII_SENSITIVE_TAG_LABEL, PERSONAL_DATA_TAG_LABEL)) + .withType(SuggestionType.SuggestTagLabel) + .withEntityLink(TABLE_LINK); + } + + public Suggestion getSuggestion(UUID id, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/" + id); + return TestUtils.get(target, Suggestion.class, authHeaders); + } + + public void acceptSuggestion(UUID id, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/" + id + "/accept"); + TestUtils.put(target, null, Response.Status.OK, authHeaders); + } + + public void rejectSuggestion(UUID id, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("suggestions/" + id + "/reject"); + TestUtils.put(target, null, Response.Status.OK, authHeaders); + } + + public void deleteSuggestions( + String entityType, String entityFQN, Map authHeaders) + throws HttpResponseException { + WebTarget target = + getResource("suggestions/" + entityType + "/name/" + URLEncoder.encode(entityFQN)); + TestUtils.delete(target, authHeaders); + } + + public SuggestionsResource.SuggestionList listSuggestions( + String entityFQN, + Integer limit, + Map authHeaders, + UUID userId, + String suggestionType, + String status, + String before, + String after) + throws HttpResponseException { + WebTarget target = getResource("suggestions"); + target = entityFQN != null ? target.queryParam("entityFQN", entityFQN) : target; + target = userId != null ? target.queryParam("userId", userId) : target; + target = suggestionType != null ? target.queryParam("suggestionType", suggestionType) : target; + target = status != null ? target.queryParam("status", status) : target; + target = before != null ? target.queryParam("before", before) : target; + target = after != null ? target.queryParam("after", after) : target; + target = limit != null ? target.queryParam("limit", limit) : target; + return TestUtils.get(target, SuggestionsResource.SuggestionList.class, authHeaders); + } + + public SuggestionsResource.SuggestionList listSuggestions( + String entityFQN, Integer limit, Map authHeaders) + throws HttpResponseException { + return listSuggestions(entityFQN, limit, authHeaders, null, null, null, null, null); + } + + public Suggestion createAndCheck(CreateSuggestion create, Map authHeaders) + throws HttpResponseException { + // Validate returned thread from POST + Suggestion suggestion = createSuggestion(create, authHeaders); + validateSuggestion( + suggestion, + create.getEntityLink(), + authHeaders.get(CatalogOpenIdAuthorizationRequestFilter.X_AUTH_PARAMS_EMAIL_HEADER), + create.getType(), + create.getDescription(), + create.getTagLabels()); + + // Validate returned thread again from GET + Suggestion getSuggestion = getSuggestion(suggestion.getId(), authHeaders); + validateSuggestion( + getSuggestion, + create.getEntityLink(), + authHeaders.get(CatalogOpenIdAuthorizationRequestFilter.X_AUTH_PARAMS_EMAIL_HEADER), + create.getType(), + create.getDescription(), + create.getTagLabels()); + return suggestion; + } + + private void validateSuggestion( + Suggestion suggestion, + String entityLink, + String createdBy, + SuggestionType type, + String description, + List tags) { + assertNotNull(suggestion.getId()); + assertEquals(entityLink, suggestion.getEntityLink()); + assertEquals(createdBy, suggestion.getCreatedBy().getName()); + assertEquals(type, suggestion.getType()); + assertEquals(tags, suggestion.getTagLabels()); + assertEquals(description, suggestion.getDescription()); + } + + private void validateAppliedTags(List appliedTags, List entityTags) { + for (TagLabel tagLabel : appliedTags) { + Assertions.assertTrue(entityTags.contains(tagLabel)); + } } } diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java new file mode 100644 index 000000000000..76d31abf2271 --- /dev/null +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SuggestionException.java @@ -0,0 +1,40 @@ +package org.openmetadata.sdk.exception; + +import javax.ws.rs.core.Response; + +public class SuggestionException extends WebServiceException { + private static final String BY_NAME_MESSAGE = + "Search Index Not Found Exception [%s] due to [%s]."; + private static final String ERROR_TYPE = "SUGGESTION_EXCEPTION"; + + public SuggestionException(String message) { + super(Response.Status.INTERNAL_SERVER_ERROR, ERROR_TYPE, message); + } + + private SuggestionException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public SuggestionException(Response.Status status, String errorType, String message) { + super(status, errorType, message); + } + + public static SuggestionException byMessage( + String name, String errorMessage, Response.Status status) { + return new SuggestionException(status, buildMessageByName(name, errorMessage)); + } + + public static SuggestionException byMessage( + String name, String errorType, String errorMessage, Response.Status status) { + return new SuggestionException(status, errorType, buildMessageByName(name, errorMessage)); + } + + public static SuggestionException byMessage(String name, String errorMessage) { + return new SuggestionException( + Response.Status.BAD_REQUEST, buildMessageByName(name, errorMessage)); + } + + private static String buildMessageByName(String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, name, errorMessage); + } +} From 93057248595f8df9aca3d9f5de02cba2f1873afd Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 24 Jan 2024 18:36:53 +0530 Subject: [PATCH 17/49] initial ui changes --- .../src/main/resources/ui/src/App.tsx | 9 +- .../ui/src/assets/svg/ic-metapilot.svg | 24 +++ .../ui/src/assets/svg/ic-suggestions.svg | 24 +++ .../components/AppContainer/AppContainer.tsx | 16 +- .../MetaPilotDescriptionAlert.component.tsx | 84 +++++++++++ .../MetaPilotDescriptionAlert.interface.ts | 15 ++ .../MetaPilotProvider.interface.ts | 33 ++++ .../MetaPilotProvider/MetaPilotProvider.tsx | 142 ++++++++++++++++++ .../MetaPilotSidebar/MetaPilotSidebar.tsx | 100 ++++++++++++ .../MetaPilotSidebar/meta-pilot-sidebar.less | 40 +++++ .../SchemaTable/SchemaTable.component.tsx | 1 + .../TableDescription.component.tsx | 23 ++- .../TableDescription.interface.ts | 2 + .../EntityDescription/DescriptionV1.tsx | 24 ++- .../ui/src/locale/languages/en-us.json | 3 + .../TableDetailsPageV1/TableDetailsPageV1.tsx | 9 ++ .../resources/ui/src/rest/suggestionsAPI.ts | 32 ++++ .../src/main/resources/ui/src/styles/app.less | 7 + 18 files changed, 577 insertions(+), 11 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-metapilot.svg create mode 100644 openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less create mode 100644 openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts diff --git a/openmetadata-ui/src/main/resources/ui/src/App.tsx b/openmetadata-ui/src/main/resources/ui/src/App.tsx index 3579fb6730f9..3c923647cdc2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/App.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/App.tsx @@ -25,6 +25,7 @@ import DomainProvider from './components/Domain/DomainProvider/DomainProvider'; import { EntityExportModalProvider } from './components/Entity/EntityExportModalProvider/EntityExportModalProvider.component'; import ErrorBoundary from './components/ErrorBoundary/ErrorBoundary'; import GlobalSearchProvider from './components/GlobalSearchProvider/GlobalSearchProvider'; +import MetaPilotProvider from './components/MetaPilot/MetaPilotProvider/MetaPilotProvider'; import PermissionProvider from './components/PermissionProvider/PermissionProvider'; import TourProvider from './components/TourProvider/TourProvider'; import WebAnalyticsProvider from './components/WebAnalytics/WebAnalyticsProvider'; @@ -54,9 +55,11 @@ const App: FC = ({ routeElements }) => { - - - + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-metapilot.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-metapilot.svg new file mode 100644 index 000000000000..0b3526e03bd6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-metapilot.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg new file mode 100644 index 000000000000..7cc52b60ab00 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-suggestions.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx index a694a946b5ee..da967b5c0c4a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx @@ -19,12 +19,15 @@ import SignUpPage from '../../pages/SignUp/SignUpPage'; import Appbar from '../AppBar/Appbar'; import AuthenticatedAppRouter from '../AppRouter/AuthenticatedAppRouter'; import { useAuthContext } from '../Auth/AuthProviders/AuthProvider'; +import { useMetaPilotContext } from '../MetaPilot/MetaPilotProvider/MetaPilotProvider'; +import MetaPilotSidebar from '../MetaPilot/MetaPilotSidebar/MetaPilotSidebar'; import LeftSidebar from '../MyData/LeftSidebar/LeftSidebar.component'; import './app-container.less'; const AppContainer = () => { const { Header, Sider, Content } = Layout; const { currentUser } = useAuthContext(); + const { suggestionsVisible } = useMetaPilotContext(); return ( @@ -40,9 +43,16 @@ const AppContainer = () => {
- - - + + + + + {suggestionsVisible && ( + + + + )} + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx new file mode 100644 index 000000000000..e6ddc76760ed --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx @@ -0,0 +1,84 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; +import { Button, Card, Space, Typography } from 'antd'; +import React, { useLayoutEffect } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; +import { ReactComponent as SuggestionsIcon } from '../../../assets/svg/ic-suggestions.svg'; +import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import { useMetaPilotContext } from '../MetaPilotProvider/MetaPilotProvider'; +import { MetaPilotDescriptionAlertProps } from './MetaPilotDescriptionAlert.interface'; + +const MetaPilotDescriptionAlert = ({ + showHeading = true, +}: MetaPilotDescriptionAlertProps) => { + const { t } = useTranslation(); + const { activeSuggestion, onUpdateActiveSuggestion } = useMetaPilotContext(); + + useLayoutEffect(() => { + const element = document.querySelector('.suggested-description-card'); + if (element) { + element.scrollIntoView({ block: 'center', behavior: 'smooth' }); + } + }, []); + + if (!activeSuggestion) { + return null; + } + + return ( + + {showHeading && ( + + + {t('label.description')} + + + + )} + +
+
+ + + {t('label.metapilot-suggested-description')} + +
+ onUpdateActiveSuggestion(undefined)} /> +
+ +
+ + +
+ + + ); +}; + +export default MetaPilotDescriptionAlert; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts new file mode 100644 index 000000000000..db6570c5f345 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts @@ -0,0 +1,15 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export interface MetaPilotDescriptionAlertProps { + showHeading?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts new file mode 100644 index 000000000000..858ca6b7c828 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ReactNode } from 'react'; +import { Suggestion } from '../../../generated/entity/feed/suggestion'; + +export interface MetaPilotContextType { + suggestionsVisible: boolean; + isMetaPilotEnabled: boolean; + onToggleSuggestionsVisible: (state: boolean) => void; + onMetaPilotEnableUpdate: (state: boolean) => void; + activeSuggestion?: Suggestion; + suggestions: Suggestion[]; + loading: boolean; + entityFqn: string; + onUpdateActiveSuggestion: (suggestion?: Suggestion) => void; + fetchSuggestions: (entityFqn: string) => void; + onUpdateEntityFqn: (entityFqn: string) => void; + resetMetaPilot: () => void; +} + +export interface MetaPilotContextProps { + children: ReactNode; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx new file mode 100644 index 000000000000..5bae10bb6425 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx @@ -0,0 +1,142 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button } from 'antd'; +import { AxiosError } from 'axios'; +import React, { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; +import { Suggestion } from '../../../generated/entity/feed/suggestion'; +import { getMetaPilotSuggestionsList } from '../../../rest/suggestionsAPI'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { + MetaPilotContextProps, + MetaPilotContextType, +} from './MetaPilotProvider.interface'; + +export const MetaPilotContext = createContext({} as MetaPilotContextType); + +const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { + const { t } = useTranslation(); + const [suggestionsVisible, setSuggestionsVisible] = useState(false); + const [isMetaPilotEnabled, setIsMetaPilotEnabled] = useState(false); + const [activeSuggestion, setActiveSuggestion] = useState< + Suggestion | undefined + >(); + const [entityFqn, setEntityFqn] = useState(''); + const [suggestions, setSuggestions] = useState([]); + const [loading, setLoading] = useState(false); + + const fetchSuggestions = useCallback(async (entityFQN: string) => { + setLoading(true); + try { + const res = await getMetaPilotSuggestionsList({ + entityFQN, + }); + setSuggestions(res.data); + } catch (err) { + showErrorToast( + err as AxiosError, + t('server.entity-fetch-error', { + entity: t('label.lineage-data-lowercase'), + }) + ); + } finally { + setLoading(false); + } + }, []); + + const onToggleSuggestionsVisible = useCallback((state: boolean) => { + setSuggestionsVisible(state); + }, []); + + const onMetaPilotEnableUpdate = useCallback((state: boolean) => { + setIsMetaPilotEnabled(state); + }, []); + + const onUpdateActiveSuggestion = useCallback((suggestion?: Suggestion) => { + setActiveSuggestion(suggestion); + }, []); + + const onUpdateEntityFqn = useCallback((entityFqn: string) => { + setEntityFqn(entityFqn); + }, []); + + const resetMetaPilot = () => { + setSuggestionsVisible(false); + setIsMetaPilotEnabled(false); + setActiveSuggestion(undefined); + setEntityFqn(''); + }; + + useEffect(() => { + fetchSuggestions(entityFqn); + }, [entityFqn]); + + const metaPilotContextObj = useMemo(() => { + return { + suggestionsVisible, + isMetaPilotEnabled, + suggestions, + activeSuggestion, + entityFqn, + loading, + onToggleSuggestionsVisible, + onUpdateEntityFqn, + onMetaPilotEnableUpdate, + onUpdateActiveSuggestion, + fetchSuggestions, + resetMetaPilot, + }; + }, [ + suggestionsVisible, + isMetaPilotEnabled, + suggestions, + activeSuggestion, + entityFqn, + loading, + onToggleSuggestionsVisible, + onUpdateEntityFqn, + onMetaPilotEnableUpdate, + onUpdateActiveSuggestion, + fetchSuggestions, + resetMetaPilot, + ]); + + return ( + + {children} + {isMetaPilotEnabled && ( +
+
+ )} + + ); +}; + +export const useMetaPilotContext = () => useContext(MetaPilotContext); + +export default MetaPilotProvider; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx new file mode 100644 index 000000000000..017f6c1d0d8a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx @@ -0,0 +1,100 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CloseOutlined } from '@ant-design/icons'; +import { Card, Drawer, Typography } from 'antd'; +import React, { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; +import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; +import { Suggestion } from '../../../generated/entity/feed/suggestion'; +import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; +import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import Loader from '../../Loader/Loader'; +import { useMetaPilotContext } from '../MetaPilotProvider/MetaPilotProvider'; +import './meta-pilot-sidebar.less'; + +const MetaPilotSidebar = () => { + const { t } = useTranslation(); + const { + onUpdateActiveSuggestion, + suggestions, + loading, + suggestionsVisible, + onToggleSuggestionsVisible, + } = useMetaPilotContext(); + + const descriptionsView = useMemo(() => { + return suggestions.map((item: Suggestion) => { + return ( + onUpdateActiveSuggestion(item)}> + + + {item.entityLink} + + + ); + }); + }, [suggestions]); + + return ( + onToggleSuggestionsVisible(false)} + /> + } + getContainer={false} + headerStyle={{ padding: 16 }} + mask={false} + open={suggestionsVisible} + title={ +
+ + + {t('label.metapilot')} + +
+ } + width={340}> + {loading ? ( + + ) : ( + <> + {suggestions?.length === 0 && ( + + )} + {suggestions.length > 0 && ( + <> + + {t('label.suggested-description-plural')} + + {descriptionsView} + + )} + + )} + + ); +}; + +export default MetaPilotSidebar; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less new file mode 100644 index 000000000000..e5906f0cb76a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import url('../../../styles/variables.less'); + +.meta-pilot-drawer { + .ant-drawer-content { + background-color: #f6f9fd; + } + .ant-drawer-body { + padding: 0 16px 16px; + } + .suggestion-card { + border-radius: 10px; + cursor: pointer; + &:hover { + .ant-typography { + color: @primary-color; + } + } + } + .ant-drawer-content-wrapper { + box-shadow: none !important; + } +} + +.suggested-description-card { + border-radius: 10px !important; + border-color: @grey-4 !important; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx index 189eb8a6ba27..ce491fdf2d2c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SchemaTable/SchemaTable.component.tsx @@ -228,6 +228,7 @@ const SchemaTable = ({ columnData={{ fqn: record.fullyQualifiedName ?? '', field: record.description, + record, }} entityFqn={decodedEntityFqn} entityType={EntityType.TABLE} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx index 8c3ae8427839..0cab3755a274 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx @@ -12,13 +12,18 @@ */ import { Button, Space } from 'antd'; -import React from 'react'; +import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../assets/svg/edit-new.svg'; import RichTextEditorPreviewer from '../../components/common/RichTextEditor/RichTextEditorPreviewer'; import { DE_ACTIVE_COLOR } from '../../constants/constants'; import { EntityField } from '../../constants/Feeds.constants'; +import { EntityType } from '../../enums/entity.enum'; import EntityTasks from '../../pages/TasksPage/EntityTasks/EntityTasks.component'; +import EntityLink from '../../utils/EntityLink'; +import { getEntityFeedLink } from '../../utils/EntityUtils'; +import MetaPilotDescriptionAlert from '../MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component'; +import { useMetaPilotContext } from '../MetaPilot/MetaPilotProvider/MetaPilotProvider'; import { TableDescriptionProps } from './TableDescription.interface'; const TableDescription = ({ @@ -32,6 +37,22 @@ const TableDescription = ({ onThreadLinkSelect, }: TableDescriptionProps) => { const { t } = useTranslation(); + const { activeSuggestion } = useMetaPilotContext(); + + const entityLink = useMemo( + () => + entityType === EntityType.TABLE + ? EntityLink.getTableEntityLink( + entityFqn, + columnData.record?.name ?? '' + ) + : getEntityFeedLink(entityType, columnData.fqn), + [entityType, entityFqn] + ); + + if (activeSuggestion?.entityLink === entityLink) { + return ; + } return ( { const history = useHistory(); + const { activeSuggestion } = useMetaPilotContext(); const handleRequestDescription = useCallback(() => { history.push( @@ -86,10 +89,19 @@ const DescriptionV1 = ({ ); }, [entityType, entityFqn]); - const entityLink = useMemo( - () => getEntityFeedLink(entityType, entityFqn, EntityField.DESCRIPTION), - [entityType, entityFqn] - ); + const { entityLink, entityLinkWithoutField } = useMemo(() => { + const entityLink = getEntityFeedLink( + entityType, + entityFqn, + EntityField.DESCRIPTION + ); + const entityLinkWithoutField = getEntityFeedLink(entityType, entityFqn); + + return { + entityLink, + entityLinkWithoutField, + }; + }, [entityType, entityFqn]); const taskActionButton = useMemo(() => { const hasDescription = Boolean(description.trim()); @@ -159,6 +171,10 @@ const DescriptionV1 = ({ ] ); + if (activeSuggestion?.entityLink === entityLinkWithoutField) { + return ; + } + const content = ( { ThreadType.Conversation ); const [queryCount, setQueryCount] = useState(0); + const { onMetaPilotEnableUpdate, onUpdateEntityFqn, resetMetaPilot } = + useMetaPilotContext(); const [loading, setLoading] = useState(!isTourOpen); const [tablePermissions, setTablePermissions] = useState( @@ -276,7 +279,13 @@ const TableDetailsPageV1 = () => { useEffect(() => { if (tableFqn) { fetchResourcePermission(tableFqn); + onMetaPilotEnableUpdate(true); + onUpdateEntityFqn(tableFqn); } + + return () => { + resetMetaPilot(); + }; }, [tableFqn]); const getEntityFeedCount = () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts new file mode 100644 index 000000000000..6afd16fbdb94 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PagingResponse } from 'Models'; +import { Suggestion } from '../generated/entity/feed/suggestion'; +import { ListParams } from '../interface/API.interface'; +import APIClient from './index'; + +const BASE_URL = '/suggestions'; + +export type ListSuggestionsParams = ListParams & { + entityFQN?: string; +}; + +export const getMetaPilotSuggestionsList = async ( + params?: ListSuggestionsParams +) => { + const response = await APIClient.get>(BASE_URL, { + params, + }); + + return response.data; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/app.less b/openmetadata-ui/src/main/resources/ui/src/styles/app.less index c9a1e6fc8730..bc7f8f1b0e63 100644 --- a/openmetadata-ui/src/main/resources/ui/src/styles/app.less +++ b/openmetadata-ui/src/main/resources/ui/src/styles/app.less @@ -734,3 +734,10 @@ a[href].link-text-grey, width: 20px; height: 20px; } + +.floating-button-container { + position: fixed; + bottom: 20px; + right: 20px; + z-index: 1001; +} From 6ddea6788c12b481a0aa83b09a263bbde5a5e58c Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 24 Jan 2024 18:38:25 +0530 Subject: [PATCH 18/49] localisation --- .../src/components/common/DeleteWidget/DeleteWidgetModal.tsx | 3 ++- .../src/main/resources/ui/src/locale/languages/de-de.json | 3 +++ .../src/main/resources/ui/src/locale/languages/es-es.json | 3 +++ .../src/main/resources/ui/src/locale/languages/fr-fr.json | 3 +++ .../src/main/resources/ui/src/locale/languages/he-he.json | 3 +++ .../src/main/resources/ui/src/locale/languages/ja-jp.json | 3 +++ .../src/main/resources/ui/src/locale/languages/nl-nl.json | 3 +++ .../src/main/resources/ui/src/locale/languages/pt-br.json | 3 +++ .../src/main/resources/ui/src/locale/languages/ru-ru.json | 3 +++ .../src/main/resources/ui/src/locale/languages/zh-cn.json | 3 +++ 10 files changed, 29 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx index d7f3e97ff6e3..7ecb6b465127 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DeleteWidget/DeleteWidgetModal.tsx @@ -124,7 +124,8 @@ const DeleteWidgetModal = ({ const isDeleteTextPresent = useMemo(() => { return ( - deleteConfirmationText.toLowerCase() === DELETE_CONFIRMATION_TEXT.toLowerCase() && + deleteConfirmationText.toLowerCase() === + DELETE_CONFIRMATION_TEXT.toLowerCase() && (deletionType === DeleteType.SOFT_DELETE || deletionType === DeleteType.HARD_DELETE) ); diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 088e7dd9255a..26d199cb2c1f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -642,6 +642,8 @@ "metadata-lowercase": "metadaten", "metadata-plural": "Metadaten", "metadata-to-es-config-optional": "Metadaten-zu-ES-Konfiguration (optional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Metriktyp", "metric-value": "Metrikwert", "metrics-summary": "Metrikzusammenfassung", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "Erfolgreich hochgeladen", "suggest": "Vorschlagen", "suggest-entity": "{{entity}} vorschlagen", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Vorschlag", "suggestion-lowercase-plural": "Vorschläge", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index c4012d85c86d..14b78a2b353b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -642,6 +642,8 @@ "metadata-lowercase": "metadatos", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Configuración de Metadatos a ES (Opcional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Tipo de Métrica", "metric-value": "Valor de Métrica", "metrics-summary": "Resumen de Métricas", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "Cargado Exitosamente", "suggest": "Sugerir", "suggest-entity": "Sugerir {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Sugerencia", "suggestion-lowercase-plural": "sugerencias", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index 654d18892fa6..9aecf4c4ecf1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -642,6 +642,8 @@ "metadata-lowercase": "métadonnées", "metadata-plural": "Métadonnées", "metadata-to-es-config-optional": "Configuration de Métadonnées vers ES (Optionnelle)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Type de Mesure", "metric-value": "Valeur de la Mesure", "metrics-summary": "Résumé des Mesures", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "Téléchargé avec succès", "suggest": "Suggérer", "suggest-entity": "Suggérer {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Suggestion", "suggestion-lowercase-plural": "suggestions", "suite": "Ensemble", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json index be695f9dea47..4c4704f927ff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json @@ -642,6 +642,8 @@ "metadata-lowercase": "מטא-דאטה", "metadata-plural": "מטא-דאטה", "metadata-to-es-config-optional": "קונפיגורציית מטא-דאטה ל-Elasticsearch (אופציונלי)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "סוג מדד", "metric-value": "ערך מדד", "metrics-summary": "סיכום מדדים", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "הועלה בהצלחה", "suggest": "הצע", "suggest-entity": "הצע {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "הצעה", "suggestion-lowercase-plural": "הצעות", "suite": "יחידת בדיקה", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 47099c0f9a50..5bd2ce25f54c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -642,6 +642,8 @@ "metadata-lowercase": "メタデータ", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Metadata To ES Config (Optional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "メトリクスのタイプ", "metric-value": "メトリクスの値", "metrics-summary": "メトリクスの要約", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "アップロード成功", "suggest": "提案", "suggest-entity": "{{entity}}を提案", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "提案", "suggestion-lowercase-plural": "提案", "suite": "スイート", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json index bd7ddc5122ec..60133d7e6c6c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json @@ -642,6 +642,8 @@ "metadata-lowercase": "metadata", "metadata-plural": "Metadata", "metadata-to-es-config-optional": "Metadata naar ES-configuratie (Optioneel)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Metriektype", "metric-value": "Metriekwaarde", "metrics-summary": "Samenvatting van metingen", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "Succesvol Geüpload", "suggest": "Suggestie", "suggest-entity": "Suggereer {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Suggestie", "suggestion-lowercase-plural": "suggesties", "suite": "Suite", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index 77323d795b6b..02e805cef6a8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -642,6 +642,8 @@ "metadata-lowercase": "metadados", "metadata-plural": "Metadados", "metadata-to-es-config-optional": "Metadados para Configuração ES (Opcional)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Tipo de Métrica", "metric-value": "Valor da Métrica", "metrics-summary": "Resumo de Métricas", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "Carregado com Sucesso", "suggest": "Sugerir", "suggest-entity": "Sugerir {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Sugestão", "suggestion-lowercase-plural": "sugestões", "suite": "Conjuto de Testes", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index 6b81e846f271..e4355e164335 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -642,6 +642,8 @@ "metadata-lowercase": "метаданные", "metadata-plural": "Метаданные", "metadata-to-es-config-optional": "Метаданные для конфигурации ES (необязательно)", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "Тип метрики", "metric-value": "Значение метрики", "metrics-summary": "Сводка метрик", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "Успешно загружено", "suggest": "Предложить", "suggest-entity": "Предложить {{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "Предложение", "suggestion-lowercase-plural": "предложения", "suite": "Набор", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 61bdb7fd2696..199d880ecd20 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -642,6 +642,8 @@ "metadata-lowercase": "元数据", "metadata-plural": "元数据", "metadata-to-es-config-optional": "元数据到 ES 配置（可选）", + "metapilot": "MetaPilot", + "metapilot-suggested-description": "Metapilot Suggested Description", "metric-type": "指标类型", "metric-value": "指标值", "metrics-summary": "指标概要", @@ -1019,6 +1021,7 @@ "successfully-uploaded": "上传成功", "suggest": "建议", "suggest-entity": "建议{{entity}}", + "suggested-description-plural": "Suggested Descriptions", "suggestion": "建议", "suggestion-lowercase-plural": "建议", "suite": "套件", From 84a9213531ca44d89b0e959796fc515ca8c8448c Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 25 Jan 2024 19:28:42 +0530 Subject: [PATCH 19/49] show suggestion for empty description --- .../MetaPilotDescriptionAlert.component.tsx | 22 +++++++++++++------ .../MetaPilotDescriptionAlert.interface.ts | 3 +++ .../MetaPilotProvider.interface.ts | 9 ++++++++ .../MetaPilotProvider/MetaPilotProvider.tsx | 20 ++++++++++++++++- .../TableDescription.component.tsx | 7 +++++- .../EntityDescription/DescriptionV1.tsx | 19 ++++++++++++++-- .../resources/ui/src/rest/suggestionsAPI.ts | 11 ++++++++++ 7 files changed, 80 insertions(+), 11 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx index e6ddc76760ed..f1dc56d991ac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx @@ -18,13 +18,16 @@ import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilo import { ReactComponent as SuggestionsIcon } from '../../../assets/svg/ic-suggestions.svg'; import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; import { useMetaPilotContext } from '../MetaPilotProvider/MetaPilotProvider'; +import { SuggestionAction } from '../MetaPilotProvider/MetaPilotProvider.interface'; import { MetaPilotDescriptionAlertProps } from './MetaPilotDescriptionAlert.interface'; const MetaPilotDescriptionAlert = ({ showHeading = true, + suggestion, }: MetaPilotDescriptionAlertProps) => { const { t } = useTranslation(); - const { activeSuggestion, onUpdateActiveSuggestion } = useMetaPilotContext(); + const { onUpdateActiveSuggestion, acceptRejectSuggestion } = + useMetaPilotContext(); useLayoutEffect(() => { const element = document.querySelector('.suggested-description-card'); @@ -33,7 +36,7 @@ const MetaPilotDescriptionAlert = ({ } }, []); - if (!activeSuggestion) { + if (!suggestion) { return null; } @@ -61,18 +64,23 @@ const MetaPilotDescriptionAlert = ({ onUpdateActiveSuggestion(undefined)} /> - +
-
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts index db6570c5f345..dc2ff6138890 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts @@ -10,6 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Suggestion } from '../../../generated/entity/feed/suggestion'; + export interface MetaPilotDescriptionAlertProps { showHeading?: boolean; + suggestion: Suggestion; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts index 858ca6b7c828..8950dc1f056e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts @@ -24,6 +24,10 @@ export interface MetaPilotContextType { entityFqn: string; onUpdateActiveSuggestion: (suggestion?: Suggestion) => void; fetchSuggestions: (entityFqn: string) => void; + acceptRejectSuggestion: ( + suggestion: Suggestion, + action: SuggestionAction + ) => void; onUpdateEntityFqn: (entityFqn: string) => void; resetMetaPilot: () => void; } @@ -31,3 +35,8 @@ export interface MetaPilotContextType { export interface MetaPilotContextProps { children: ReactNode; } + +export enum SuggestionAction { + Accept = 'accept', + Reject = 'reject', +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx index 5bae10bb6425..874b97c9f7c3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx @@ -23,11 +23,15 @@ import React, { import { useTranslation } from 'react-i18next'; import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; import { Suggestion } from '../../../generated/entity/feed/suggestion'; -import { getMetaPilotSuggestionsList } from '../../../rest/suggestionsAPI'; +import { + getMetaPilotSuggestionsList, + updateSuggestionStatus, +} from '../../../rest/suggestionsAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; import { MetaPilotContextProps, MetaPilotContextType, + SuggestionAction, } from './MetaPilotProvider.interface'; export const MetaPilotContext = createContext({} as MetaPilotContextType); @@ -62,6 +66,18 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { } }, []); + const acceptRejectSuggestion = useCallback( + async (suggestion: Suggestion, status: SuggestionAction) => { + try { + await updateSuggestionStatus(suggestion, status); + await fetchSuggestions(entityFqn); + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [entityFqn] + ); + const onToggleSuggestionsVisible = useCallback((state: boolean) => { setSuggestionsVisible(state); }, []); @@ -102,6 +118,7 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { onMetaPilotEnableUpdate, onUpdateActiveSuggestion, fetchSuggestions, + acceptRejectSuggestion, resetMetaPilot, }; }, [ @@ -116,6 +133,7 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { onMetaPilotEnableUpdate, onUpdateActiveSuggestion, fetchSuggestions, + acceptRejectSuggestion, resetMetaPilot, ]); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx index 0cab3755a274..7896159682c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx @@ -51,7 +51,12 @@ const TableDescription = ({ ); if (activeSuggestion?.entityLink === entityLink) { - return ; + return ( + + ); } return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx index 00df4f916287..a2eef6269b1d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx @@ -14,6 +14,7 @@ import Icon from '@ant-design/icons'; import { Card, Space, Tooltip, Typography } from 'antd'; import { t } from 'i18next'; +import { isEmpty } from 'lodash'; import React, { useCallback, useMemo } from 'react'; import { useHistory } from 'react-router'; import { ReactComponent as CommentIcon } from '../../../assets/svg/comment.svg'; @@ -75,7 +76,7 @@ const DescriptionV1 = ({ reduceDescription, }: Props) => { const history = useHistory(); - const { activeSuggestion } = useMetaPilotContext(); + const { activeSuggestion, suggestions } = useMetaPilotContext(); const handleRequestDescription = useCallback(() => { history.push( @@ -171,8 +172,22 @@ const DescriptionV1 = ({ ] ); + const suggestionForEmptyData = useMemo(() => { + if (isEmpty(description.trim())) { + return suggestions.find( + (suggestion) => suggestion.entityLink === entityLinkWithoutField + ); + } + + return null; + }, [suggestions, description]); + if (activeSuggestion?.entityLink === entityLinkWithoutField) { - return ; + return ; + } + + if (suggestionForEmptyData) { + return ; } const content = ( diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts index 6afd16fbdb94..812877e4f6fa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/suggestionsAPI.ts @@ -10,7 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { AxiosResponse } from 'axios'; import { PagingResponse } from 'Models'; +import { SuggestionAction } from '../components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface'; import { Suggestion } from '../generated/entity/feed/suggestion'; import { ListParams } from '../interface/API.interface'; import APIClient from './index'; @@ -30,3 +32,12 @@ export const getMetaPilotSuggestionsList = async ( return response.data; }; + +export const updateSuggestionStatus = ( + data: Suggestion, + action: SuggestionAction +): Promise => { + const url = `${BASE_URL}/${data.id}/${action}`; + + return APIClient.put(url, {}); +}; From 05af100ead998d6c3b2f8cd2adae189acec81ed3 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Thu, 25 Jan 2024 22:23:12 +0530 Subject: [PATCH 20/49] ui feedbacks --- ...c-metapilot.svg => MetaPilotApplication.svg} | 0 .../MetaPilotDescriptionAlert.component.tsx | 2 +- .../MetaPilotProvider/MetaPilotProvider.tsx | 2 +- .../MetaPilotSidebar/MetaPilotSidebar.tsx | 6 ++++-- .../MetaPilotSidebar/meta-pilot-sidebar.less | 2 +- .../TableDescription.component.tsx | 17 ++++++++++++++++- 6 files changed, 23 insertions(+), 6 deletions(-) rename openmetadata-ui/src/main/resources/ui/src/assets/svg/{ic-metapilot.svg => MetaPilotApplication.svg} (100%) diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-metapilot.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg similarity index 100% rename from openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-metapilot.svg rename to openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx index f1dc56d991ac..1abd3172f206 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx @@ -14,8 +14,8 @@ import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; import { Button, Card, Space, Typography } from 'antd'; import React, { useLayoutEffect } from 'react'; import { useTranslation } from 'react-i18next'; -import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; import { ReactComponent as SuggestionsIcon } from '../../../assets/svg/ic-suggestions.svg'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; import { useMetaPilotContext } from '../MetaPilotProvider/MetaPilotProvider'; import { SuggestionAction } from '../MetaPilotProvider/MetaPilotProvider.interface'; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx index 874b97c9f7c3..70882044ee4d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx @@ -21,7 +21,7 @@ import React, { useState, } from 'react'; import { useTranslation } from 'react-i18next'; -import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; import { Suggestion } from '../../../generated/entity/feed/suggestion'; import { getMetaPilotSuggestionsList, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx index 017f6c1d0d8a..ff5b9c54cbd5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/MetaPilotSidebar.tsx @@ -14,9 +14,10 @@ import { CloseOutlined } from '@ant-design/icons'; import { Card, Drawer, Typography } from 'antd'; import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; -import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/ic-metapilot.svg'; +import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; import { Suggestion } from '../../../generated/entity/feed/suggestion'; +import EntityLink from '../../../utils/EntityLink'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; import Loader from '../../Loader/Loader'; @@ -45,7 +46,8 @@ const MetaPilotSidebar = () => { markdown={item.description ?? ''} /> - {item.entityLink} + {EntityLink.getTableColumnName(item.entityLink) ?? + EntityLink.getEntityFqn(item.entityLink)} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less index e5906f0cb76a..7bfa1b0da7bc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotSidebar/meta-pilot-sidebar.less @@ -24,7 +24,7 @@ border-radius: 10px; cursor: pointer; &:hover { - .ant-typography { + p { color: @primary-color; } } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx index 7896159682c1..f9429d62953a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx @@ -12,6 +12,7 @@ */ import { Button, Space } from 'antd'; +import { isEmpty } from 'lodash'; import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../assets/svg/edit-new.svg'; @@ -37,7 +38,7 @@ const TableDescription = ({ onThreadLinkSelect, }: TableDescriptionProps) => { const { t } = useTranslation(); - const { activeSuggestion } = useMetaPilotContext(); + const { activeSuggestion, suggestions } = useMetaPilotContext(); const entityLink = useMemo( () => @@ -50,6 +51,16 @@ const TableDescription = ({ [entityType, entityFqn] ); + const suggestionForEmptyData = useMemo(() => { + if (isEmpty(columnData.field ?? ''.trim())) { + return suggestions.find( + (suggestion) => suggestion.entityLink === entityLink + ); + } + + return null; + }, [suggestions, columnData.field, entityLink]); + if (activeSuggestion?.entityLink === entityLink) { return ( ; + } + return ( Date: Thu, 25 Jan 2024 11:46:57 -0800 Subject: [PATCH 21/49] Fix permission check for entities without owner --- .../service/jdbi3/SuggestionRepository.java | 41 +++++++++++++------ .../resources/feeds/SuggestionsResource.java | 3 +- .../feeds/SuggestionResourceTest.java | 24 ++++++++++- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java index a4dfcdf193e6..146c2846795a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -38,6 +38,9 @@ import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.SuggestionsResource; import org.openmetadata.service.security.AuthorizationException; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.RestUtil; @@ -183,14 +186,19 @@ public EntityInterface acceptSuggestions( } public RestUtil.PutResponse acceptSuggestion( - UriInfo uriInfo, Suggestion suggestion, String user) { + UriInfo uriInfo, + Suggestion suggestion, + SecurityContext securityContext, + Authorizer authorizer) { suggestion.setStatus(SuggestionStatus.Accepted); - acceptSuggestion(suggestion, user); + acceptSuggestion(suggestion, securityContext, authorizer); Suggestion updatedHref = SuggestionsResource.addHref(uriInfo, suggestion); return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_ACCEPTED); } - protected void acceptSuggestion(Suggestion suggestion, String user) { + protected void acceptSuggestion( + Suggestion suggestion, SecurityContext securityContext, Authorizer authorizer) { + String user = securityContext.getUserPrincipal().getName(); MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(suggestion.getEntityLink()); EntityInterface entity = @@ -202,6 +210,11 @@ protected void acceptSuggestion(Suggestion suggestion, String user) { EntityInterface updatedEntity = suggestionWorkflow.acceptSuggestions(repository, entity); String updatedEntityJson = JsonUtils.pojoToJson(updatedEntity); JsonPatch patch = JsonUtils.getJsonPatch(origJson, updatedEntityJson); + OperationContext operationContext = new OperationContext(entityLink.getEntityType(), patch); + authorizer.authorize( + securityContext, + operationContext, + new ResourceContext<>(entityLink.getEntityType(), entity.getId(), null)); repository.patch(null, entity.getId(), user, patch); suggestion.setStatus(SuggestionStatus.Accepted); update(suggestion, user); @@ -223,21 +236,25 @@ public void checkPermissionsForAcceptOrRejectSuggestion( EntityReference aboutRef = EntityUtil.validateEntityLink(about); EntityReference ownerRef = Entity.getOwner(aboutRef); List ownerTeamNames = new ArrayList<>(); - try { - User owner = - Entity.getEntityByName(USER, ownerRef.getFullyQualifiedName(), TEAMS_FIELD, NON_DELETED); - ownerTeamNames = - owner.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); - } catch (EntityNotFoundException e) { - Team owner = Entity.getEntityByName(TEAM, ownerRef.getFullyQualifiedName(), "", NON_DELETED); - ownerTeamNames.add(owner.getFullyQualifiedName()); + if (ownerRef != null) { + try { + User owner = + Entity.getEntityByName( + USER, ownerRef.getFullyQualifiedName(), TEAMS_FIELD, NON_DELETED); + ownerTeamNames = + owner.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); + } catch (EntityNotFoundException e) { + Team owner = + Entity.getEntityByName(TEAM, ownerRef.getFullyQualifiedName(), "", NON_DELETED); + ownerTeamNames.add(owner.getFullyQualifiedName()); + } } List userTeamNames = user.getTeams().stream().map(EntityReference::getFullyQualifiedName).toList(); if (Boolean.FALSE.equals(user.getIsAdmin()) - && !ownerRef.getName().equals(userName) + && (ownerRef != null && !ownerRef.getName().equals(userName)) && Collections.disjoint(userTeamNames, ownerTeamNames)) { throw new AuthorizationException( CatalogExceptionMessage.suggestionOperationNotAllowed(userName, status.value())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index f0d9a4aa269f..f14bc70330c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -227,8 +227,7 @@ public Response acceptSuggestion( Suggestion suggestion = dao.get(id); dao.checkPermissionsForAcceptOrRejectSuggestion( suggestion, SuggestionStatus.Accepted, securityContext); - return dao.acceptSuggestion(uriInfo, suggestion, securityContext.getUserPrincipal().getName()) - .toResponse(); + return dao.acceptSuggestion(uriInfo, suggestion, securityContext, authorizer).toResponse(); } @PUT diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java index 9e011b6070ed..ea6202ad5f06 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java @@ -63,8 +63,11 @@ public class SuggestionResourceTest extends OpenMetadataApplicationTest { public static Table TABLE; public static Table TABLE2; + + public static Table TABLE_WITHOUT_OWNER; public static String TABLE_LINK; public static String TABLE2_LINK; + public static String TABLE_WITHOUT_OWNER_LINK; public static String TABLE_COLUMN1_LINK; public static String TABLE_COLUMN2_LINK; public static List COLUMNS; @@ -107,11 +110,18 @@ public void setup(TestInfo test) throws IOException, URISyntaxException { createTable2.withName("table2").withOwner(TEAM2_REF); TABLE2 = TABLE_RESOURCE_TEST.createAndCheckEntity(createTable2, ADMIN_AUTH_HEADERS); + CreateTable createTable3 = TABLE_RESOURCE_TEST.createRequest(test); + createTable3.withName("table_without_owner").withOwner(null); + TABLE_WITHOUT_OWNER = + TABLE_RESOURCE_TEST.createAndCheckEntity(createTable3, ADMIN_AUTH_HEADERS); + COLUMNS = Collections.singletonList( new Column().withName("column1").withDataType(ColumnDataType.BIGINT)); TABLE_LINK = String.format("<#E::table::%s>", TABLE.getFullyQualifiedName()); TABLE2_LINK = String.format("<#E::table::%s>", TABLE2.getFullyQualifiedName()); + TABLE_WITHOUT_OWNER_LINK = + String.format("<#E::table::%s>", TABLE_WITHOUT_OWNER.getFullyQualifiedName()); TABLE_COLUMN1_LINK = String.format("<#E::table::%s::columns::" + C1 + ">", TABLE.getFullyQualifiedName()); TABLE_COLUMN2_LINK = @@ -227,7 +237,13 @@ void post_validSuggestionAndList_200(TestInfo test) throws IOException { null) .getPaging() .getTotal()); - + create = create().withEntityLink(TABLE_WITHOUT_OWNER_LINK); + createAndCheck(create, USER_AUTH_HEADERS); + assertEquals( + 1, + listSuggestions(TABLE_WITHOUT_OWNER.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + .getPaging() + .getTotal()); /* deleteSuggestions("table", TABLE.getFullyQualifiedName(), USER_AUTH_HEADERS); assertEquals( 0, @@ -279,6 +295,12 @@ void put_acceptSuggestion_200(TestInfo test) throws IOException { expectedTags.addAll(suggestion2.getTagLabels()); validateAppliedTags(expectedTags, column.getTags()); } + String description = "Table without owner"; + create = create().withEntityLink(TABLE_WITHOUT_OWNER_LINK).withDescription(description); + Suggestion suggestion3 = createSuggestion(create, USER_AUTH_HEADERS); + acceptSuggestion(suggestion3.getId(), USER2_AUTH_HEADERS); + table = tableResourceTest.getEntity(TABLE_WITHOUT_OWNER.getId(), "", USER_AUTH_HEADERS); + assertEquals(description, table.getDescription()); } @Test From cdce603e97ea835cfc152bc820bf9aebf6b99b74 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 26 Jan 2024 11:32:30 +0100 Subject: [PATCH 22/49] Fix entityLink and add tests --- .../src/metadata/ingestion/ometa/routes.py | 2 + ingestion/src/metadata/utils/entity_link.py | 24 +++++++----- ingestion/src/metadata/utils/helpers.py | 12 +++++- .../tests/integration/integration_base.py | 5 --- .../ometa/test_ometa_suggestion_api.py | 38 ++++++++++++++++++- .../tests/integration/utils/test_helpers.py | 3 ++ ingestion/tests/unit/test_entity_link.py | 19 ++++++++++ ...Test.java => SuggestionsResourceTest.java} | 2 +- 8 files changed, 87 insertions(+), 18 deletions(-) rename openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/{SuggestionResourceTest.java => SuggestionsResourceTest.java} (99%) diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index 3bb28585d580..f625459a179f 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -91,6 +91,7 @@ from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart from metadata.generated.schema.dataInsight.kpi.kpi import Kpi from metadata.generated.schema.entity.automations.workflow import Workflow +from metadata.generated.schema.entity.bot import Bot from metadata.generated.schema.entity.classification.classification import ( Classification, ) @@ -183,6 +184,7 @@ User.__name__: "/users", CreateUserRequest.__name__: "/users", AuthenticationMechanism.__name__: "/users/auth-mechanism", + Bot.__name__: "/bots", # We won't allow bot creation from the client # Roles Role.__name__: "/roles", CreateRoleRequest.__name__: "/roles", diff --git a/ingestion/src/metadata/utils/entity_link.py b/ingestion/src/metadata/utils/entity_link.py index 4c34073c190c..0fbac0f0e0d8 100644 --- a/ingestion/src/metadata/utils/entity_link.py +++ b/ingestion/src/metadata/utils/entity_link.py @@ -13,8 +13,7 @@ Filter information has been taken from the ES indexes definitions """ -from functools import singledispatch -from typing import List, Optional, Type, TypeVar +from typing import Any, List, Optional, TypeVar from antlr4.CommonTokenStream import CommonTokenStream from antlr4.error.ErrorStrategy import BailErrorStrategy @@ -28,6 +27,7 @@ from metadata.generated.antlr.EntityLinkParser import EntityLinkParser from metadata.generated.schema.entity.data.table import Table from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP +from metadata.utils.dispatch import class_register T = TypeVar("T", bound=BaseModel) @@ -92,8 +92,10 @@ def get_table_or_column_fqn(entity_link: str) -> str: ) -@singledispatch -def get_entity_link(entity_type: Type[T], fqn: str, **__) -> str: +get_entity_link_registry = class_register() + + +def get_entity_link(entity_type: Any, fqn: str, **kwargs) -> str: """From table fqn and column name get the entity_link Args: @@ -101,15 +103,19 @@ def get_entity_link(entity_type: Type[T], fqn: str, **__) -> str: fqn: Entity fqn """ - return f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}>" + func = get_entity_link_registry.registry.get(entity_type.__name__) + if not func: + return f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}>" + + return func(fqn, **kwargs) -@get_entity_link.register(Table) -def _(entity_type: Table, fqn: str, column_name: Optional[str]) -> str: +@get_entity_link_registry.add(Table) +def _(fqn: str, column_name: Optional[str] = None) -> str: """From table fqn and column name get the entity_link""" if column_name: - entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}::columns::{column_name}>" + entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[Table.__name__]}::{fqn}::columns::{column_name}>" else: - entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[entity_type.__name__]}::{fqn}>" + entity_link = f"<#E::{ENTITY_REFERENCE_TYPE_MAP[Table.__name__]}::{fqn}>" return entity_link diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index f76c979934b8..55e878e421d1 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -227,12 +227,20 @@ def find_in_iter(element: Any, container: Iterable[Any]) -> Optional[Any]: return next((elem for elem in container if elem == element), None) -def find_column_in_table(column_name: str, table: Table) -> Optional[Column]: +def find_column_in_table( + column_name: str, table: Table, case_sensitive: bool = True +) -> Optional[Column]: """ If the column exists in the table, return it """ + + def equals(first: str, second: str) -> bool: + if case_sensitive: + return first == second + return first.lower() == second.lower() + return next( - (col for col in table.columns if col.name.__root__ == column_name), None + (col for col in table.columns if equals(col.name.__root__, column_name)), None ) diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index 13433c6de8a6..af0aacdc1bf6 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -60,11 +60,6 @@ DatabaseService, DatabaseServiceType, ) -from metadata.generated.schema.entity.services.databaseService import ( - DatabaseConnection, - DatabaseService, - DatabaseServiceType, -) from metadata.generated.schema.entity.services.pipelineService import ( PipelineConnection, PipelineService, diff --git a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py index 0f4018a5e2cc..1fd8bb5e8f07 100644 --- a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py @@ -18,8 +18,9 @@ from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.table import Table -from metadata.generated.schema.entity.feed.suggestion import SuggestionType +from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.entity.teams.user import User from metadata.generated.schema.type.basic import EntityLink from metadata.generated.schema.type.tagLabel import ( LabelType, @@ -134,3 +135,38 @@ def test_create_tag_suggestion(self): # Suggestions only support POST (not PUT) self.metadata.create(suggestion_request) + + def test_list(self): + """List filtering by creator""" + + admin_user: User = self.metadata.get_by_name( + entity=User, fqn="admin", nullable=False + ) + + create_table = get_create_entity( + entity=Table, + name=self.schema_name, + reference=self.schema.fullyQualifiedName.__root__, + ) + table: Table = self.metadata.create_or_update(create_table) + + suggestion_request = CreateSuggestionRequest( + description="something", + type=SuggestionType.SuggestDescription, + entityLink=EntityLink( + __root__=get_entity_link(Table, fqn=table.fullyQualifiedName.__root__) + ), + ) + + # Suggestions only support POST (not PUT) + self.metadata.create(suggestion_request) + + suggestions = self.metadata.list_all_entities( + entity=Suggestion, + params={ + "entityFQN": table.fullyQualifiedName.__root__, + "userId": str(admin_user.id.__root__), + }, + ) + + self.assertTrue(len(list(suggestions))) diff --git a/ingestion/tests/integration/utils/test_helpers.py b/ingestion/tests/integration/utils/test_helpers.py index 004842afda34..53ef680c0c86 100644 --- a/ingestion/tests/integration/utils/test_helpers.py +++ b/ingestion/tests/integration/utils/test_helpers.py @@ -63,3 +63,6 @@ def test_find_column_in_table(self): ) self.assertIsNone(not_found) self.assertIsNone(not_found_idx) + + col = find_column_in_table(column_name="FOO", table=table, case_sensitive=False) + self.assertEqual(col, Column(name="foo", dataType=DataType.BIGINT)) diff --git a/ingestion/tests/unit/test_entity_link.py b/ingestion/tests/unit/test_entity_link.py index ca45c858b1a6..fe02499c8997 100644 --- a/ingestion/tests/unit/test_entity_link.py +++ b/ingestion/tests/unit/test_entity_link.py @@ -13,7 +13,10 @@ """ from unittest import TestCase +from metadata.generated.schema.entity.data.dashboard import Dashboard +from metadata.generated.schema.entity.data.table import Table from metadata.utils import entity_link +from metadata.utils.entity_link import get_entity_link class TestEntityLink(TestCase): @@ -107,3 +110,19 @@ def validate(self, fn_resp, check_split): ] for x in xs: x.validate(entity_link.split(x.entitylink), x.split_list) + + def test_get_entity_link(self): + """We can get entity link for different entities""" + + table_link = get_entity_link(Table, fqn="service.db.schema.table") + self.assertEqual(table_link, "<#E::table::service.db.schema.table>") + + dashboard_link = get_entity_link(Dashboard, fqn="service.dashboard") + self.assertEqual(dashboard_link, "<#E::dashboard::service.dashboard>") + + column_link = get_entity_link( + Table, fqn="service.db.schema.table", column_name="col" + ) + self.assertEqual( + column_link, "<#E::table::service.db.schema.table::columns::col>" + ) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java similarity index 99% rename from openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java rename to openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java index ea6202ad5f06..a9140756a53c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java @@ -60,7 +60,7 @@ @Slf4j @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class SuggestionResourceTest extends OpenMetadataApplicationTest { +public class SuggestionsResourceTest extends OpenMetadataApplicationTest { public static Table TABLE; public static Table TABLE2; From d6df074cc2eec0e4b023c33c7174222ef38a9e0a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 26 Jan 2024 12:49:11 +0100 Subject: [PATCH 23/49] Add update suggestion WIP --- .../ometa/mixins/suggestions_mixin.py | 42 ++++++++++++++++ .../src/metadata/ingestion/ometa/ometa_api.py | 2 + ingestion/src/metadata/utils/helpers.py | 20 ++++++++ .../ometa/test_ometa_suggestion_api.py | 26 ++++++++++ ingestion/tests/unit/test_helpers.py | 50 +++++++++++++++++++ 5 files changed, 140 insertions(+) create mode 100644 ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py new file mode 100644 index 000000000000..87640b71d1a8 --- /dev/null +++ b/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py @@ -0,0 +1,42 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Mixin class containing Suggestions specific methods + +To be used by OpenMetadata class +""" +from metadata.generated.schema.entity.feed.suggestion import Suggestion +from metadata.ingestion.ometa.client import REST +from metadata.utils.logger import ometa_logger + +logger = ometa_logger() + + +class OMetaSuggestionsMixin: + """ + OpenMetadata API methods related to the Suggestion Entity + + To be inherited by OpenMetadata + """ + + client: REST + + def update_suggestion(self, suggestion: Suggestion) -> Suggestion: + """ + Update an existing Suggestion with new fields + """ + # TODO: FIXME + resp = self.client.put( + f"{self.get_suffix(Suggestion)}/{str(suggestion.id.__root__)}", + data=suggestion.json(), + ) + + return Suggestion(**resp) diff --git a/ingestion/src/metadata/ingestion/ometa/ometa_api.py b/ingestion/src/metadata/ingestion/ometa/ometa_api.py index e17a302143ca..b791581bc935 100644 --- a/ingestion/src/metadata/ingestion/ometa/ometa_api.py +++ b/ingestion/src/metadata/ingestion/ometa/ometa_api.py @@ -50,6 +50,7 @@ from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin +from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin @@ -108,6 +109,7 @@ class OpenMetadata( OMetaRolePolicyMixin, OMetaSearchIndexMixin, OMetaCustomPropertyMixin, + OMetaSuggestionsMixin, Generic[T, C], ): """ diff --git a/ingestion/src/metadata/utils/helpers.py b/ingestion/src/metadata/utils/helpers.py index 55e878e421d1..94692ed1475b 100644 --- a/ingestion/src/metadata/utils/helpers.py +++ b/ingestion/src/metadata/utils/helpers.py @@ -31,7 +31,9 @@ from metadata.generated.schema.entity.data.chart import ChartType from metadata.generated.schema.entity.data.table import Column, Table +from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.type.basic import EntityLink from metadata.generated.schema.type.tagLabel import TagLabel from metadata.utils.constants import DEFAULT_DATABASE from metadata.utils.logger import utils_logger @@ -244,6 +246,24 @@ def equals(first: str, second: str) -> bool: ) +def find_suggestion( + suggestions: List[Suggestion], + suggestion_type: SuggestionType, + entity_link: EntityLink, +) -> Optional[Suggestion]: + """Given a list of suggestions, a suggestion type and an entity link, find + one suggestion in the list that matches the criteria + """ + return next( + ( + sugg + for sugg in suggestions + if sugg.type == suggestion_type and sugg.entityLink == entity_link + ), + None, + ) + + def find_column_in_table_with_index( column_name: str, table: Table ) -> Optional[Tuple[int, Column]]: diff --git a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py index 1fd8bb5e8f07..be0260543dda 100644 --- a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py @@ -170,3 +170,29 @@ def test_list(self): ) self.assertTrue(len(list(suggestions))) + + def test_update_suggestion(self): + """Update an existing suggestion""" + + create_table = get_create_entity( + entity=Table, + name=self.schema_name, + reference=self.schema.fullyQualifiedName.__root__, + ) + table: Table = self.metadata.create_or_update(create_table) + + suggestion_request = CreateSuggestionRequest( + description="something", + type=SuggestionType.SuggestDescription, + entityLink=EntityLink( + __root__=get_entity_link(Table, fqn=table.fullyQualifiedName.__root__) + ), + ) + + # Suggestions only support POST (not PUT) + res: Suggestion = self.metadata.create(suggestion_request) + self.assertEqual(res.description, "something") + + res.description = "new" + new = self.metadata.update_suggestion(res) + self.assertEqual(new.description, "new") diff --git a/ingestion/tests/unit/test_helpers.py b/ingestion/tests/unit/test_helpers.py index d84f3a97360a..ded406b82923 100644 --- a/ingestion/tests/unit/test_helpers.py +++ b/ingestion/tests/unit/test_helpers.py @@ -15,6 +15,8 @@ from unittest import TestCase from metadata.generated.schema.entity.data.table import Column, DataType, Table +from metadata.generated.schema.entity.feed.suggestion import Suggestion, SuggestionType +from metadata.generated.schema.type.basic import EntityLink from metadata.generated.schema.type.tagLabel import ( LabelType, State, @@ -24,6 +26,7 @@ from metadata.utils.helpers import ( clean_up_starting_ending_double_quotes_in_string, deep_size_of_dict, + find_suggestion, format_large_string_numbers, get_entity_tier_from_tags, is_safe_sql_query, @@ -154,3 +157,50 @@ def test_format_large_string_numbers(self): assert format_large_string_numbers(1000000) == "1.000M" assert format_large_string_numbers(1000000000) == "1.000B" assert format_large_string_numbers(1000000000000) == "1.000T" + + def test_find_suggestion(self): + """we can get one possible suggestion""" + suggestions = [ + Suggestion( + id=uuid.uuid4(), + type=SuggestionType.SuggestDescription, + entityLink=EntityLink(__root__="<#E::table::tableFQN>"), + description="something", + ), + Suggestion( + id=uuid.uuid4(), + type=SuggestionType.SuggestDescription, + entityLink=EntityLink(__root__="<#E::table::tableFQN::columns::col>"), + description="something", + ), + ] + + self.assertIsNone( + find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestTagLabel, + entity_link=..., + ) + ) + + self.assertIsNone( + find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestDescription, + entity_link=..., + ) + ) + + suggestion_table = find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestDescription, + entity_link=EntityLink(__root__="<#E::table::tableFQN>"), + ) + self.assertEqual(suggestion_table, suggestions[0]) + + suggestion_col = find_suggestion( + suggestions=suggestions, + suggestion_type=SuggestionType.SuggestDescription, + entity_link=EntityLink(__root__="<#E::table::tableFQN::columns::col>"), + ) + self.assertEqual(suggestion_col, suggestions[1]) From db88279ea1e94a665b87d35f4363ae76fb8c094f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 26 Jan 2024 14:26:00 +0100 Subject: [PATCH 24/49] Fix test --- ingestion/tests/integration/integration_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index af0aacdc1bf6..3043683d7b18 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -77,7 +77,7 @@ from metadata.ingestion.models.custom_pydantic import CustomSecretStr from metadata.ingestion.ometa.ometa_api import C, OpenMetadata, T from metadata.utils.dispatch import class_register -from src.metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.database import Database OM_JWT = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" From c999676a84535402cbc19c96a9071ae1da4efd8c Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Fri, 26 Jan 2024 23:35:50 -0800 Subject: [PATCH 25/49] Fix PUT and Pagination --- .../native/1.3.0/postgres/schemaChanges.sql | 12 +-- .../service/jdbi3/CollectionDAO.java | 4 +- .../service/jdbi3/SuggestionRepository.java | 14 +++- .../resources/feeds/SuggestionsResource.java | 13 ++-- .../feeds/SuggestionsResourceTest.java | 76 +++++++++++++------ 5 files changed, 81 insertions(+), 38 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql index 4a1bde76cccb..b9761443d963 100644 --- a/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.3.0/postgres/schemaChanges.sql @@ -182,13 +182,13 @@ DELETE FROM change_event_consumers; DELETE FROM consumers_dlq; CREATE TABLE IF NOT EXISTS suggestions ( - id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, fqnHash VARCHAR(256) NOT NULL, - entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.entityLink') NOT NULL, - suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.type') NOT NULL, + entityLink VARCHAR(256) GENERATED ALWAYS AS (json ->> 'entityLink') STORED NOT NULL, + suggestionType VARCHAR(36) GENERATED ALWAYS AS (json ->> 'type') STORED NOT NULL, json JSON NOT NULL, - updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL, - updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL, - status VARCHAR(256) GENERATED ALWAYS AS (json -> '$.status') NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + status VARCHAR(256) GENERATED ALWAYS AS (json ->> 'status') STORED NOT NULL, PRIMARY KEY (id) ); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 6e5452aff23d..71c205f4f53a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4382,10 +4382,10 @@ List listBefore( @Bind("before") String before); @ConnectionAwareSqlQuery( - value = "SELECT json FROM suggestions ORDER BY updatedAt LIMIT :limit", + value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", connectionType = MYSQL) @ConnectionAwareSqlQuery( - value = "SELECT json FROM suggestions ORDER BY updatedAt LIMIT :limit", + value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", connectionType = POSTGRES) List listAfter( @Define("mysqlCond") String mysqlCond, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java index 146c2846795a..dcf2a63bf1e0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -228,6 +228,16 @@ public RestUtil.PutResponse rejectSuggestion( return new RestUtil.PutResponse<>(Response.Status.OK, updatedHref, SUGGESTION_REJECTED); } + public void checkPermissionsForUpdateSuggestion( + Suggestion suggestion, SecurityContext securityContext) { + String userName = securityContext.getUserPrincipal().getName(); + User user = Entity.getEntityByName(USER, userName, TEAMS_FIELD, NON_DELETED); + if (Boolean.FALSE.equals(user.getIsAdmin()) && !userName.equalsIgnoreCase(suggestion.getCreatedBy().getName())) { + throw new AuthorizationException( + CatalogExceptionMessage.suggestionOperationNotAllowed(userName, "Update")); + } + } + public void checkPermissionsForAcceptOrRejectSuggestion( Suggestion suggestion, SuggestionStatus status, SecurityContext securityContext) { String userName = securityContext.getUserPrincipal().getName(); @@ -279,7 +289,7 @@ public ResultList listBefore(SuggestionFilter filter, int limit, Str String mySqlCondition = filter.getCondition(true); String postgresCondition = filter.getCondition(true); List jsons = - dao.suggestionDAO().listBefore(mySqlCondition, postgresCondition, limit, before); + dao.suggestionDAO().listBefore(mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(before)); List suggestions = getSuggestionList(jsons); String beforeCursor = null; String afterCursor; @@ -299,7 +309,7 @@ public ResultList listAfter(SuggestionFilter filter, int limit, Stri String mySqlCondition = filter.getCondition(true); String postgresCondition = filter.getCondition(true); List jsons = - dao.suggestionDAO().listAfter(mySqlCondition, postgresCondition, limit, after); + dao.suggestionDAO().listAfter(mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(after)); List suggestions = getSuggestionList(jsons); String beforeCursor; String afterCursor = null; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index f14bc70330c5..2666913265b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -168,9 +168,9 @@ public ResultList list( .build(); ResultList suggestions; if (before != null) { - suggestions = dao.listAfter(filter, limitParam, after); - } else { suggestions = dao.listBefore(filter, limitParam, before); + } else { + suggestions = dao.listAfter(filter, limitParam, after); } addHref(uriInfo, suggestions.getData()); return suggestions; @@ -269,16 +269,17 @@ public Response rejectSuggestion( @ExternalDocumentation( description = "JsonPatch RFC", url = "https://tools.ietf.org/html/rfc6902")) - @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) public Response updateSuggestion( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter(description = "Id of the Suggestion", schema = @Schema(type = "string")) @PathParam("id") - String id, + UUID id, @Valid Suggestion suggestion) { - suggestion.setCreatedBy(UserUtil.getUserOrBot(securityContext.getUserPrincipal().getName())); - suggestion.setCreatedAt(System.currentTimeMillis()); + Suggestion origSuggestion = dao.get(id); + dao.checkPermissionsForUpdateSuggestion(origSuggestion, securityContext); + suggestion.setCreatedAt(origSuggestion.getCreatedAt()); + suggestion.setCreatedBy(origSuggestion.getCreatedBy()); addHref(uriInfo, dao.update(suggestion, securityContext.getUserPrincipal().getName())); return Response.created(suggestion.getHref()) .entity(suggestion) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java index a9140756a53c..902b7bf4b973 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java @@ -1,10 +1,12 @@ package org.openmetadata.service.resources.feeds; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.resources.EntityResourceTest.C1; import static org.openmetadata.service.resources.EntityResourceTest.C2; @@ -199,49 +201,57 @@ void post_validSuggestionAndList_200(TestInfo test) throws IOException { // List all the threads and make sure the number of threads increased by 1 assertEquals( ++suggestionCount, - listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); } + SuggestionsResource.SuggestionList suggestionList = + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS); + assertEquals(suggestionCount, suggestionList.getPaging().getTotal()); + assertEquals(10, suggestionList.getData().size()); + suggestionList = + listSuggestions(TABLE.getFullyQualifiedName(), 10, null, suggestionList.getPaging().getAfter(), USER_AUTH_HEADERS); + assertEquals(1, suggestionList.getData().size()); + suggestionList = listSuggestions(TABLE.getFullyQualifiedName(), null, suggestionList.getPaging().getBefore(), null, USER_AUTH_HEADERS); + assertEquals(10, suggestionList.getData().size()); create = create().withEntityLink(TABLE_COLUMN1_LINK); createAndCheck(create, USER2_AUTH_HEADERS); create = create().withEntityLink(TABLE_COLUMN2_LINK); createAndCheck(create, USER2_AUTH_HEADERS); assertEquals( suggestionCount + 2, - listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); create = create().withEntityLink(TABLE2_LINK); createAndCheck(create, USER_AUTH_HEADERS); assertEquals( suggestionCount + 2, - listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); assertEquals( 1, - listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS) - .getPaging() - .getTotal()); - assertEquals( - 2, - listSuggestions( - TABLE.getFullyQualifiedName(), - null, - USER_AUTH_HEADERS, - USER2.getId(), - null, - null, - null, - null) + listSuggestions(TABLE2.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); + suggestionList = listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + USER2.getId(), + null, + null, + null, + null); + assertEquals(2, suggestionList.getPaging().getTotal()); + assertNull(suggestionList.getPaging().getBefore()); + assertNull(suggestionList.getPaging().getAfter()); create = create().withEntityLink(TABLE_WITHOUT_OWNER_LINK); createAndCheck(create, USER_AUTH_HEADERS); assertEquals( 1, - listSuggestions(TABLE_WITHOUT_OWNER.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + listSuggestions(TABLE_WITHOUT_OWNER.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); /* deleteSuggestions("table", TABLE.getFullyQualifiedName(), USER_AUTH_HEADERS); @@ -254,6 +264,23 @@ void post_validSuggestionAndList_200(TestInfo test) throws IOException { listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS).getPaging().getTotal());*/ } + @Test + void put_updateSuggestion_200(TestInfo test) throws IOException { + CreateSuggestion create = create(); + Suggestion suggestion = createSuggestion(create, USER_AUTH_HEADERS); + Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); + suggestion.setDescription("updated description"); + updateSuggestion(suggestion.getId(), suggestion, USER_AUTH_HEADERS); + Suggestion updatedSuggestion = getSuggestion(suggestion.getId(), USER_AUTH_HEADERS); + assertEquals(suggestion.getId(), updatedSuggestion.getId()); + assertEquals(suggestion.getDescription(), updatedSuggestion.getDescription()); + updatedSuggestion.setDescription("updated description with different user"); + assertResponse( + () -> updateSuggestion(updatedSuggestion.getId(), updatedSuggestion, USER2_AUTH_HEADERS), + FORBIDDEN, + CatalogExceptionMessage.taskOperationNotAllowed(USER2.getName(), "Update")); + } + @Test @Order(1) void put_acceptSuggestion_200(TestInfo test) throws IOException { @@ -311,7 +338,7 @@ void put_rejectSuggestion_200(TestInfo test) throws IOException { Assertions.assertEquals(create.getEntityLink(), suggestion.getEntityLink()); assertEquals( 1, - listSuggestions(TABLE.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + listSuggestions(TABLE.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); rejectSuggestion(suggestion.getId(), USER_AUTH_HEADERS); @@ -322,7 +349,7 @@ void put_rejectSuggestion_200(TestInfo test) throws IOException { Assertions.assertEquals(create1.getEntityLink(), suggestion1.getEntityLink()); assertEquals( 1, - listSuggestions(TABLE2.getFullyQualifiedName(), null, USER_AUTH_HEADERS) + listSuggestions(TABLE2.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); assertResponse( @@ -339,6 +366,11 @@ public Suggestion createSuggestion(CreateSuggestion create, Map return TestUtils.post(getResource("suggestions"), create, Suggestion.class, authHeaders); } + public void updateSuggestion(UUID id, Suggestion update, Map authHeaders) + throws HttpResponseException { + TestUtils.put(getResource("suggestions/" + id), update, CREATED, authHeaders); + } + public CreateSuggestion create() { return new CreateSuggestion() .withDescription("Update description") @@ -401,9 +433,9 @@ public SuggestionsResource.SuggestionList listSuggestions( } public SuggestionsResource.SuggestionList listSuggestions( - String entityFQN, Integer limit, Map authHeaders) + String entityFQN, Integer limit, String before, String after, Map authHeaders) throws HttpResponseException { - return listSuggestions(entityFQN, limit, authHeaders, null, null, null, null, null); + return listSuggestions(entityFQN, limit, authHeaders, null, null, null, before, after); } public Suggestion createAndCheck(CreateSuggestion create, Map authHeaders) From 233fc2e2779b90b29c9f1e7718f20bf2f8d20ece Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Fri, 26 Jan 2024 23:57:15 -0800 Subject: [PATCH 26/49] Fix styling --- .../service/jdbi3/SuggestionRepository.java | 10 +++-- .../resources/feeds/SuggestionsResource.java | 2 +- .../feeds/SuggestionsResourceTest.java | 39 ++++++++++++------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java index dcf2a63bf1e0..7b442f655416 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SuggestionRepository.java @@ -232,7 +232,8 @@ public void checkPermissionsForUpdateSuggestion( Suggestion suggestion, SecurityContext securityContext) { String userName = securityContext.getUserPrincipal().getName(); User user = Entity.getEntityByName(USER, userName, TEAMS_FIELD, NON_DELETED); - if (Boolean.FALSE.equals(user.getIsAdmin()) && !userName.equalsIgnoreCase(suggestion.getCreatedBy().getName())) { + if (Boolean.FALSE.equals(user.getIsAdmin()) + && !userName.equalsIgnoreCase(suggestion.getCreatedBy().getName())) { throw new AuthorizationException( CatalogExceptionMessage.suggestionOperationNotAllowed(userName, "Update")); } @@ -289,7 +290,9 @@ public ResultList listBefore(SuggestionFilter filter, int limit, Str String mySqlCondition = filter.getCondition(true); String postgresCondition = filter.getCondition(true); List jsons = - dao.suggestionDAO().listBefore(mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(before)); + dao.suggestionDAO() + .listBefore( + mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(before)); List suggestions = getSuggestionList(jsons); String beforeCursor = null; String afterCursor; @@ -309,7 +312,8 @@ public ResultList listAfter(SuggestionFilter filter, int limit, Stri String mySqlCondition = filter.getCondition(true); String postgresCondition = filter.getCondition(true); List jsons = - dao.suggestionDAO().listAfter(mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(after)); + dao.suggestionDAO() + .listAfter(mySqlCondition, postgresCondition, limit + 1, RestUtil.decodeCursor(after)); List suggestions = getSuggestionList(jsons); String beforeCursor; String afterCursor = null; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index 2666913265b0..6e74397708d8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -277,7 +277,7 @@ public Response updateSuggestion( UUID id, @Valid Suggestion suggestion) { Suggestion origSuggestion = dao.get(id); - dao.checkPermissionsForUpdateSuggestion(origSuggestion, securityContext); + dao.checkPermissionsForUpdateSuggestion(origSuggestion, securityContext); suggestion.setCreatedAt(origSuggestion.getCreatedAt()); suggestion.setCreatedBy(origSuggestion.getCreatedBy()); addHref(uriInfo, dao.update(suggestion, securityContext.getUserPrincipal().getName())); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java index 902b7bf4b973..ae90ddb55083 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/SuggestionsResourceTest.java @@ -210,9 +210,20 @@ void post_validSuggestionAndList_200(TestInfo test) throws IOException { assertEquals(suggestionCount, suggestionList.getPaging().getTotal()); assertEquals(10, suggestionList.getData().size()); suggestionList = - listSuggestions(TABLE.getFullyQualifiedName(), 10, null, suggestionList.getPaging().getAfter(), USER_AUTH_HEADERS); + listSuggestions( + TABLE.getFullyQualifiedName(), + 10, + null, + suggestionList.getPaging().getAfter(), + USER_AUTH_HEADERS); assertEquals(1, suggestionList.getData().size()); - suggestionList = listSuggestions(TABLE.getFullyQualifiedName(), null, suggestionList.getPaging().getBefore(), null, USER_AUTH_HEADERS); + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + suggestionList.getPaging().getBefore(), + null, + USER_AUTH_HEADERS); assertEquals(10, suggestionList.getData().size()); create = create().withEntityLink(TABLE_COLUMN1_LINK); createAndCheck(create, USER2_AUTH_HEADERS); @@ -235,15 +246,16 @@ void post_validSuggestionAndList_200(TestInfo test) throws IOException { listSuggestions(TABLE2.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); - suggestionList = listSuggestions( - TABLE.getFullyQualifiedName(), - null, - USER_AUTH_HEADERS, - USER2.getId(), - null, - null, - null, - null); + suggestionList = + listSuggestions( + TABLE.getFullyQualifiedName(), + null, + USER_AUTH_HEADERS, + USER2.getId(), + null, + null, + null, + null); assertEquals(2, suggestionList.getPaging().getTotal()); assertNull(suggestionList.getPaging().getBefore()); assertNull(suggestionList.getPaging().getAfter()); @@ -251,7 +263,8 @@ void post_validSuggestionAndList_200(TestInfo test) throws IOException { createAndCheck(create, USER_AUTH_HEADERS); assertEquals( 1, - listSuggestions(TABLE_WITHOUT_OWNER.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) + listSuggestions( + TABLE_WITHOUT_OWNER.getFullyQualifiedName(), null, null, null, USER_AUTH_HEADERS) .getPaging() .getTotal()); /* deleteSuggestions("table", TABLE.getFullyQualifiedName(), USER_AUTH_HEADERS); @@ -368,7 +381,7 @@ public Suggestion createSuggestion(CreateSuggestion create, Map public void updateSuggestion(UUID id, Suggestion update, Map authHeaders) throws HttpResponseException { - TestUtils.put(getResource("suggestions/" + id), update, CREATED, authHeaders); + TestUtils.put(getResource("suggestions/" + id), update, CREATED, authHeaders); } public CreateSuggestion create() { From fb9636d32196baafa4566eadda3b65865eb76e84 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 29 Jan 2024 11:19:37 +0100 Subject: [PATCH 27/49] update test --- ingestion/tests/integration/ometa/test_ometa_suggestion_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py index be0260543dda..634634c7f774 100644 --- a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py @@ -169,7 +169,7 @@ def test_list(self): }, ) - self.assertTrue(len(list(suggestions))) + self.assertEqual(len(list(suggestions)), 1) def test_update_suggestion(self): """Update an existing suggestion""" From 9633d47070c6c3752dd54dd14e4d650c91525584 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 29 Jan 2024 13:04:26 +0100 Subject: [PATCH 28/49] Update status --- ingestion/src/metadata/ingestion/api/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ingestion/src/metadata/ingestion/api/status.py b/ingestion/src/metadata/ingestion/api/status.py index 49068e79aca6..8e5089d30601 100644 --- a/ingestion/src/metadata/ingestion/api/status.py +++ b/ingestion/src/metadata/ingestion/api/status.py @@ -56,6 +56,10 @@ def scanned(self, record: Any) -> None: else: self.records.append(log_name) + def updated(self, record: Any) -> None: + if log_name := get_log_name(record): + self.updated_records.append(log_name) + def warning(self, key: str, reason: str) -> None: self.warnings.append({key: reason}) From 6cb3d9933410a588d55ea4a286bc30869a674fac Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 29 Jan 2024 13:04:51 +0100 Subject: [PATCH 29/49] add OM server connection in apps --- .../service/resources/apps/AppResource.java | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index b96ec1df2efb..fc94e0c502d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -139,6 +139,9 @@ public void initialize(OpenMetadataApplicationConfig config) { // Schedule if (app.getScheduleType().equals(ScheduleType.Scheduled)) { + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } } @@ -571,10 +574,15 @@ public Response patchApplication( App app = repository.get(null, id, repository.getFields("bot,pipelines")); AppScheduler.getInstance().deleteScheduledApplication(app); Response response = patchInternal(uriInfo, securityContext, id, patch); + App updatedApp = (App) response.getEntity(); + updatedApp.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication( - (App) response.getEntity(), Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication(updatedApp, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + updatedApp.setOpenMetadataServerConnection(null); return response; } @@ -604,9 +612,14 @@ public Response createOrUpdate( new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); AppScheduler.getInstance().deleteScheduledApplication(app); + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + app.setOpenMetadataServerConnection(null); return createOrUpdate(uriInfo, securityContext, app); } @@ -684,9 +697,14 @@ public Response restoreApp( Response response = restoreEntity(uriInfo, securityContext, restore.getId()); if (response.getStatus() == Response.Status.OK.getStatusCode()) { App app = (App) response.getEntity(); + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } + // We don't want to store this information + app.setOpenMetadataServerConnection(null); } return response; } @@ -717,6 +735,9 @@ public Response scheduleApplication( @Context SecurityContext securityContext) { App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); From a350230016e55acb738fa40f71e32abf76aa08cc Mon Sep 17 00:00:00 2001 From: karanh37 Date: Mon, 29 Jan 2024 18:52:42 +0530 Subject: [PATCH 30/49] add permissions check --- .../MetaPilotDescriptionAlert.component.tsx | 41 ++++++++++--------- .../MetaPilotDescriptionAlert.interface.ts | 1 + .../MetaPilotProvider.interface.ts | 2 + .../MetaPilotProvider/MetaPilotProvider.tsx | 34 ++++++++++++--- .../TableDescription.component.tsx | 9 +++- .../EntityDescription/DescriptionV1.tsx | 14 ++++++- .../TableDetailsPageV1/TableDetailsPageV1.tsx | 14 +++---- 7 files changed, 80 insertions(+), 35 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx index 1abd3172f206..4dc382a80feb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.component.tsx @@ -24,6 +24,7 @@ import { MetaPilotDescriptionAlertProps } from './MetaPilotDescriptionAlert.inte const MetaPilotDescriptionAlert = ({ showHeading = true, suggestion, + hasEditAccess = false, }: MetaPilotDescriptionAlertProps) => { const { t } = useTranslation(); const { onUpdateActiveSuggestion, acceptRejectSuggestion } = @@ -65,25 +66,27 @@ const MetaPilotDescriptionAlert = ({ onUpdateActiveSuggestion(undefined)} /> -
- - -
+ {hasEditAccess && ( +
+ + +
+ )} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts index dc2ff6138890..b47460693138 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotDescriptionAlert/MetaPilotDescriptionAlert.interface.ts @@ -15,4 +15,5 @@ import { Suggestion } from '../../../generated/entity/feed/suggestion'; export interface MetaPilotDescriptionAlertProps { showHeading?: boolean; suggestion: Suggestion; + hasEditAccess?: boolean; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts index 8950dc1f056e..4ee0ed9f6075 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts @@ -22,6 +22,7 @@ export interface MetaPilotContextType { suggestions: Suggestion[]; loading: boolean; entityFqn: string; + refreshEntity: (() => void) | undefined; onUpdateActiveSuggestion: (suggestion?: Suggestion) => void; fetchSuggestions: (entityFqn: string) => void; acceptRejectSuggestion: ( @@ -30,6 +31,7 @@ export interface MetaPilotContextType { ) => void; onUpdateEntityFqn: (entityFqn: string) => void; resetMetaPilot: () => void; + initMetaPilot: (entityFqn: string, refreshEntity?: () => void) => void; } export interface MetaPilotContextProps { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx index 70882044ee4d..8783915bfe34 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx @@ -12,6 +12,7 @@ */ import { Button } from 'antd'; import { AxiosError } from 'axios'; +import { isEmpty } from 'lodash'; import React, { createContext, useCallback, @@ -28,6 +29,7 @@ import { updateSuggestionStatus, } from '../../../rest/suggestionsAPI'; import { showErrorToast } from '../../../utils/ToastUtils'; +import { usePermissionProvider } from '../../PermissionProvider/PermissionProvider'; import { MetaPilotContextProps, MetaPilotContextType, @@ -46,6 +48,8 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { const [entityFqn, setEntityFqn] = useState(''); const [suggestions, setSuggestions] = useState([]); const [loading, setLoading] = useState(false); + const [refreshEntity, setRefreshEntity] = useState<() => void>(); + const { permissions } = usePermissionProvider(); const fetchSuggestions = useCallback(async (entityFQN: string) => { setLoading(true); @@ -71,11 +75,16 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { try { await updateSuggestionStatus(suggestion, status); await fetchSuggestions(entityFqn); + + setActiveSuggestion(undefined); + if (status === SuggestionAction.Accept) { + refreshEntity?.(); + } } catch (err) { showErrorToast(err as AxiosError); } }, - [entityFqn] + [entityFqn, refreshEntity] ); const onToggleSuggestionsVisible = useCallback((state: boolean) => { @@ -94,16 +103,27 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { setEntityFqn(entityFqn); }, []); - const resetMetaPilot = () => { + const resetMetaPilot = useCallback(() => { setSuggestionsVisible(false); setIsMetaPilotEnabled(false); setActiveSuggestion(undefined); setEntityFqn(''); - }; + }, []); + + const initMetaPilot = useCallback( + (entityFqn: string, refreshEntity?: () => void) => { + setIsMetaPilotEnabled(true); + setEntityFqn(entityFqn); + setRefreshEntity(() => refreshEntity); + }, + [] + ); useEffect(() => { - fetchSuggestions(entityFqn); - }, [entityFqn]); + if (!isEmpty(permissions) && !isEmpty(entityFqn)) { + fetchSuggestions(entityFqn); + } + }, [permissions, entityFqn]); const metaPilotContextObj = useMemo(() => { return { @@ -113,12 +133,14 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { activeSuggestion, entityFqn, loading, + refreshEntity, onToggleSuggestionsVisible, onUpdateEntityFqn, onMetaPilotEnableUpdate, onUpdateActiveSuggestion, fetchSuggestions, acceptRejectSuggestion, + initMetaPilot, resetMetaPilot, }; }, [ @@ -128,12 +150,14 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { activeSuggestion, entityFqn, loading, + refreshEntity, onToggleSuggestionsVisible, onUpdateEntityFqn, onMetaPilotEnableUpdate, onUpdateActiveSuggestion, fetchSuggestions, acceptRejectSuggestion, + initMetaPilot, resetMetaPilot, ]); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx index f9429d62953a..0dfcfec65905 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TableDescription/TableDescription.component.tsx @@ -64,6 +64,7 @@ const TableDescription = ({ if (activeSuggestion?.entityLink === entityLink) { return ( @@ -71,7 +72,13 @@ const TableDescription = ({ } if (suggestionForEmptyData) { - return ; + return ( + + ); } return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx index a2eef6269b1d..5120f429e5bb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx @@ -183,11 +183,21 @@ const DescriptionV1 = ({ }, [suggestions, description]); if (activeSuggestion?.entityLink === entityLinkWithoutField) { - return ; + return ( + + ); } if (suggestionForEmptyData) { - return ; + return ( + + ); } const content = ( diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx index 4a21ddb663bb..b6bb15afe076 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx @@ -115,8 +115,7 @@ const TableDetailsPageV1 = () => { ThreadType.Conversation ); const [queryCount, setQueryCount] = useState(0); - const { onMetaPilotEnableUpdate, onUpdateEntityFqn, resetMetaPilot } = - useMetaPilotContext(); + const { resetMetaPilot, initMetaPilot } = useMetaPilotContext(); const [loading, setLoading] = useState(!isTourOpen); const [tablePermissions, setTablePermissions] = useState( @@ -138,7 +137,7 @@ const TableDetailsPageV1 = () => { [datasetFQN] ); - const fetchTableDetails = async () => { + const fetchTableDetails = useCallback(async () => { setLoading(true); try { let fields = defaultFields; @@ -162,7 +161,7 @@ const TableDetailsPageV1 = () => { } finally { setLoading(false); } - }; + }, [tableFqn]); const fetchQueryCount = async () => { if (!tableDetails?.id) { @@ -281,16 +280,15 @@ const TableDetailsPageV1 = () => { ); useEffect(() => { - if (tableFqn) { + if (tableFqn && fetchTableDetails) { fetchResourcePermission(tableFqn); - onMetaPilotEnableUpdate(true); - onUpdateEntityFqn(tableFqn); + initMetaPilot(tableFqn, fetchTableDetails); } return () => { resetMetaPilot(); }; - }, [tableFqn]); + }, [tableFqn, fetchTableDetails]); const handleFeedCount = useCallback((data: FeedCounts) => { setFeedCount(data); From 128645ca8208d8ff780511745fb9737e481a7cef Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 29 Jan 2024 15:57:15 +0100 Subject: [PATCH 31/49] Fix CI --- ingestion/tests/integration/integration_base.py | 1 - .../tests/integration/ometa/test_ometa_suggestion_api.py | 3 +-- .../openmetadata/service/resources/apps/AppResource.java | 6 ++++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index 3043683d7b18..1404a530edd2 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -77,7 +77,6 @@ from metadata.ingestion.models.custom_pydantic import CustomSecretStr from metadata.ingestion.ometa.ometa_api import C, OpenMetadata, T from metadata.utils.dispatch import class_register -from metadata.generated.schema.entity.data.database import Database OM_JWT = "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py index 634634c7f774..0d3da0ab62bc 100644 --- a/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_suggestion_api.py @@ -75,7 +75,7 @@ def setUpClass(cls) -> None: create_table = get_create_entity( entity=Table, - name=cls.schema_name, + name=cls.table_name, reference=cls.schema.fullyQualifiedName.__root__, ) cls.table: Table = cls.metadata.create_or_update(create_table) @@ -145,7 +145,6 @@ def test_list(self): create_table = get_create_entity( entity=Table, - name=self.schema_name, reference=self.schema.fullyQualifiedName.__root__, ) table: Table = self.metadata.create_or_update(create_table) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index fc94e0c502d6..b808431b85ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -140,7 +140,8 @@ public void initialize(OpenMetadataApplicationConfig config) { // Schedule if (app.getScheduleType().equals(ScheduleType.Scheduled)) { app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + new OpenMetadataConnectionBuilder( + openMetadataApplicationConfig, app.getBot().getName()) .build()); ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } @@ -579,7 +580,8 @@ public Response patchApplication( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(updatedApp, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication( + updatedApp, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information updatedApp.setOpenMetadataServerConnection(null); From 91cdf62381d3dcd3fb88f7e2d9fb60293de8e573 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 29 Jan 2024 17:49:32 +0100 Subject: [PATCH 32/49] Remove TODO --- .../src/metadata/ingestion/ometa/mixins/suggestions_mixin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py index 87640b71d1a8..c2fc27cb1d10 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/suggestions_mixin.py @@ -33,7 +33,6 @@ def update_suggestion(self, suggestion: Suggestion) -> Suggestion: """ Update an existing Suggestion with new fields """ - # TODO: FIXME resp = self.client.put( f"{self.get_suffix(Suggestion)}/{str(suggestion.id.__root__)}", data=suggestion.json(), From 7645191d6f4fe683f624455f59ad64476dab5568 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 29 Jan 2024 12:17:52 -0800 Subject: [PATCH 33/49] Fix feedResourceTest --- .../service/resources/feeds/FeedResourceTest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java index faad55062af3..a2212ec285f9 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java @@ -34,6 +34,7 @@ import static org.openmetadata.service.exception.CatalogExceptionMessage.permissionNotAllowed; import static org.openmetadata.service.resources.EntityResourceTest.C1; import static org.openmetadata.service.resources.EntityResourceTest.USER1; +import static org.openmetadata.service.resources.EntityResourceTest.USER2_REF; import static org.openmetadata.service.resources.EntityResourceTest.USER_ADDRESS_TAG_LABEL; import static org.openmetadata.service.security.SecurityUtil.authHeaders; import static org.openmetadata.service.security.SecurityUtil.getPrincipalName; @@ -559,12 +560,15 @@ void post_invalidAnnouncement_400() throws IOException { } @Test - void put_resolveTaskByUser_description_200() throws IOException { + void put_resolveTaskByUser_description_200(TestInfo testInfo) throws IOException { + TableResourceTest tableResourceTest = new TableResourceTest(); + CreateTable createTable = tableResourceTest.createRequest(testInfo).withOwner(USER2_REF); + Table table = tableResourceTest.createAndCheckEntity(createTable, ADMIN_AUTH_HEADERS); // Create a task from User to User2 String about = String.format( "<#E::%s::%s::columns::%s::description>", - Entity.TABLE, TABLE.getFullyQualifiedName(), C1); + Entity.TABLE, table.getFullyQualifiedName(), C1); Thread taskThread = createTaskThread( USER.getName(), @@ -588,7 +592,7 @@ void put_resolveTaskByUser_description_200() throws IOException { // User2 who is assigned the task can resolve the task resolveTask(taskId, resolveTask, USER2_AUTH_HEADERS); - Table table = TABLE_RESOURCE_TEST.getEntity(TABLE.getId(), null, USER_AUTH_HEADERS); + table = TABLE_RESOURCE_TEST.getEntity(table.getId(), null, USER_AUTH_HEADERS); assertEquals("accepted", EntityUtil.getColumn(table, (C1)).getDescription()); taskThread = getTask(taskId, USER_AUTH_HEADERS); From 3e9f8377aef10e09900b63896efa346fac03e8fe Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 30 Jan 2024 11:46:14 +0530 Subject: [PATCH 34/49] fix unit tests --- .../GlossaryDetails/GlossaryDetails.test.tsx | 4 ++++ .../TopicDetails/TopicSchema/TopicSchema.test.tsx | 6 ++++++ .../src/components/Users/Users.component.test.tsx | 8 ++++++++ .../TableDetailsPageV1/TableDetailsPageV1.test.tsx | 13 ++++++++++++- 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx index 3d562cefd175..1c36b0b6dc24 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx @@ -53,6 +53,10 @@ jest.mock( } ); +jest.mock('../../common/EntityDescription/DescriptionV1', () => + jest.fn().mockImplementation(() =>
DescriptionV1
) +); + const mockProps = { glossary: mockedGlossaries[0], glossaryTerms: [], diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicSchema/TopicSchema.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicSchema/TopicSchema.test.tsx index e7b26b283bcd..773239e519a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicSchema/TopicSchema.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TopicDetails/TopicSchema/TopicSchema.test.tsx @@ -74,6 +74,12 @@ jest.mock('../../TableTags/TableTags.component', () => )) ); +jest.mock('../../MetaPilot/MetaPilotProvider/MetaPilotProvider', () => ({ + useMetaPilotContext: jest.fn().mockReturnValue({ + suggestions: [], + }), +})); + jest.mock('../../common/ErrorWithPlaceholder/ErrorPlaceHolder', () => jest .fn() diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx index 52beca237e8b..ba28e7bd49a0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Users/Users.component.test.tsx @@ -43,6 +43,14 @@ jest.mock('react-router-dom', () => ({ useParams: jest.fn().mockImplementation(() => mockParams), })); +jest.mock('../MetaPilot/MetaPilotProvider/MetaPilotProvider', () => ({ + useMetaPilotContext: jest.fn().mockReturnValue({ + suggestions: [], + initMetaPilot: jest.fn(), + resetMetaPilot: jest.fn(), + }), +})); + jest.mock('../../rest/rolesAPIV1', () => ({ getRoles: jest.fn().mockImplementation(() => Promise.resolve(mockUserRole)), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx index 0abaa7d2e046..6eae69704600 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx @@ -30,6 +30,17 @@ jest.mock('../../components/PermissionProvider/PermissionProvider', () => ({ })), })); +jest.mock( + '../../components/MetaPilot/MetaPilotProvider/MetaPilotProvider', + () => ({ + useMetaPilotContext: jest.fn().mockReturnValue({ + suggestions: [], + initMetaPilot: jest.fn(), + resetMetaPilot: jest.fn(), + }), + }) +); + jest.mock('../../rest/tableAPI', () => ({ getTableDetailsByFQN: jest.fn().mockImplementation(() => Promise.resolve({ @@ -215,7 +226,7 @@ describe('TestDetailsPageV1 component', () => { }); expect(getTableDetailsByFQN).toHaveBeenCalledWith('fqn', { - fields: `${COMMON_API_FIELDS},usageSummary`, + fields: `${COMMON_API_FIELDS}`, }); }); From 221b3cf5bfc66b526bf7caafba5cb6416f28c586 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 30 Jan 2024 15:00:58 +0100 Subject: [PATCH 35/49] add private configs for apps --- conf/openmetadata.yaml | 9 +++- .../OpenMetadataApplicationConfig.java | 4 ++ .../apps/AbstractNativeApplication.java | 32 +++++++++++++- .../service/apps/ApplicationHandler.java | 39 +++++++++++++---- .../service/apps/NativeApplication.java | 7 ++- .../bundles/insights/DataInsightsApp.java | 8 ---- .../bundles/searchIndex/SearchIndexApp.java | 9 +++- .../bundles/test/NoOpTestApplication.java | 9 +++- .../service/apps/scheduler/AppScheduler.java | 18 ++++++-- .../service/resources/apps/AppResource.java | 32 +++++++++----- .../service/util/OpenMetadataOperations.java | 2 +- .../configuration/appsConfiguration.json | 43 +++++++++++++++++++ .../json/schema/entity/applications/app.json | 4 ++ .../configuration/applicationConfig.json | 7 +++ .../external/metaPilotAppConfig.json | 19 -------- .../external/metaPilotAppPrivateConfig.json | 31 +++++++++++++ .../MetaPilotApplication.json | 19 -------- 17 files changed, 214 insertions(+), 78 deletions(-) create mode 100644 openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 793cbc41a79f..d7de8ea77298 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -369,4 +369,11 @@ web: permission-policy: enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false} option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""} - + +#applications: +# appsConfiguration: # Defined as private configs +# - name: MetaPilotApplication +# parameters: +# waiiInstance: waiiInstance +# token: token +# collateURL: collateURL diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java index 2874d41d6a0b..ca145ca63d89 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java @@ -22,6 +22,7 @@ import javax.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration; import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; import org.openmetadata.schema.api.fernet.FernetConfiguration; @@ -93,6 +94,9 @@ public class OpenMetadataApplicationConfig extends Configuration { @JsonProperty("web") private OMWebConfiguration webConfiguration = new OMWebConfiguration(); + @JsonProperty("applications") + private AppsPrivateConfiguration appsPrivateConfiguration; + @Override public String toString() { return "catalogConfig{" diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index cfba9b91aee6..10f809511a5d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -1,7 +1,9 @@ package org.openmetadata.service.apps; import static com.cronutils.model.CronType.QUARTZ; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.JOB_LISTENER_NAME; +import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_PRIVATE_CONFIG_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.COLLECTION_DAO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.SEARCH_CLIENT_KEY; @@ -16,6 +18,9 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.AppRuntime; +import org.openmetadata.schema.api.configuration.apps.AppConfig; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; +import org.openmetadata.schema.api.configuration.apps.Parameters; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; @@ -50,6 +55,7 @@ public class AbstractNativeApplication implements NativeApplication { protected CollectionDAO collectionDAO; private @Getter App app; + private @Getter Parameters privateParameters = null; protected SearchRepository searchRepository; private final @Getter CronMapper cronMapper = CronMapper.fromQuartzToUnix(); private final @Getter CronParser cronParser = @@ -59,10 +65,29 @@ public class AbstractNativeApplication implements NativeApplication { private static final String SERVICE_NAME = "OpenMetadata"; @Override - public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { + public void init( + App app, + CollectionDAO dao, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration) { this.collectionDAO = dao; this.searchRepository = searchRepository; this.app = app; + + this.loadPrivateParameters(privateConfiguration); + } + + /** + * Load the apps' private parameters, if needed + */ + private void loadPrivateParameters(AppsPrivateConfiguration privateConfiguration) { + if (privateConfiguration != null && !nullOrEmpty(privateConfiguration.getAppsConfiguration())) { + for (AppConfig appConfig : privateConfiguration.getAppsConfiguration()) { + if (this.app.getName().equals(appConfig.getName())) { + this.privateParameters = appConfig.getParameters(); + } + } + } } @Override @@ -211,8 +236,11 @@ public void execute(JobExecutionContext jobExecutionContext) { SearchRepository searchRepositoryForJob = (SearchRepository) jobExecutionContext.getJobDetail().getJobDataMap().get(SEARCH_CLIENT_KEY); + AppsPrivateConfiguration privateConfiguration = + (AppsPrivateConfiguration) + jobExecutionContext.getJobDetail().getJobDataMap().get(APPS_PRIVATE_CONFIG_KEY); // Initialise the Application - this.init(jobApp, dao, searchRepositoryForJob); + this.init(jobApp, dao, searchRepositoryForJob, privateConfiguration); // Trigger this.startApp(jobExecutionContext); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 35b3afa19c20..619116a50039 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -3,6 +3,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -16,23 +17,38 @@ private ApplicationHandler() { } public static void triggerApplicationOnDemand( - App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand"); + App app, + CollectionDAO daoCollection, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration) { + runMethodFromApplication( + app, daoCollection, searchRepository, privateConfiguration, "triggerOnDemand"); } public static void installApplication( - App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "install"); + App app, + CollectionDAO daoCollection, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration) { + runMethodFromApplication(app, daoCollection, searchRepository, privateConfiguration, "install"); } public static void configureApplication( - App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "configure"); + App app, + CollectionDAO daoCollection, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration) { + runMethodFromApplication( + app, daoCollection, searchRepository, privateConfiguration, "configure"); } /** Load an App from its className and call its methods dynamically */ public static void runMethodFromApplication( - App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) { + App app, + CollectionDAO daoCollection, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration, + String methodName) { // Native Application try { Class clz = Class.forName(app.getClassName()); @@ -42,8 +58,13 @@ public static void runMethodFromApplication( Method initMethod = resource .getClass() - .getMethod("init", App.class, CollectionDAO.class, SearchRepository.class); - initMethod.invoke(resource, app, daoCollection, searchRepository); + .getMethod( + "init", + App.class, + CollectionDAO.class, + SearchRepository.class, + AppsPrivateConfiguration.class); + initMethod.invoke(resource, app, daoCollection, searchRepository, privateConfiguration); // Call method on demand Method scheduleMethod = resource.getClass().getMethod(methodName); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java index e0ed3e228746..b5ff82dcf867 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java @@ -1,5 +1,6 @@ package org.openmetadata.service.apps; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.search.SearchRepository; @@ -7,7 +8,11 @@ import org.quartz.JobExecutionContext; public interface NativeApplication extends Job { - void init(App app, CollectionDAO dao, SearchRepository searchRepository); + void init( + App app, + CollectionDAO dao, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration); void install(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index a05f64f31f8b..f50c977df569 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -4,7 +4,6 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; -import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.services.ServiceType; import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; @@ -20,7 +19,6 @@ import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; -import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.util.FullyQualifiedName; @Slf4j @@ -31,12 +29,6 @@ public class DataInsightsApp extends AbstractNativeApplication { private static final String SERVICE_TYPE = "Metadata"; private static final String PIPELINE_DESCRIPTION = "OpenMetadata DataInsight Pipeline"; - @Override - public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { - super.init(app, dao, searchRepository); - LOG.info("Data Insights App is initialized"); - } - @Override public void install() { IngestionPipelineRepository ingestionPipelineRepository = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 9f5682ecd2ad..2a9bc71e081f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -17,6 +17,7 @@ import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.analytics.ReportData; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppRunType; @@ -94,8 +95,12 @@ public class SearchIndexApp extends AbstractNativeApplication { private volatile boolean stopped = false; @Override - public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { - super.init(app, dao, searchRepository); + public void init( + App app, + CollectionDAO dao, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration) { + super.init(app, dao, searchRepository, privateConfiguration); // request for reindexing EventPublisherJob request = JsonUtils.convertValue(app.getAppConfiguration(), EventPublisherJob.class) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java index 94a0c0e8a7a3..73c68372fe7c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java @@ -1,6 +1,7 @@ package org.openmetadata.service.apps.bundles.test; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.apps.AbstractNativeApplication; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -10,8 +11,12 @@ public class NoOpTestApplication extends AbstractNativeApplication { @Override - public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { - super.init(app, dao, searchRepository); + public void init( + App app, + CollectionDAO dao, + SearchRepository searchRepository, + AppsPrivateConfiguration privateConfiguration) { + super.init(app, dao, searchRepository, privateConfiguration); LOG.info("NoOpTestApplication is initialized"); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java index 6a8134c82f9e..fa5067947ce0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java @@ -9,6 +9,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.AppRuntime; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunType; import org.openmetadata.schema.entity.app.AppSchedule; @@ -35,16 +36,23 @@ public class AppScheduler { public static final String APP_INFO_KEY = "applicationInfoKey"; public static final String COLLECTION_DAO_KEY = "daoKey"; public static final String SEARCH_CLIENT_KEY = "searchClientKey"; + public static final String APPS_PRIVATE_CONFIG_KEY = "privateConfigurationKey"; private static AppScheduler instance; private static volatile boolean initialized = false; private final Scheduler scheduler; private static final ConcurrentHashMap appJobsKeyMap = new ConcurrentHashMap<>(); private final CollectionDAO collectionDAO; private final SearchRepository searchClient; + private final AppsPrivateConfiguration privateConfiguration; - private AppScheduler(CollectionDAO dao, SearchRepository searchClient) throws SchedulerException { + private AppScheduler( + CollectionDAO dao, + SearchRepository searchClient, + AppsPrivateConfiguration privateConfiguration) + throws SchedulerException { this.collectionDAO = dao; this.searchClient = searchClient; + this.privateConfiguration = privateConfiguration; this.scheduler = new StdSchedulerFactory().getScheduler(); // Add OMJob Listener this.scheduler @@ -53,10 +61,13 @@ private AppScheduler(CollectionDAO dao, SearchRepository searchClient) throws Sc this.scheduler.start(); } - public static void initialize(CollectionDAO dao, SearchRepository searchClient) + public static void initialize( + CollectionDAO dao, + SearchRepository searchClient, + AppsPrivateConfiguration privateConfiguration) throws SchedulerException { if (!initialized) { - instance = new AppScheduler(dao, searchClient); + instance = new AppScheduler(dao, searchClient, privateConfiguration); initialized = true; } else { LOG.info("Reindexing Handler is already initialized"); @@ -104,6 +115,7 @@ private JobDetail jobBuilder(App app, String jobIdentity) throws ClassNotFoundEx dataMap.put(APP_INFO_KEY, app); dataMap.put(COLLECTION_DAO_KEY, collectionDAO); dataMap.put(SEARCH_CLIENT_KEY, searchClient); + dataMap.put(APPS_PRIVATE_CONFIG_KEY, privateConfiguration); dataMap.put("triggerType", AppRunType.Scheduled.value()); Class clz = (Class) Class.forName(app.getClassName()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index b808431b85ab..4fc72d199614 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -46,6 +46,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; @@ -97,6 +98,7 @@ public class AppResource extends EntityResource { public static final String COLLECTION_PATH = "v1/apps/"; private OpenMetadataApplicationConfig openMetadataApplicationConfig; + private AppsPrivateConfiguration privateConfiguration; private PipelineServiceClient pipelineServiceClient; static final String FIELDS = "owner"; private SearchRepository searchRepository; @@ -104,6 +106,7 @@ public class AppResource extends EntityResource { @Override public void initialize(OpenMetadataApplicationConfig config) { this.openMetadataApplicationConfig = config; + this.privateConfiguration = config.getAppsPrivateConfiguration(); this.pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); @@ -114,7 +117,7 @@ public void initialize(OpenMetadataApplicationConfig config) { new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory()); try { - AppScheduler.initialize(dao, searchRepository); + AppScheduler.initialize(dao, searchRepository, privateConfiguration); // Get Create App Requests List createAppsReq = @@ -143,7 +146,8 @@ public void initialize(OpenMetadataApplicationConfig config) { new OpenMetadataConnectionBuilder( openMetadataApplicationConfig, app.getBot().getName()) .build()); - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication( + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); } } } catch (SchedulerException | IOException ex) { @@ -538,8 +542,10 @@ public Response create( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); - ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication( + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + ApplicationHandler.configureApplication( + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); } // We don't want to store this information app.setOpenMetadataServerConnection(null); @@ -581,7 +587,7 @@ public Response patchApplication( .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication( - updatedApp, Entity.getCollectionDAO(), searchRepository); + updatedApp, Entity.getCollectionDAO(), searchRepository, privateConfiguration); } // We don't want to store this information updatedApp.setOpenMetadataServerConnection(null); @@ -618,7 +624,8 @@ public Response createOrUpdate( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication( + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); } // We don't want to store this information app.setOpenMetadataServerConnection(null); @@ -703,7 +710,8 @@ public Response restoreApp( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication( + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); } // We don't want to store this information app.setOpenMetadataServerConnection(null); @@ -741,7 +749,8 @@ public Response scheduleApplication( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication(app, repository.getDaoCollection(), searchRepository); + ApplicationHandler.installApplication( + app, repository.getDaoCollection(), searchRepository, privateConfiguration); return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); } throw new IllegalArgumentException("App is not of schedule type Scheduled."); @@ -779,7 +788,7 @@ public Response configureApplication( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); try { - ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository); + ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository, privateConfiguration); return Response.status(Response.Status.OK).entity("App has been configured.").build(); } catch (RuntimeException e) { return Response.status(Response.Status.INTERNAL_SERVER_ERROR) @@ -813,7 +822,7 @@ public Response triggerApplicationRun( App app = repository.getByName(uriInfo, name, fields); if (app.getAppType().equals(AppType.Internal)) { ApplicationHandler.triggerApplicationOnDemand( - app, Entity.getCollectionDAO(), searchRepository); + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); return Response.status(Response.Status.OK).entity("Application Triggered").build(); } else { if (!app.getPipelines().isEmpty()) { @@ -862,7 +871,8 @@ public Response deployApplicationFlow( EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); if (app.getAppType().equals(AppType.Internal)) { - ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.installApplication( + app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); return Response.status(Response.Status.OK).entity("Application Deployed").build(); } else { if (!app.getPipelines().isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index cc40a5559442..58fa4d39ebb6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -235,7 +235,7 @@ public Integer reIndex( boolean recreateIndexes) { try { parseConfig(); - AppScheduler.initialize(collectionDAO, searchRepository); + AppScheduler.initialize(collectionDAO, searchRepository, config.getAppsPrivateConfiguration()); App searchIndexApp = new App() .withId(UUID.randomUUID()) diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json new file mode 100644 index 000000000000..d826545ba5ba --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json @@ -0,0 +1,43 @@ +{ + "$id": "https://open-metadata.org/schema/entity/configuration/appsConfiguration.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AppsConfiguration", + "description": "This schema defines a list of configurations for the Application Framework", + "type": "object", + "javaType": "org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration", + "definitions": { + "appConfig": { + "type": "object", + "javaType": "org.openmetadata.schema.api.configuration.apps.AppConfig", + "title": "Application Configuration", + "description": "Single Application Configuration Definition", + "properties": { + "name": { + "type": "string", + "description": "Application Name" + }, + "parameters": { + "javaType": "org.openmetadata.schema.api.configuration.apps.Parameters", + "description": "Parameters to initialize the Applications.", + "type": "object", + "additionalProperties": { + ".{1,}": { "type": "string" } + } + } + }, + "required": ["name", "parameters"], + "additionalProperties": false + } + }, + "properties": { + "appsConfiguration": { + "description": "List of configuration for apps", + "type": "array", + "items": { + "$ref": "#/definitions/appConfig" + } + } + }, + "required": ["appsConfiguration"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index 13651561eeae..2546da2a2bab 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -199,6 +199,10 @@ "description": "Application Configuration object.", "$ref": "./configuration/applicationConfig.json#/definitions/appConfig" }, + "privateConfiguration": { + "description": "Application Private configuration loaded at runtime.", + "$ref": "./configuration/applicationConfig.json#/definitions/privateConfig" + }, "pipelines": { "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", "$ref": "../../type/entityReferenceList.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 2a85c58f4d1c..d963dc9be84a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -20,6 +20,13 @@ "$ref": "internal/searchIndexingAppConfig.json" } ] + }, + "privateConfig": { + "oneOf": [ + { + "$ref": "./private/external/metaPilotAppPrivateConfig.json" + } + ] } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json index c9f085d19a4d..2943f351169e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -48,25 +48,6 @@ "$ref": "#/definitions/metaPilotAppType", "default": "MetaPilot" }, - "waiiInstance": { - "title": "WAII Instance", - "description": "WAII API host URL", - "type": "string", - "format": "URI", - "default": "https://tweakit.waii.ai/api/" - }, - "collateURL": { - "title": "Collate URL", - "description": "Collate Server public URL. WAII will use this information to interact with the server. E.g., https://sandbox.getcollate.io", - "type": "string", - "format": "URI" - }, - "token": { - "title": "WAII API Token", - "description": "WAII API Token", - "type": "string", - "format": "password" - }, "serviceDatabases": { "title": "Service Databases", "description": "Services and Databases configured to get the descriptions from.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json new file mode 100644 index 000000000000..aed35d739edd --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json @@ -0,0 +1,31 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/metaPilotAppConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MetaPilotAppPrivateConfig", + "description": "PRivate Configuration for the MetaPilot External Application.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.app.external.MetaPilotAppPrivateConfig", + "properties": { + "waiiInstance": { + "title": "WAII Instance", + "description": "WAII API host URL", + "type": "string", + "format": "URI", + "default": "https://tweakit.waii.ai/api/" + }, + "collateURL": { + "title": "Collate URL", + "description": "Collate Server public URL. WAII will use this information to interact with the server. E.g., https://sandbox.getcollate.io", + "type": "string", + "format": "URI" + }, + "token": { + "title": "WAII API Token", + "description": "WAII API Token", + "type": "string", + "format": "password" + } + }, + "additionalProperties": false, + "required": ["waiiInstance", "collateURL", "token"] +} diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json index 099d382bf49d..b4e85a65da3c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json @@ -51,25 +51,6 @@ "$ref": "#/definitions/metaPilotAppType", "default": "MetaPilot" }, - "waiiInstance": { - "title": "WAII Instance", - "description": "WAII API host URL", - "type": "string", - "format": "URI", - "default": "https://tweakit.waii.ai/api/" - }, - "collateURL": { - "title": "Collate URL", - "description": "Collate Server public URL. WAII will use this information to interact with the server. E.g., https://sandbox.getcollate.io", - "type": "string", - "format": "URI" - }, - "token": { - "title": "WAII API Token", - "description": "WAII API Token", - "type": "string", - "format": "password" - }, "serviceDatabases": { "title": "Service Databases", "description": "Services and Databases configured to get the descriptions from.", From 834800ffbd154e491d3a6c56bc59799a4c6b857a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 30 Jan 2024 15:33:18 +0100 Subject: [PATCH 36/49] add private configs for apps --- conf/openmetadata.yaml | 8 -------- .../service/apps/AbstractNativeApplication.java | 4 ++-- ...psConfiguration.json => appsPrivateConfiguration.json} | 6 +++--- .../resources/json/schema/entity/applications/app.json | 4 ---- .../applications/configuration/applicationConfig.json | 7 ------- 5 files changed, 5 insertions(+), 24 deletions(-) rename openmetadata-spec/src/main/resources/json/schema/configuration/{appsConfiguration.json => appsPrivateConfiguration.json} (91%) diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index d7de8ea77298..5e7704e38cf8 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -369,11 +369,3 @@ web: permission-policy: enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false} option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""} - -#applications: -# appsConfiguration: # Defined as private configs -# - name: MetaPilotApplication -# parameters: -# waiiInstance: waiiInstance -# token: token -# collateURL: collateURL diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index 10f809511a5d..a93ca38fd9d8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -81,8 +81,8 @@ public void init( * Load the apps' private parameters, if needed */ private void loadPrivateParameters(AppsPrivateConfiguration privateConfiguration) { - if (privateConfiguration != null && !nullOrEmpty(privateConfiguration.getAppsConfiguration())) { - for (AppConfig appConfig : privateConfiguration.getAppsConfiguration()) { + if (privateConfiguration != null && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { + for (AppConfig appConfig : privateConfiguration.getAppsPrivateConfiguration()) { if (this.app.getName().equals(appConfig.getName())) { this.privateParameters = appConfig.getParameters(); } diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json similarity index 91% rename from openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json rename to openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json index d826545ba5ba..f04f52fd8185 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/appsConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/configuration/appsConfiguration.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AppsConfiguration", + "title": "AppsPrivateConfiguration", "description": "This schema defines a list of configurations for the Application Framework", "type": "object", "javaType": "org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration", @@ -30,7 +30,7 @@ } }, "properties": { - "appsConfiguration": { + "appsPrivateConfiguration": { "description": "List of configuration for apps", "type": "array", "items": { @@ -38,6 +38,6 @@ } } }, - "required": ["appsConfiguration"], + "required": ["appsPrivateConfiguration"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index 2546da2a2bab..13651561eeae 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -199,10 +199,6 @@ "description": "Application Configuration object.", "$ref": "./configuration/applicationConfig.json#/definitions/appConfig" }, - "privateConfiguration": { - "description": "Application Private configuration loaded at runtime.", - "$ref": "./configuration/applicationConfig.json#/definitions/privateConfig" - }, "pipelines": { "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", "$ref": "../../type/entityReferenceList.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index d963dc9be84a..2a85c58f4d1c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -20,13 +20,6 @@ "$ref": "internal/searchIndexingAppConfig.json" } ] - }, - "privateConfig": { - "oneOf": [ - { - "$ref": "./private/external/metaPilotAppPrivateConfig.json" - } - ] } } } From f151418157b09190e19625db3d53b9735f4783b3 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 30 Jan 2024 20:21:58 +0530 Subject: [PATCH 37/49] fix update application icons --- .../external/metaPilotAppConfig.json | 3 +- .../assets/svg/DataInsightsApplication.svg | 40 +++++++++--------- .../svg/DataInsightsReportApplication.svg | 34 ++++++--------- .../assets/svg/SearchIndexingApplication.svg | 42 ++++++++++--------- 4 files changed, 55 insertions(+), 64 deletions(-) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json index 2943f351169e..4ac75978557f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/metaPilotAppConfig.json @@ -57,6 +57,5 @@ } } }, - "additionalProperties": false, - "required": ["token"] + "additionalProperties": false } diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg index 0ed0c62850cc..64708284d223 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsApplication.svg @@ -1,22 +1,20 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg index 0beefce29883..5aeecd41f534 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataInsightsReportApplication.svg @@ -1,22 +1,14 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg index 840ece058f4a..45d0613f5a8b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/SearchIndexingApplication.svg @@ -1,21 +1,23 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + From 36d9ce8a6a3fac001e103b7d180ed01feafe6762 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Tue, 30 Jan 2024 20:25:02 +0530 Subject: [PATCH 38/49] minor center align icon --- .../src/components/Applications/AppLogo/AppLogo.component.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx index 50c974cf9bfe..6ddafbb8e08f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppLogo/AppLogo.component.tsx @@ -28,7 +28,7 @@ const AppLogo = ({ const Icon = data.ReactComponent as React.ComponentType< JSX.IntrinsicElements['svg'] >; - setAppLogo(); + setAppLogo(); }); } else { setAppLogo(logo); From a9945be730a227bae32a3f4e905dde8b69f67d6d Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 30 Jan 2024 16:57:00 +0100 Subject: [PATCH 39/49] add private configs for apps --- .../json/schema/configuration/appsPrivateConfiguration.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json index f04f52fd8185..bac328c60b7c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json @@ -1,5 +1,5 @@ { - "$id": "https://open-metadata.org/schema/entity/configuration/appsConfiguration.json", + "$id": "https://open-metadata.org/schema/entity/configuration/appsPrivateConfiguration.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "AppsPrivateConfiguration", "description": "This schema defines a list of configurations for the Application Framework", From 8ba8e0a45d7507ab7b07c265d00663ca74eee620 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 30 Jan 2024 17:06:34 +0100 Subject: [PATCH 40/49] Format --- .../openmetadata/service/apps/AbstractNativeApplication.java | 3 ++- .../org/openmetadata/service/resources/apps/AppResource.java | 3 ++- .../org/openmetadata/service/util/OpenMetadataOperations.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index a93ca38fd9d8..4059b3387bd5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -81,7 +81,8 @@ public void init( * Load the apps' private parameters, if needed */ private void loadPrivateParameters(AppsPrivateConfiguration privateConfiguration) { - if (privateConfiguration != null && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { + if (privateConfiguration != null + && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { for (AppConfig appConfig : privateConfiguration.getAppsPrivateConfiguration()) { if (this.app.getName().equals(appConfig.getName())) { this.privateParameters = appConfig.getParameters(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 4fc72d199614..a68fc68e9074 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -788,7 +788,8 @@ public Response configureApplication( new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) .build()); try { - ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository, privateConfiguration); + ApplicationHandler.configureApplication( + app, repository.getDaoCollection(), searchRepository, privateConfiguration); return Response.status(Response.Status.OK).entity("App has been configured.").build(); } catch (RuntimeException e) { return Response.status(Response.Status.INTERNAL_SERVER_ERROR) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 58fa4d39ebb6..df4c46e371a9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -235,7 +235,8 @@ public Integer reIndex( boolean recreateIndexes) { try { parseConfig(); - AppScheduler.initialize(collectionDAO, searchRepository, config.getAppsPrivateConfiguration()); + AppScheduler.initialize( + collectionDAO, searchRepository, config.getAppsPrivateConfiguration()); App searchIndexApp = new App() .withId(UUID.randomUUID()) From 95a713dabc3121065a3120863d258c37427240ce Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 08:18:35 +0100 Subject: [PATCH 41/49] Fix pydantic gen --- .../entity/applications/configuration/applicationConfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 2a85c58f4d1c..c8d224545408 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -21,5 +21,6 @@ } ] } - } + }, + "additionalProperties": false } From 842623ff591e10686471348619c14804954ed0c4 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 10:05:15 +0100 Subject: [PATCH 42/49] Remove token --- .../ui/src/utils/ApplicationSchemas/MetaPilotApplication.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json index b4e85a65da3c..ea86671af3b2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/MetaPilotApplication.json @@ -60,6 +60,5 @@ } } }, - "additionalProperties": false, - "required": ["token"] + "additionalProperties": false } From b8098eeeadb52c110cb39eeb0ef0d4187fbba922 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 10:47:07 +0100 Subject: [PATCH 43/49] Update name --- .../service/apps/AbstractNativeApplication.java | 8 ++++---- .../schema/configuration/appsPrivateConfiguration.json | 8 ++++---- .../applications/configuration/applicationConfig.json | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index 4059b3387bd5..413b9a00e9b8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -18,7 +18,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.AppRuntime; -import org.openmetadata.schema.api.configuration.apps.AppConfig; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.configuration.apps.Parameters; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; @@ -83,9 +83,9 @@ public void init( private void loadPrivateParameters(AppsPrivateConfiguration privateConfiguration) { if (privateConfiguration != null && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppConfig appConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (this.app.getName().equals(appConfig.getName())) { - this.privateParameters = appConfig.getParameters(); + for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { + if (this.app.getName().equals(appPrivateConfig.getName())) { + this.privateParameters = appPrivateConfig.getParameters(); } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json index bac328c60b7c..f0e1672ea96a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/appsPrivateConfiguration.json @@ -6,10 +6,10 @@ "type": "object", "javaType": "org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration", "definitions": { - "appConfig": { + "appPrivateConfig": { "type": "object", - "javaType": "org.openmetadata.schema.api.configuration.apps.AppConfig", - "title": "Application Configuration", + "javaType": "org.openmetadata.schema.api.configuration.apps.AppPrivateConfig", + "title": "AppPrivateConfig", "description": "Single Application Configuration Definition", "properties": { "name": { @@ -34,7 +34,7 @@ "description": "List of configuration for apps", "type": "array", "items": { - "$ref": "#/definitions/appConfig" + "$ref": "#/definitions/appPrivateConfig" } } }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index c8d224545408..2a85c58f4d1c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -21,6 +21,5 @@ } ] } - }, - "additionalProperties": false + } } From 73226a9513f1d47ba5961b54b7ff0c325700acc4 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 12:59:53 +0100 Subject: [PATCH 44/49] Rework private conf --- .../src/metadata/workflow/application.py | 12 +- .../workflows/ingestion/application.py | 4 + .../apps/AbstractNativeApplication.java | 36 +----- .../service/apps/ApplicationHandler.java | 39 ++----- .../service/apps/NativeApplication.java | 7 +- .../bundles/searchIndex/SearchIndexApp.java | 9 +- .../bundles/test/NoOpTestApplication.java | 9 +- .../service/apps/scheduler/AppScheduler.java | 18 +-- .../service/resources/apps/AppResource.java | 104 +++++++++--------- .../service/util/OpenMetadataOperations.java | 3 +- .../json/schema/entity/applications/app.json | 4 + .../configuration/applicationConfig.json | 7 ++ .../external/metaPilotAppPrivateConfig.json | 2 +- .../schema/metadataIngestion/application.json | 4 + .../applicationPipeline.json | 4 + 15 files changed, 110 insertions(+), 152 deletions(-) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 09ebd2461a5d..64f3e5e72b64 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -15,6 +15,9 @@ from typing import List, Optional from metadata.config.common import WorkflowExecutionError +from metadata.generated.schema.configuration.appsPrivateConfiguration import ( + AppPrivateConfig, +) from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( AppConfig, ) @@ -50,9 +53,13 @@ class AppRunner(Step, ABC): """Class that knows how to execute the Application logic.""" def __init__( - self, config: AppConfig.__fields__["__root__"].type_, metadata: OpenMetadata + self, + config: AppConfig.__fields__["__root__"].type_, + private_config: AppPrivateConfig.__fields__["__root__"].type_, + metadata: OpenMetadata, ): self.config = config + self.private_config = private_config self.metadata = metadata super().__init__() @@ -117,6 +124,9 @@ def post_init(self) -> None: config=self.config.appConfig.__root__ if self.config.appConfig else None, + private_config=self.config.appPrivateConfig.__root__ + if self.config.appPrivateConfig + else None, metadata=self.metadata, ) except Exception as exc: diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py index 3b8a770ce209..1cc91ddcc069 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py @@ -22,6 +22,7 @@ from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( AppConfig, + PrivateConfig, ) from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import ( IngestionPipeline, @@ -76,6 +77,9 @@ def build_application_workflow_config( appConfig=AppConfig( __root__=application_pipeline_conf.appConfig.__root__, ), + appPrivateConfig=PrivateConfig( + __root__=application_pipeline_conf.appPrivateConfig.__root__ + ), workflowConfig=build_workflow_config_property(ingestion_pipeline), ingestionPipelineFQN=ingestion_pipeline.fullyQualifiedName.__root__, ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java index 413b9a00e9b8..cd01c50638fb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/AbstractNativeApplication.java @@ -1,9 +1,7 @@ package org.openmetadata.service.apps; import static com.cronutils.model.CronType.QUARTZ; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.JOB_LISTENER_NAME; -import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_PRIVATE_CONFIG_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.COLLECTION_DAO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.SEARCH_CLIENT_KEY; @@ -18,9 +16,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.AppRuntime; -import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; -import org.openmetadata.schema.api.configuration.apps.Parameters; import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; @@ -55,7 +50,6 @@ public class AbstractNativeApplication implements NativeApplication { protected CollectionDAO collectionDAO; private @Getter App app; - private @Getter Parameters privateParameters = null; protected SearchRepository searchRepository; private final @Getter CronMapper cronMapper = CronMapper.fromQuartzToUnix(); private final @Getter CronParser cronParser = @@ -65,30 +59,10 @@ public class AbstractNativeApplication implements NativeApplication { private static final String SERVICE_NAME = "OpenMetadata"; @Override - public void init( - App app, - CollectionDAO dao, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration) { + public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { this.collectionDAO = dao; this.searchRepository = searchRepository; this.app = app; - - this.loadPrivateParameters(privateConfiguration); - } - - /** - * Load the apps' private parameters, if needed - */ - private void loadPrivateParameters(AppsPrivateConfiguration privateConfiguration) { - if (privateConfiguration != null - && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (this.app.getName().equals(appPrivateConfig.getName())) { - this.privateParameters = appPrivateConfig.getParameters(); - } - } - } } @Override @@ -189,7 +163,8 @@ private void createAndBindIngestionPipeline( .withConfig( new ApplicationPipeline() .withSourcePythonClass(this.getApp().getSourcePythonClass()) - .withAppConfig(config))) + .withAppConfig(config) + .withAppPrivateConfig(this.getApp().getPrivateConfiguration()))) .withAirflowConfig( new AirflowConfig() .withScheduleInterval(this.getCronMapper().map(quartzCron).asString())) @@ -237,11 +212,8 @@ public void execute(JobExecutionContext jobExecutionContext) { SearchRepository searchRepositoryForJob = (SearchRepository) jobExecutionContext.getJobDetail().getJobDataMap().get(SEARCH_CLIENT_KEY); - AppsPrivateConfiguration privateConfiguration = - (AppsPrivateConfiguration) - jobExecutionContext.getJobDetail().getJobDataMap().get(APPS_PRIVATE_CONFIG_KEY); // Initialise the Application - this.init(jobApp, dao, searchRepositoryForJob, privateConfiguration); + this.init(jobApp, dao, searchRepositoryForJob); // Trigger this.startApp(jobExecutionContext); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 619116a50039..35b3afa19c20 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -3,7 +3,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -17,38 +16,23 @@ private ApplicationHandler() { } public static void triggerApplicationOnDemand( - App app, - CollectionDAO daoCollection, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration) { - runMethodFromApplication( - app, daoCollection, searchRepository, privateConfiguration, "triggerOnDemand"); + App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand"); } public static void installApplication( - App app, - CollectionDAO daoCollection, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration) { - runMethodFromApplication(app, daoCollection, searchRepository, privateConfiguration, "install"); + App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + runMethodFromApplication(app, daoCollection, searchRepository, "install"); } public static void configureApplication( - App app, - CollectionDAO daoCollection, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration) { - runMethodFromApplication( - app, daoCollection, searchRepository, privateConfiguration, "configure"); + App app, CollectionDAO daoCollection, SearchRepository searchRepository) { + runMethodFromApplication(app, daoCollection, searchRepository, "configure"); } /** Load an App from its className and call its methods dynamically */ public static void runMethodFromApplication( - App app, - CollectionDAO daoCollection, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration, - String methodName) { + App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) { // Native Application try { Class clz = Class.forName(app.getClassName()); @@ -58,13 +42,8 @@ public static void runMethodFromApplication( Method initMethod = resource .getClass() - .getMethod( - "init", - App.class, - CollectionDAO.class, - SearchRepository.class, - AppsPrivateConfiguration.class); - initMethod.invoke(resource, app, daoCollection, searchRepository, privateConfiguration); + .getMethod("init", App.class, CollectionDAO.class, SearchRepository.class); + initMethod.invoke(resource, app, daoCollection, searchRepository); // Call method on demand Method scheduleMethod = resource.getClass().getMethod(methodName); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java index b5ff82dcf867..e0ed3e228746 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/NativeApplication.java @@ -1,6 +1,5 @@ package org.openmetadata.service.apps; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.search.SearchRepository; @@ -8,11 +7,7 @@ import org.quartz.JobExecutionContext; public interface NativeApplication extends Job { - void init( - App app, - CollectionDAO dao, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration); + void init(App app, CollectionDAO dao, SearchRepository searchRepository); void install(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 2a9bc71e081f..9f5682ecd2ad 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -17,7 +17,6 @@ import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppRunType; @@ -95,12 +94,8 @@ public class SearchIndexApp extends AbstractNativeApplication { private volatile boolean stopped = false; @Override - public void init( - App app, - CollectionDAO dao, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration) { - super.init(app, dao, searchRepository, privateConfiguration); + public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { + super.init(app, dao, searchRepository); // request for reindexing EventPublisherJob request = JsonUtils.convertValue(app.getAppConfiguration(), EventPublisherJob.class) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java index 73c68372fe7c..94a0c0e8a7a3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/test/NoOpTestApplication.java @@ -1,7 +1,6 @@ package org.openmetadata.service.apps.bundles.test; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.apps.AbstractNativeApplication; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -11,12 +10,8 @@ public class NoOpTestApplication extends AbstractNativeApplication { @Override - public void init( - App app, - CollectionDAO dao, - SearchRepository searchRepository, - AppsPrivateConfiguration privateConfiguration) { - super.init(app, dao, searchRepository, privateConfiguration); + public void init(App app, CollectionDAO dao, SearchRepository searchRepository) { + super.init(app, dao, searchRepository); LOG.info("NoOpTestApplication is initialized"); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java index fa5067947ce0..6a8134c82f9e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java @@ -9,7 +9,6 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.AppRuntime; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppRunType; import org.openmetadata.schema.entity.app.AppSchedule; @@ -36,23 +35,16 @@ public class AppScheduler { public static final String APP_INFO_KEY = "applicationInfoKey"; public static final String COLLECTION_DAO_KEY = "daoKey"; public static final String SEARCH_CLIENT_KEY = "searchClientKey"; - public static final String APPS_PRIVATE_CONFIG_KEY = "privateConfigurationKey"; private static AppScheduler instance; private static volatile boolean initialized = false; private final Scheduler scheduler; private static final ConcurrentHashMap appJobsKeyMap = new ConcurrentHashMap<>(); private final CollectionDAO collectionDAO; private final SearchRepository searchClient; - private final AppsPrivateConfiguration privateConfiguration; - private AppScheduler( - CollectionDAO dao, - SearchRepository searchClient, - AppsPrivateConfiguration privateConfiguration) - throws SchedulerException { + private AppScheduler(CollectionDAO dao, SearchRepository searchClient) throws SchedulerException { this.collectionDAO = dao; this.searchClient = searchClient; - this.privateConfiguration = privateConfiguration; this.scheduler = new StdSchedulerFactory().getScheduler(); // Add OMJob Listener this.scheduler @@ -61,13 +53,10 @@ private AppScheduler( this.scheduler.start(); } - public static void initialize( - CollectionDAO dao, - SearchRepository searchClient, - AppsPrivateConfiguration privateConfiguration) + public static void initialize(CollectionDAO dao, SearchRepository searchClient) throws SchedulerException { if (!initialized) { - instance = new AppScheduler(dao, searchClient, privateConfiguration); + instance = new AppScheduler(dao, searchClient); initialized = true; } else { LOG.info("Reindexing Handler is already initialized"); @@ -115,7 +104,6 @@ private JobDetail jobBuilder(App app, String jobIdentity) throws ClassNotFoundEx dataMap.put(APP_INFO_KEY, app); dataMap.put(COLLECTION_DAO_KEY, collectionDAO); dataMap.put(SEARCH_CLIENT_KEY, searchClient); - dataMap.put(APPS_PRIVATE_CONFIG_KEY, privateConfiguration); dataMap.put("triggerType", AppRunType.Scheduled.value()); Class clz = (Class) Class.forName(app.getClassName()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index a68fc68e9074..34189dba23d3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -46,6 +46,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.app.App; @@ -117,7 +118,7 @@ public void initialize(OpenMetadataApplicationConfig config) { new SearchRepository(config.getElasticSearchConfiguration(), new SearchIndexFactory()); try { - AppScheduler.initialize(dao, searchRepository, privateConfiguration); + AppScheduler.initialize(dao, searchRepository); // Get Create App Requests List createAppsReq = @@ -142,12 +143,8 @@ public void initialize(OpenMetadataApplicationConfig config) { // Schedule if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder( - openMetadataApplicationConfig, app.getBot().getName()) - .build()); - ApplicationHandler.installApplication( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + setAppRuntimeProperties(app); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } } } catch (SchedulerException | IOException ex) { @@ -167,6 +164,32 @@ public static class AppRunList extends ResultList { /* Required for serde */ } + /** + * Load the apps' OM configuration and private parameters + */ + private void setAppRuntimeProperties(App app) { + app.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) + .build()); + + if (privateConfiguration != null + && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { + for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { + if (app.getName().equals(appPrivateConfig.getName())) { + app.setPrivateConfiguration(appPrivateConfig.getParameters()); + } + } + } + } + + /** + * We don't want to store runtime information into the DB + */ + private void unsetAppRuntimeProperties(App app) { + app.setOpenMetadataServerConnection(null); + app.setPrivateConfiguration(null); + } + @GET @Operation( operationId = "listInstalledApplications", @@ -538,17 +561,13 @@ public Response create( create.getName(), new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); - ApplicationHandler.configureApplication( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); + ApplicationHandler.configureApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information - app.setOpenMetadataServerConnection(null); + unsetAppRuntimeProperties(app); return create(uriInfo, securityContext, app); } @@ -582,15 +601,13 @@ public Response patchApplication( AppScheduler.getInstance().deleteScheduledApplication(app); Response response = patchInternal(uriInfo, securityContext, id, patch); App updatedApp = (App) response.getEntity(); - updatedApp.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(updatedApp); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { ApplicationHandler.installApplication( - updatedApp, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + updatedApp, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information - updatedApp.setOpenMetadataServerConnection(null); + unsetAppRuntimeProperties(updatedApp); return response; } @@ -620,15 +637,12 @@ public Response createOrUpdate( new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); AppScheduler.getInstance().deleteScheduledApplication(app); - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information - app.setOpenMetadataServerConnection(null); + unsetAppRuntimeProperties(app); return createOrUpdate(uriInfo, securityContext, app); } @@ -706,15 +720,12 @@ public Response restoreApp( Response response = restoreEntity(uriInfo, securityContext, restore.getId()); if (response.getStatus() == Response.Status.OK.getStatusCode()) { App app = (App) response.getEntity(); - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); } // We don't want to store this information - app.setOpenMetadataServerConnection(null); + unsetAppRuntimeProperties(app); } return response; } @@ -745,12 +756,9 @@ public Response scheduleApplication( @Context SecurityContext securityContext) { App app = repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); if (app.getScheduleType().equals(ScheduleType.Scheduled)) { - ApplicationHandler.installApplication( - app, repository.getDaoCollection(), searchRepository, privateConfiguration); + ApplicationHandler.installApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); } throw new IllegalArgumentException("App is not of schedule type Scheduled."); @@ -784,12 +792,9 @@ public Response configureApplication( repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); // The application will have the updated appConfiguration we can use to run the `configure` // logic - app.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + setAppRuntimeProperties(app); try { - ApplicationHandler.configureApplication( - app, repository.getDaoCollection(), searchRepository, privateConfiguration); + ApplicationHandler.configureApplication(app, repository.getDaoCollection(), searchRepository); return Response.status(Response.Status.OK).entity("App has been configured.").build(); } catch (RuntimeException e) { return Response.status(Response.Status.INTERNAL_SERVER_ERROR) @@ -821,9 +826,10 @@ public Response triggerApplicationRun( String name) { EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); + setAppRuntimeProperties(app); if (app.getAppType().equals(AppType.Internal)) { ApplicationHandler.triggerApplicationOnDemand( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + app, Entity.getCollectionDAO(), searchRepository); return Response.status(Response.Status.OK).entity("Application Triggered").build(); } else { if (!app.getPipelines().isEmpty()) { @@ -834,9 +840,7 @@ public Response triggerApplicationRun( IngestionPipeline ingestionPipeline = ingestionPipelineRepository.get( uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNER)); - ingestionPipeline.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + ingestionPipeline.setOpenMetadataServerConnection(app.getOpenMetadataServerConnection()); decryptOrNullify(securityContext, ingestionPipeline, app.getBot().getName(), true); ServiceEntityInterface service = Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); @@ -871,9 +875,9 @@ public Response deployApplicationFlow( String name) { EntityUtil.Fields fields = getFields(String.format("%s,bot,pipelines", FIELD_OWNER)); App app = repository.getByName(uriInfo, name, fields); + setAppRuntimeProperties(app); if (app.getAppType().equals(AppType.Internal)) { - ApplicationHandler.installApplication( - app, Entity.getCollectionDAO(), searchRepository, privateConfiguration); + ApplicationHandler.installApplication(app, Entity.getCollectionDAO(), searchRepository); return Response.status(Response.Status.OK).entity("Application Deployed").build(); } else { if (!app.getPipelines().isEmpty()) { @@ -885,9 +889,7 @@ public Response deployApplicationFlow( ingestionPipelineRepository.get( uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNER)); - ingestionPipeline.setOpenMetadataServerConnection( - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig, app.getBot().getName()) - .build()); + ingestionPipeline.setOpenMetadataServerConnection(app.getOpenMetadataServerConnection()); decryptOrNullify(securityContext, ingestionPipeline, app.getBot().getName(), true); ServiceEntityInterface service = Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index df4c46e371a9..cc40a5559442 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -235,8 +235,7 @@ public Integer reIndex( boolean recreateIndexes) { try { parseConfig(); - AppScheduler.initialize( - collectionDAO, searchRepository, config.getAppsPrivateConfiguration()); + AppScheduler.initialize(collectionDAO, searchRepository); App searchIndexApp = new App() .withId(UUID.randomUUID()) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index 13651561eeae..2546da2a2bab 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -199,6 +199,10 @@ "description": "Application Configuration object.", "$ref": "./configuration/applicationConfig.json#/definitions/appConfig" }, + "privateConfiguration": { + "description": "Application Private configuration loaded at runtime.", + "$ref": "./configuration/applicationConfig.json#/definitions/privateConfig" + }, "pipelines": { "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", "$ref": "../../type/entityReferenceList.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 2a85c58f4d1c..d963dc9be84a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -20,6 +20,13 @@ "$ref": "internal/searchIndexingAppConfig.json" } ] + }, + "privateConfig": { + "oneOf": [ + { + "$ref": "./private/external/metaPilotAppPrivateConfig.json" + } + ] } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json index aed35d739edd..61b26a069ce8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.json @@ -1,5 +1,5 @@ { - "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/metaPilotAppConfig.json", + "$id": "https://open-metadata.org/schema/entity/applications/configuration/private/external/metaPilotAppConfig.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "MetaPilotAppPrivateConfig", "description": "PRivate Configuration for the MetaPilot External Application.", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json index 2ee79748248c..bd37ce5d74e2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/application.json @@ -18,6 +18,10 @@ "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/appConfig", "description": "External Application configuration" }, + "appPrivateConfig": { + "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/privateConfig", + "description": "External Application Private configuration" + }, "ingestionPipelineFQN": { "description": "Fully qualified name of ingestion pipeline, used to identify the current ingestion pipeline", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json index 72248e41ce4c..dc029f796a96 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/applicationPipeline.json @@ -25,6 +25,10 @@ "appConfig": { "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/appConfig", "description": "Application configuration" + }, + "appPrivateConfig": { + "$ref": "../entity/applications/configuration/applicationConfig.json#/definitions/privateConfig", + "description": "Application private configuration" } }, "additionalProperties": false From 49ca846242acbcfd9893b33adcf72de6ffa118c6 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 15:26:09 +0100 Subject: [PATCH 45/49] Fix apps --- .../src/metadata/applications/auto_tagger.py | 11 ++++++--- .../src/metadata/workflow/application.py | 23 +++++-------------- .../workflows/ingestion/application.py | 8 +++++-- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/ingestion/src/metadata/applications/auto_tagger.py b/ingestion/src/metadata/applications/auto_tagger.py index 3583fd373652..6719cde48e35 100644 --- a/ingestion/src/metadata/applications/auto_tagger.py +++ b/ingestion/src/metadata/applications/auto_tagger.py @@ -21,6 +21,9 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) +from metadata.generated.schema.metadataIngestion.application import ( + OpenMetadataApplicationConfig, +) from metadata.generated.schema.type.tagLabel import ( LabelType, State, @@ -60,16 +63,18 @@ class AutoTaggerApp(AppRunner): jwtToken: "..." """ - def __init__(self, config: AutoTaggerAppConfig, metadata: OpenMetadata): + def __init__(self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata): super().__init__(config, metadata) - if not isinstance(config, AutoTaggerAppConfig): + if not isinstance(self.app_config, AutoTaggerAppConfig): raise InvalidAppConfiguration( f"AutoTagger Runner expects an AutoTaggerAppConfig, we got [{config}]" ) self._ner_scanner = None - self.confidence_threshold = config.confidenceLevel or DEFAULT_CONFIDENCE + self.confidence_threshold = ( + self.app_config.confidenceLevel or DEFAULT_CONFIDENCE + ) @property def name(self) -> str: diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 64f3e5e72b64..0ecc594c4983 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -15,12 +15,6 @@ from typing import List, Optional from metadata.config.common import WorkflowExecutionError -from metadata.generated.schema.configuration.appsPrivateConfiguration import ( - AppPrivateConfig, -) -from metadata.generated.schema.entity.applications.configuration.applicationConfig import ( - AppConfig, -) from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) @@ -54,12 +48,11 @@ class AppRunner(Step, ABC): def __init__( self, - config: AppConfig.__fields__["__root__"].type_, - private_config: AppPrivateConfig.__fields__["__root__"].type_, + config: OpenMetadataApplicationConfig, metadata: OpenMetadata, ): - self.config = config - self.private_config = private_config + self.app_config = config.appConfig + self.private_config = config.appPrivateConfig self.metadata = metadata super().__init__() @@ -74,7 +67,8 @@ def run(self) -> None: @classmethod def create(cls, config_dict: dict, metadata: OpenMetadata) -> "Step": - return cls(config=config_dict, metadata=metadata) + config = OpenMetadataApplicationConfig.parse_obj(config_dict) + return cls(config=config, metadata=metadata) class ApplicationWorkflow(BaseWorkflow, ABC): @@ -121,12 +115,7 @@ def post_init(self) -> None: try: self.runner = runner_class( - config=self.config.appConfig.__root__ - if self.config.appConfig - else None, - private_config=self.config.appPrivateConfig.__root__ - if self.config.appPrivateConfig - else None, + config=self.config, metadata=self.metadata, ) except Exception as exc: diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py index 1cc91ddcc069..dea9b0d48c12 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/workflows/ingestion/application.py @@ -76,10 +76,14 @@ def build_application_workflow_config( # We pass the generic class and let each app cast the actual object appConfig=AppConfig( __root__=application_pipeline_conf.appConfig.__root__, - ), + ) + if application_pipeline_conf.appConfig + else None, appPrivateConfig=PrivateConfig( __root__=application_pipeline_conf.appPrivateConfig.__root__ - ), + ) + if application_pipeline_conf.appPrivateConfig + else None, workflowConfig=build_workflow_config_property(ingestion_pipeline), ingestionPipelineFQN=ingestion_pipeline.fullyQualifiedName.__root__, ) From b3ac6b971e7cc120318cb1dcfde85c63fdbfae26 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 15:39:51 +0100 Subject: [PATCH 46/49] Fix apps --- ingestion/src/metadata/workflow/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 0ecc594c4983..e9a67f461d19 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -51,8 +51,8 @@ def __init__( config: OpenMetadataApplicationConfig, metadata: OpenMetadata, ): - self.app_config = config.appConfig - self.private_config = config.appPrivateConfig + self.app_config = config.appConfig.__root__ if config.appConfig else None + self.private_config = config.appPrivateConfig.__root__ if config.appConfig else None self.metadata = metadata super().__init__() From 10414b95f5cf126c591c57a9e6459cfd20ad352b Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 15:56:28 +0100 Subject: [PATCH 47/49] Format --- ingestion/src/metadata/workflow/application.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index e9a67f461d19..324fc125f831 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -52,7 +52,9 @@ def __init__( metadata: OpenMetadata, ): self.app_config = config.appConfig.__root__ if config.appConfig else None - self.private_config = config.appPrivateConfig.__root__ if config.appConfig else None + self.private_config = ( + config.appPrivateConfig.__root__ if config.appConfig else None + ) self.metadata = metadata super().__init__() From d5de1e115bee3d88c9f6fd3b6ea92ce3eddb5bd9 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Wed, 31 Jan 2024 16:16:05 +0100 Subject: [PATCH 48/49] Format --- ingestion/src/metadata/workflow/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 324fc125f831..07ce0688bb5f 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -53,7 +53,7 @@ def __init__( ): self.app_config = config.appConfig.__root__ if config.appConfig else None self.private_config = ( - config.appPrivateConfig.__root__ if config.appConfig else None + config.appPrivateConfig.__root__ if config.appPrivateConfig else None ) self.metadata = metadata From 285e6a16c92c04899fd36cbbaa91620fe3d255b0 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Wed, 31 Jan 2024 23:08:34 +0530 Subject: [PATCH 49/49] show metapilot only if its installed --- .../src/assets/svg/MetaPilotApplication.svg | 34 ++++++------------- .../AppDetails/AppDetails.component.tsx | 2 ++ .../MetaPilotProvider.interface.ts | 1 - .../MetaPilotProvider/MetaPilotProvider.tsx | 32 +++++++++++------ .../ui/src/constants/Applications.constant.ts | 2 ++ .../pages/AppInstall/AppInstall.component.tsx | 6 +++- 6 files changed, 42 insertions(+), 35 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg index 0b3526e03bd6..ad5a5f7d5193 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/MetaPilotApplication.svg @@ -1,24 +1,12 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx index b601ffea562d..e73d72732c8f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Applications/AppDetails/AppDetails.component.tsx @@ -43,6 +43,7 @@ import { ReactComponent as IconDropdown } from '../../../assets/svg/menu.svg'; import Loader from '../../../components/Loader/Loader'; import PageLayoutV1 from '../../../components/PageLayoutV1/PageLayoutV1'; import TabsLabel from '../../../components/TabsLabel/TabsLabel.component'; +import { APP_UI_SCHEMA } from '../../../constants/Applications.constant'; import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { GlobalSettingOptions } from '../../../constants/GlobalSettings.constants'; import { ServiceCategory } from '../../../enums/service.enum'; @@ -325,6 +326,7 @@ const AppDetails = () => { okText={t('label.submit')} schema={jsonSchema} serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + uiSchema={APP_UI_SCHEMA} validator={validator} onCancel={noop} onSubmit={onConfigSave} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts index 4ee0ed9f6075..3f853076ab7e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.interface.ts @@ -17,7 +17,6 @@ export interface MetaPilotContextType { suggestionsVisible: boolean; isMetaPilotEnabled: boolean; onToggleSuggestionsVisible: (state: boolean) => void; - onMetaPilotEnableUpdate: (state: boolean) => void; activeSuggestion?: Suggestion; suggestions: Suggestion[]; loading: boolean; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx index 8783915bfe34..ebc3ed89eeaf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MetaPilot/MetaPilotProvider/MetaPilotProvider.tsx @@ -24,6 +24,8 @@ import React, { import { useTranslation } from 'react-i18next'; import { ReactComponent as MetaPilotIcon } from '../../../assets/svg/MetaPilotApplication.svg'; import { Suggestion } from '../../../generated/entity/feed/suggestion'; +import { Include } from '../../../generated/type/include'; +import { getApplicationByName } from '../../../rest/applicationAPI'; import { getMetaPilotSuggestionsList, updateSuggestionStatus, @@ -51,6 +53,18 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { const [refreshEntity, setRefreshEntity] = useState<() => void>(); const { permissions } = usePermissionProvider(); + const fetchMetaPilotAppDetails = useCallback(async () => { + try { + await getApplicationByName('MetaPilotApplication', { + fields: 'owner', + include: Include.All, + }); + setIsMetaPilotEnabled(true); + } catch (error) { + setIsMetaPilotEnabled(false); + } + }, []); + const fetchSuggestions = useCallback(async (entityFQN: string) => { setLoading(true); try { @@ -91,10 +105,6 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { setSuggestionsVisible(state); }, []); - const onMetaPilotEnableUpdate = useCallback((state: boolean) => { - setIsMetaPilotEnabled(state); - }, []); - const onUpdateActiveSuggestion = useCallback((suggestion?: Suggestion) => { setActiveSuggestion(suggestion); }, []); @@ -105,14 +115,12 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { const resetMetaPilot = useCallback(() => { setSuggestionsVisible(false); - setIsMetaPilotEnabled(false); setActiveSuggestion(undefined); setEntityFqn(''); }, []); const initMetaPilot = useCallback( (entityFqn: string, refreshEntity?: () => void) => { - setIsMetaPilotEnabled(true); setEntityFqn(entityFqn); setRefreshEntity(() => refreshEntity); }, @@ -120,10 +128,16 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { ); useEffect(() => { - if (!isEmpty(permissions) && !isEmpty(entityFqn)) { + if (isMetaPilotEnabled && !isEmpty(entityFqn)) { fetchSuggestions(entityFqn); } - }, [permissions, entityFqn]); + }, [isMetaPilotEnabled, entityFqn]); + + useEffect(() => { + if (!isEmpty(permissions)) { + fetchMetaPilotAppDetails(); + } + }, [permissions]); const metaPilotContextObj = useMemo(() => { return { @@ -136,7 +150,6 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { refreshEntity, onToggleSuggestionsVisible, onUpdateEntityFqn, - onMetaPilotEnableUpdate, onUpdateActiveSuggestion, fetchSuggestions, acceptRejectSuggestion, @@ -153,7 +166,6 @@ const MetaPilotProvider = ({ children }: MetaPilotContextProps) => { refreshEntity, onToggleSuggestionsVisible, onUpdateEntityFqn, - onMetaPilotEnableUpdate, onUpdateActiveSuggestion, fetchSuggestions, acceptRejectSuggestion, diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts index 136ca3f460bf..4afd2223b850 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Applications.constant.ts @@ -21,3 +21,5 @@ export const STEPS_FOR_APP_INSTALL: Array = [ { name: t('label.configure'), step: 2 }, { name: t('label.schedule'), step: 3 }, ]; + +export const APP_UI_SCHEMA = { metaPilotAppType: { 'ui:widget': 'hidden' } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx index a4306e2fdcb9..b6ee11899a67 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx @@ -25,7 +25,10 @@ import FormBuilder from '../../components/common/FormBuilder/FormBuilder'; import IngestionStepper from '../../components/IngestionStepper/IngestionStepper.component'; import Loader from '../../components/Loader/Loader'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; -import { STEPS_FOR_APP_INSTALL } from '../../constants/Applications.constant'; +import { + APP_UI_SCHEMA, + STEPS_FOR_APP_INSTALL, +} from '../../constants/Applications.constant'; import { GlobalSettingOptions } from '../../constants/GlobalSettings.constants'; import { ServiceCategory } from '../../enums/service.enum'; import { AppType } from '../../generated/entity/applications/app'; @@ -163,6 +166,7 @@ const AppInstall = () => { okText={t('label.submit')} schema={jsonSchema} serviceCategory={ServiceCategory.DASHBOARD_SERVICES} + uiSchema={APP_UI_SCHEMA} validator={validator} onCancel={() => setActiveServiceStep(1)} onSubmit={onSaveConfiguration}