From 16b9c562dfd9d7948593286beff39a37ebf6c30c Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Tue, 17 Dec 2024 19:34:00 +0530 Subject: [PATCH 1/3] fix the search in advance search custom properties --- .../AdvanceSearchProvider.component.tsx | 14 +-- .../ui/src/utils/AdvancedSearchClassBase.ts | 95 +++++++++++++++++++ 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx index 9f612e84c287..9e7cb76f857d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx @@ -29,14 +29,12 @@ import { ValueField, } from 'react-awesome-query-builder'; import { useHistory, useParams } from 'react-router-dom'; -import { - emptyJsonTree, - TEXT_FIELD_OPERATORS, -} from '../../../constants/AdvancedSearch.constants'; +import { emptyJsonTree } from '../../../constants/AdvancedSearch.constants'; import { SearchIndex } from '../../../enums/search.enum'; import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation'; import { TabsInfoData } from '../../../pages/ExplorePage/ExplorePage.interface'; import { getAllCustomProperties } from '../../../rest/metadataTypeAPI'; +import advancedSearchClassBase from '../../../utils/AdvancedSearchClassBase'; import { getTierOptions, getTreeConfig, @@ -227,11 +225,9 @@ export const AdvanceSearchProvider = ({ if (Array.isArray(fields) && fields.length > 0) { fields.forEach((field: { name: string; type: string }) => { if (field.name && field.type) { - subfields[field.name] = { - type: 'text', - valueSources: ['value'], - operators: TEXT_FIELD_OPERATORS, - }; + const { subfieldsKey, dataObject } = + advancedSearchClassBase.getCustomPropertiesSubFields(field); + subfields[subfieldsKey] = dataObject; } }); } diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts index 05da5daaffd5..91d7d01ed1ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts @@ -22,6 +22,7 @@ import { SelectFieldSettings, } from 'react-awesome-query-builder'; import AntdConfig from 'react-awesome-query-builder/lib/config/antd'; +import { TEXT_FIELD_OPERATORS } from '../constants/AdvancedSearch.constants'; import { EntityFields, SuggestionField } from '../enums/AdvancedSearch.enum'; import { SearchIndex } from '../enums/search.enum'; import { getAggregateFieldOptions } from '../rest/miscAPI'; @@ -794,6 +795,100 @@ class AdvancedSearchClassBase { }, }; }; + + public getCustomPropertiesSubFields(field: { name: string; type: string }) { + { + switch (field.type) { + case 'array': + case 'entityReference': + return { + subfieldsKey: field.name + `.fullyQualifiedName`, + dataObject: { + type: 'select', + label: field.name, + fieldSettings: { + asyncFetch: advancedSearchClassBase.autocomplete({ + searchIndex: [SearchIndex.USER, SearchIndex.TEAM], + entityField: EntityFields.DISPLAY_NAME_KEYWORD, + }), + useAsyncSearch: true, + }, + }, + }; + case 'date-cp': { + return { + subfieldsKey: field.name, + dataObject: { + type: 'date', + valueSources: ['value'], + operators: TEXT_FIELD_OPERATORS, + }, + }; + } + case 'dateTime-cp': { + return { + subfieldsKey: field.name, + dataObject: { + type: 'datetime', + valueSources: ['value'], + operators: TEXT_FIELD_OPERATORS, + }, + }; + } + case 'time-cp': { + return { + subfieldsKey: field.name, + dataObject: { + type: 'time', + valueSources: ['value'], + operators: TEXT_FIELD_OPERATORS, + }, + }; + } + case 'timeInterval': { + return { + subfieldsKey: [field.name + `.start`, field.name + `.end`], + dataObject: { + type: 'text', + label: field.name, + fieldSettings: { + minimum_should_match: 1, + }, + }, + }; + } + + case 'table-cp': { + return { + subfieldsKey: field.name + '.first', + dataObject: { + type: 'text', + label: field.name, + valueSources: ['value'], + operators: TEXT_FIELD_OPERATORS, + }, + }; + } + // case "timestamp": + // case "string": + // case "markdown": + // case "number": + // case "integer": + // case "email": + // case "enum": + // case "duration": + default: + return { + subfieldsKey: field.name, + dataObject: { + type: 'text', + valueSources: ['value'], + operators: TEXT_FIELD_OPERATORS, + }, + }; + } + } + } } const advancedSearchClassBase = new AdvancedSearchClassBase(); From 73e47d72915eae8acb9a5696ef9c2dca85b53122 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Wed, 18 Dec 2024 11:25:49 +0530 Subject: [PATCH 2/3] modify v1/metadata/types/customProperties api to get customPropertyConfig along with name and type --- .../service/util/SchemaFieldExtractor.java | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java index e6d2d446cd56..dfbb05a8e33f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java @@ -32,7 +32,7 @@ @Slf4j public class SchemaFieldExtractor { - private static final Map> entityFieldsCache = + private static final Map> entityFieldsCache = new ConcurrentHashMap<>(); public SchemaFieldExtractor() { @@ -55,7 +55,7 @@ private static void initializeEntityFieldsCache() { Schema mainSchema = loadMainSchema(schemaPath, entityType, schemaUri, schemaClient); // Extract fields from the schema - Map fieldTypesMap = new LinkedHashMap<>(); + Map fieldTypesMap = new LinkedHashMap<>(); Deque processingStack = new ArrayDeque<>(); Set processedFields = new HashSet<>(); extractFieldsFromSchema(mainSchema, "", fieldTypesMap, processingStack, processedFields); @@ -75,7 +75,7 @@ public List extractFields(Type typeEntity, String entityType) { SchemaClient schemaClient = new CustomSchemaClient(schemaUri); Deque processingStack = new ArrayDeque<>(); Set processedFields = new HashSet<>(); - Map fieldTypesMap = entityFieldsCache.get(entityType); + Map fieldTypesMap = entityFieldsCache.get(entityType); addCustomProperties( typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields); return convertMapToFieldList(fieldTypesMap); @@ -90,7 +90,7 @@ public Map> extractAllCustomProperties( SchemaClient schemaClient = new CustomSchemaClient(schemaUri); EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties")); Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, Include.ALL, false); - Map fieldTypesMap = new LinkedHashMap<>(); + Map fieldTypesMap = new LinkedHashMap<>(); Set processedFields = new HashSet<>(); Deque processingStack = new ArrayDeque<>(); addCustomProperties( @@ -170,7 +170,7 @@ private static Schema loadMainSchema( private static void extractFieldsFromSchema( Schema schema, String parentPath, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { if (processingStack.contains(schema)) { @@ -206,7 +206,8 @@ private static void extractFieldsFromSchema( arraySchema, fullFieldName, fieldTypesMap, processingStack, processedFields); } else { String fieldType = mapSchemaTypeToSimpleType(fieldSchema); - fieldTypesMap.putIfAbsent(fullFieldName, fieldType); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, fieldType, null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: '{}'", fullFieldName, fieldType); // Recursively process nested objects or arrays @@ -220,7 +221,7 @@ private static void extractFieldsFromSchema( handleArraySchema(arraySchema, parentPath, fieldTypesMap, processingStack, processedFields); } else { String fieldType = mapSchemaTypeToSimpleType(schema); - fieldTypesMap.putIfAbsent(parentPath, fieldType); + fieldTypesMap.putIfAbsent(parentPath, new FieldDefinition(parentPath, fieldType, null)); LOG.debug("Added field '{}', Type: '{}'", parentPath, fieldType); } } finally { @@ -231,7 +232,7 @@ private static void extractFieldsFromSchema( private static void handleReferenceSchema( ReferenceSchema referenceSchema, String fullFieldName, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { @@ -239,7 +240,8 @@ private static void handleReferenceSchema( String referenceType = determineReferenceType(refUri); if (referenceType != null) { - fieldTypesMap.putIfAbsent(fullFieldName, referenceType); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, referenceType, null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: '{}'", fullFieldName, referenceType); if (referenceType.startsWith("array<") && referenceType.endsWith(">")) { @@ -255,7 +257,7 @@ private static void handleReferenceSchema( referredSchema, fullFieldName, fieldTypesMap, processingStack, processedFields); } } else { - fieldTypesMap.putIfAbsent(fullFieldName, "object"); + fieldTypesMap.putIfAbsent(fullFieldName, new FieldDefinition(fullFieldName, "object", null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: 'object'", fullFieldName); extractFieldsFromSchema( @@ -270,7 +272,7 @@ private static void handleReferenceSchema( private static void handleArraySchema( ArraySchema arraySchema, String fullFieldName, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { @@ -282,7 +284,8 @@ private static void handleArraySchema( if (itemsReferenceType != null) { String arrayFieldType = "array<" + itemsReferenceType + ">"; - fieldTypesMap.putIfAbsent(fullFieldName, arrayFieldType); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, arrayFieldType, null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: '{}'", fullFieldName, arrayFieldType); Schema referredItemsSchema = itemsReferenceSchema.getReferredSchema(); @@ -292,7 +295,8 @@ private static void handleArraySchema( } } String arrayType = mapSchemaTypeToSimpleType(itemsSchema); - fieldTypesMap.putIfAbsent(fullFieldName, "array<" + arrayType + ">"); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, "array<" + arrayType + ">", null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: 'array<{}>'", fullFieldName, arrayType); @@ -306,7 +310,7 @@ private void addCustomProperties( Type typeEntity, String schemaUri, SchemaClient schemaClient, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { if (typeEntity == null || typeEntity.getCustomProperties() == null) { @@ -320,9 +324,13 @@ private void addCustomProperties( LOG.debug("Processing custom property '{}'", fullFieldName); + Object customPropertyConfigObj = customProperty.getCustomPropertyConfig(); + if (isEntityReferenceList(propertyType)) { String referenceType = "array"; - fieldTypesMap.putIfAbsent(fullFieldName, referenceType); + FieldDefinition referenceFieldDefinition = + new FieldDefinition(fullFieldName, referenceType, customPropertyConfigObj); + fieldTypesMap.putIfAbsent(fullFieldName, referenceFieldDefinition); processedFields.add(fullFieldName); LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType); @@ -337,7 +345,9 @@ private void addCustomProperties( } } else if (isEntityReference(propertyType)) { String referenceType = "entityReference"; - fieldTypesMap.putIfAbsent(fullFieldName, referenceType); + FieldDefinition referenceFieldDefinition = + new FieldDefinition(fullFieldName, referenceType, customPropertyConfigObj); + fieldTypesMap.putIfAbsent(fullFieldName, referenceFieldDefinition); processedFields.add(fullFieldName); LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType); @@ -351,17 +361,22 @@ private void addCustomProperties( fullFieldName); } } else { - fieldTypesMap.putIfAbsent(fullFieldName, propertyType); + FieldDefinition entityFieldDefinition = + new FieldDefinition(fullFieldName, propertyType, customPropertyConfigObj); + fieldTypesMap.putIfAbsent(fullFieldName, entityFieldDefinition); processedFields.add(fullFieldName); LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, propertyType); } } } - private List convertMapToFieldList(Map fieldTypesMap) { + private List convertMapToFieldList(Map fieldTypesMap) { List fieldsList = new ArrayList<>(); - for (Map.Entry entry : fieldTypesMap.entrySet()) { - fieldsList.add(new FieldDefinition(entry.getKey(), entry.getValue())); + for (Map.Entry entry : fieldTypesMap.entrySet()) { + FieldDefinition fieldDef = entry.getValue(); + fieldsList.add( + new FieldDefinition( + fieldDef.getName(), fieldDef.getType(), fieldDef.getCustomPropertyConfig())); } return fieldsList; } @@ -622,10 +637,12 @@ private String mapUrlToResourcePath(String url) { public static class FieldDefinition { private String name; private String type; + private Object customPropertyConfig; - public FieldDefinition(String name, String type) { + public FieldDefinition(String name, String type, Object customPropertyConfig) { this.name = name; this.type = type; + this.customPropertyConfig = customPropertyConfig; } } } From 8753ee9dbd2dcf67a9d12a9ff603a7cb97937281 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Wed, 18 Dec 2024 16:40:20 +0530 Subject: [PATCH 3/3] added config list in the entity ref list --- .../AdvanceSearchProvider.component.tsx | 20 ++++++--- .../src/constants/AdvancedSearch.constants.ts | 3 ++ .../ui/src/utils/AdvancedSearchClassBase.ts | 43 +++++++++---------- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx index 9e7cb76f857d..c953f7e5318f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx @@ -223,13 +223,21 @@ export const AdvanceSearchProvider = ({ Object.entries(res).forEach(([_, fields]) => { if (Array.isArray(fields) && fields.length > 0) { - fields.forEach((field: { name: string; type: string }) => { - if (field.name && field.type) { - const { subfieldsKey, dataObject } = - advancedSearchClassBase.getCustomPropertiesSubFields(field); - subfields[subfieldsKey] = dataObject; + fields.forEach( + (field: { + name: string; + type: string; + customPropertyConfig: { + config: string | string[]; + }; + }) => { + if (field.name && field.type) { + const { subfieldsKey, dataObject } = + advancedSearchClassBase.getCustomPropertiesSubFields(field); + subfields[subfieldsKey] = dataObject; + } } - }); + ); } }); } catch (error) { diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts index 800185fffd10..83240fb84ca4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts @@ -298,6 +298,9 @@ export const TEXT_FIELD_OPERATORS = [ 'is_null', 'is_not_null', ]; + +export const DATE_FIELD_OPERATORS = ['between', 'not_between']; + /** * Generates a query builder tree with a group containing an empty rule */ diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts index 91d7d01ed1ec..16a5b26f457c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts @@ -22,7 +22,10 @@ import { SelectFieldSettings, } from 'react-awesome-query-builder'; import AntdConfig from 'react-awesome-query-builder/lib/config/antd'; -import { TEXT_FIELD_OPERATORS } from '../constants/AdvancedSearch.constants'; +import { + DATE_FIELD_OPERATORS, + TEXT_FIELD_OPERATORS, +} from '../constants/AdvancedSearch.constants'; import { EntityFields, SuggestionField } from '../enums/AdvancedSearch.enum'; import { SearchIndex } from '../enums/search.enum'; import { getAggregateFieldOptions } from '../rest/miscAPI'; @@ -796,42 +799,40 @@ class AdvancedSearchClassBase { }; }; - public getCustomPropertiesSubFields(field: { name: string; type: string }) { + public getCustomPropertiesSubFields(field: { + name: string; + type: string; + customPropertyConfig: { + config: string | string[]; + }; + }) { { switch (field.type) { case 'array': case 'entityReference': return { - subfieldsKey: field.name + `.fullyQualifiedName`, + subfieldsKey: field.name + `.name`, dataObject: { type: 'select', label: field.name, fieldSettings: { asyncFetch: advancedSearchClassBase.autocomplete({ - searchIndex: [SearchIndex.USER, SearchIndex.TEAM], - entityField: EntityFields.DISPLAY_NAME_KEYWORD, + searchIndex: ( + (field.customPropertyConfig.config ?? []) as string[] + ).join(',') as SearchIndex, + entityField: EntityFields.NAME_KEYWORD, }), useAsyncSearch: true, }, }, }; - case 'date-cp': { - return { - subfieldsKey: field.name, - dataObject: { - type: 'date', - valueSources: ['value'], - operators: TEXT_FIELD_OPERATORS, - }, - }; - } + case 'date-cp': case 'dateTime-cp': { return { subfieldsKey: field.name, dataObject: { - type: 'datetime', - valueSources: ['value'], - operators: TEXT_FIELD_OPERATORS, + type: 'date', + operators: DATE_FIELD_OPERATORS, }, }; } @@ -840,8 +841,7 @@ class AdvancedSearchClassBase { subfieldsKey: field.name, dataObject: { type: 'time', - valueSources: ['value'], - operators: TEXT_FIELD_OPERATORS, + operators: DATE_FIELD_OPERATORS, }, }; } @@ -851,9 +851,6 @@ class AdvancedSearchClassBase { dataObject: { type: 'text', label: field.name, - fieldSettings: { - minimum_should_match: 1, - }, }, }; }