diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/SystemSettingsException.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/SystemSettingsException.java new file mode 100644 index 000000000000..886009acfd44 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/SystemSettingsException.java @@ -0,0 +1,12 @@ +package org.openmetadata.service.exception; + +import javax.ws.rs.core.Response; +import org.openmetadata.sdk.exception.WebServiceException; + +public class SystemSettingsException extends WebServiceException { + private static final String ERROR_TYPE = "SYSTEM_SETTINGS_EXCEPTION"; + + public SystemSettingsException(String message) { + super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java index 9bc3e78e303d..9ecb68997bf1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java @@ -2,6 +2,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.settings.SettingsType.LINEAGE_SETTINGS; +import static org.openmetadata.schema.settings.SettingsType.SEARCH_SETTINGS; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Hidden; @@ -13,6 +14,8 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.IOException; +import java.util.List; import javax.json.JsonPatch; import javax.validation.Valid; import javax.ws.rs.Consumes; @@ -30,6 +33,10 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.GlobalSettings; +import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.auth.EmailRequest; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; @@ -42,7 +49,9 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; +import org.openmetadata.service.exception.SystemSettingsException; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.resources.Collection; @@ -50,6 +59,8 @@ import org.openmetadata.service.security.JwtFilter; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.email.EmailUtil; @@ -67,6 +78,7 @@ public class SystemResource { private OpenMetadataApplicationConfig applicationConfig; private PipelineServiceClientInterface pipelineServiceClient; private JwtFilter jwtFilter; + private SearchSettings defaultSearchSettingsCache = new SearchSettings(); public SystemResource(Authorizer authorizer) { this.systemRepository = Entity.getSystemRepository(); @@ -81,12 +93,50 @@ public void initialize(OpenMetadataApplicationConfig config) { this.jwtFilter = new JwtFilter(config.getAuthenticationConfiguration(), config.getAuthorizerConfiguration()); + loadDefaultSearchSettings(false); } public static class SettingsList extends ResultList { /* Required for serde */ } + public SearchSettings readDefaultSearchSettings() { + if (defaultSearchSettingsCache != null) { + try { + List jsonDataFiles = + EntityUtil.getJsonDataResources(".*/json/data/searchSettings/searchSettings.json"); + if (!jsonDataFiles.isEmpty()) { + String json = + CommonUtil.getResourceAsStream( + EntityRepository.class.getClassLoader(), jsonDataFiles.get(0)); + defaultSearchSettingsCache = JsonUtils.readValue(json, SearchSettings.class); + } + } catch (IOException e) { + LOG.error("Failed to read default search settings. Message: {}", e.getMessage(), e); + } + } + return defaultSearchSettingsCache; + } + + public SearchSettings loadDefaultSearchSettings(boolean force) { + SearchSettings searchSettings = readDefaultSearchSettings(); + if (!force) { + Settings existingSettings = + systemRepository.getConfigWithKey(String.valueOf(SEARCH_SETTINGS)); + if (existingSettings != null && existingSettings.getConfigValue() != null) { + SearchSettings existingSearchSettings = (SearchSettings) existingSettings.getConfigValue(); + if (existingSearchSettings.getGlobalSettings() != null) { + return searchSettings; + } + } + } + Settings settings = + new Settings().withConfigType(SettingsType.SEARCH_SETTINGS).withConfigValue(searchSettings); + systemRepository.createOrUpdate(settings); + LOG.info("Default searchSettings loaded successfully."); + return searchSettings; + } + @GET @Path("/settings") @Operation( @@ -186,9 +236,93 @@ public Response createOrUpdateSetting( @Context SecurityContext securityContext, @Valid Settings settingName) { authorizer.authorizeAdmin(securityContext); + if (SettingsType.SEARCH_SETTINGS + .value() + .equalsIgnoreCase(settingName.getConfigType().toString())) { + SearchSettings defaultSearchSettings = loadDefaultSearchSettings(false); + SearchSettings incomingSearchSettings = + JsonUtils.convertValue(settingName.getConfigValue(), SearchSettings.class); + + GlobalSettings defaultGlobalSettings = defaultSearchSettings.getGlobalSettings(); + GlobalSettings incomingGlobalSettings = incomingSearchSettings.getGlobalSettings(); + + GlobalSettings mergedGlobalSettings = new GlobalSettings(); + + mergedGlobalSettings.setMaxAggregateSize( + incomingGlobalSettings != null && incomingGlobalSettings.getMaxAggregateSize() != null + ? incomingGlobalSettings.getMaxAggregateSize() + : defaultGlobalSettings.getMaxAggregateSize()); + + mergedGlobalSettings.setMaxResultHits( + incomingGlobalSettings != null && incomingGlobalSettings.getMaxResultHits() != null + ? incomingGlobalSettings.getMaxResultHits() + : defaultGlobalSettings.getMaxResultHits()); + + mergedGlobalSettings.setMaxAnalyzedOffset( + incomingGlobalSettings != null && incomingGlobalSettings.getMaxAnalyzedOffset() != null + ? incomingGlobalSettings.getMaxAnalyzedOffset() + : defaultGlobalSettings.getMaxAnalyzedOffset()); + + mergedGlobalSettings.setAggregations(defaultGlobalSettings.getAggregations()); + mergedGlobalSettings.setHighlightFields(defaultGlobalSettings.getHighlightFields()); + + incomingSearchSettings.setGlobalSettings(mergedGlobalSettings); + + if (incomingSearchSettings.getDefaultConfiguration() == null) { + incomingSearchSettings.setDefaultConfiguration( + defaultSearchSettings.getDefaultConfiguration()); + } + + List defaultAssetTypes = + defaultSearchSettings.getAssetTypeConfigurations(); + List incomingAssetTypes = + incomingSearchSettings.getAssetTypeConfigurations(); + + for (AssetTypeConfiguration defaultConfig : defaultAssetTypes) { + String assetType = defaultConfig.getAssetType().toLowerCase(); + boolean exists = + incomingAssetTypes.stream() + .anyMatch(config -> config.getAssetType().equalsIgnoreCase(assetType)); + if (!exists) { + incomingAssetTypes.add(defaultConfig); + } + } + + settingName.setConfigValue(incomingSearchSettings); + } return systemRepository.createOrUpdate(settingName); } + @PUT + @Path("/settings/reset/{name}") + @Operation( + operationId = "resetSettingToDefault", + summary = "Reset a setting to default", + description = "Reset the specified setting to its default value.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Settings reset to default", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Settings.class))) + }) + public Response resetSettingToDefault( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Name of the setting", schema = @Schema(type = "string")) + @PathParam("name") + String name) { + authorizer.authorizeAdmin(securityContext); + + if (!SettingsType.SEARCH_SETTINGS.value().equalsIgnoreCase(name)) { + throw new SystemSettingsException("Resetting of setting '" + name + "' is not supported."); + } + SearchSettings settings = loadDefaultSearchSettings(true); + return Response.ok(settings).build(); + } + @PUT @Path("/email/test") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java index 9fbfa1555395..208c9a012a03 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java @@ -1,7 +1,5 @@ package org.openmetadata.service.search.indexes; -import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -135,7 +133,6 @@ private void parseSchemaFields( public static Map getFields() { Map fields = SearchIndex.getDefaultFields(); - fields.put(ES_MESSAGE_SCHEMA_FIELD, 7.0f); fields.put("responseSchema.schemaFields.name.keyword", 5.0f); fields.put("responseSchema.schemaFields.description", 1.0f); fields.put("responseSchema.schemaFields.children.name", 7.0f); diff --git a/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json b/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json new file mode 100644 index 000000000000..b468cc4a6993 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/searchSettings/searchSettings.json @@ -0,0 +1,503 @@ +{ + "enableAccessControl": false, + "globalSettings": { + "maxAggregateSize": 10000, + "maxResultHits": 10000, + "maxAnalyzedOffset": 1000, + "aggregations": [ + { + "name": "serviceType", + "type": "terms", + "field": "serviceType" + }, + { + "name": "service.displayName.keyword", + "type": "terms", + "field": "service.displayName.keyword" + }, + { + "name": "entityType", + "type": "terms", + "field": "entityType" + }, + { + "name": "tier.tagFQN", + "type": "terms", + "field": "tier.tagFQN" + }, + { + "name": "certification.tagLabel.tagFQN", + "type": "terms", + "field": "certification.tagLabel.tagFQN" + }, + { + "name": "owners.displayName.keyword", + "type": "terms", + "field": "owners.displayName.keyword" + }, + { + "name": "domain.displayName.keyword", + "type": "terms", + "field": "domain.displayName.keyword" + }, + { + "name": "tags.tagFQN", + "type": "terms", + "field": "tags.tagFQN" + } + ], + "highlightFields": ["name", "displayName", "description", "displayName.ngram", "name.ngram"] + }, + "assetTypeConfigurations": [ + { + "assetType": "table", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0, + "columns.name": 5.0, + "columns.displayName": 5.0, + "columns.description": 2.0, + "columns.children.name": 3.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": ["displayName", "description", "columns.name", "columns.description", "columns.children.name"], + "highlightFields": ["name","description","displayName","columns.name","columns.description","columns.children.name"], + "aggregations": [ + { + "name": "database", + "type": "terms", + "field": "database.displayName.keyword" + }, + { + "name": "databaseSchema", + "type": "terms", + "field": "databaseSchema.displayName.keyword" + }, + { + "name": "columns", + "type": "terms", + "field": "columns.name.keyword" + }, + { + "name": "columnNames", + "type": "terms", + "field": "columnNames" + }, + { + "name": "tableType", + "type": "terms", + "field": "tableType" + } + ], + "boosts": [ + { "field": "columns.name", "boost": 5.0 }, + { "field": "columns.displayName", "boost": 5.0 }, + { "field": "columns.description", "boost": 2.0 }, + { "field": "columns.children.name", "boost": 3.0 } + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "pipeline", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0, + "tasks.name": 8.0, + "tasks.description": 1.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": ["displayName", "description", "tasks.name", "tasks.description"], + "highlightFields": ["tasks.name", "tasks.description"], + "aggregations": [ + { + "name": "tasks", + "type": "terms", + "field": "tasks.displayName.keyword" + } + ], + "boosts": [ + { "field": "tasks.name", "boost": 8.0 }, + { "field": "tasks.description", "boost": 1.0 } + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "mlmodel", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0, + "mlFeatures.name": 8.0, + "mlFeatures.description": 1.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": ["displayName", "description", "mlFeatures.name", "mlFeatures.description"], + "highlightFields": ["mlFeatures.name", "mlFeatures.description"], + "boosts": [ + { "field": "mlFeatures.name", "boost": 8.0 }, + { "field": "mlFeatures.description", "boost": 1.0 } + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "topic", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0, + "messageSchema.schemaText": 7.0, + "messageSchema.schemaFields.name.keyword": 5.0, + "messageSchema.schemaFields.description": 1.0, + "messageSchema.schemaFields.children.name": 7.0, + "messageSchema.schemaFields.children.keyword": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": [ + "displayName", + "description", + "messageSchema.schemaText", + "messageSchema.schemaFields.name.keyword", + "messageSchema.schemaFields.description", + "messageSchema.schemaFields.children.name" + ], + "highlightFields": [], + "aggregations": [ + { + "name": "messageSchema", + "type": "terms", + "field": "messageSchema.schemaText.keyword" + }, + { + "name": "schemaFieldNames", + "type": "terms", + "field": "schemaFieldNames" + } + ], + "boosts": [ + { "field": "messageSchema.schemaText", "boost": 7.0 }, + { "field": "messageSchema.schemaFields.name.keyword", "boost": 5.0 }, + { "field": "messageSchema.schemaFields.description", "boost": 1.0 }, + { "field": "messageSchema.schemaFields.children.name", "boost": 7.0 }, + { "field": "messageSchema.schemaFields.children.keyword", "boost": 5.0 } + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "dashboard", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0, + "charts.name": 7.0, + "charts.displayName": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": [ + "displayName", + "description", + "charts.name", + "charts.displayName" + ], + "highlightFields": [], + "aggregations": [ + { + "name": "dataModels.displayName.keyword", + "type": "terms", + "field": "dataModels.displayName.keyword" + }, + { + "name": "project.keyword", + "type": "terms", + "field": "project.keyword" + }, + { + "name": "charts.displayName.keyword", + "type": "terms", + "field": "charts.displayName.keyword" + } + ], + "boosts": [ + { "field": "charts.name", "boost": 2.0 }, + { "field": "charts.description", "boost": 5.0 } + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "dashboardDataModel", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName", "query", "query.ngram"], + "shouldMatch": ["columns.name", "columns.description", "columns.children.name", "columns.displayName"], + "highlightFields": [], + "aggregations": [ + { + "name": "dataModelType", + "type": "terms", + "field": "dataModelType" + }, + { + "name": "project.keyword", + "type": "terms", + "field": "project.keyword" + }, + { + "name": "columnNames", + "type": "terms", + "field": "columnNames" + }, + { + "name": "columns.name.keyword", + "type": "terms", + "field": "columns.name.keyword" + } + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "container", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": [ + "classfiication.name", + "description" + ], + "highlightFields": ["dataModel.columns.name", "dataModel.columns.description", "dataModel.columns.children.name"], + "aggregations": [ + { + "name": "dataModel.columns.name.keyword", + "type": "terms", + "field": "dataModel.columns.name.keyword" + }, + { + "name": "columnNames", + "type": "terms", + "field": "columnNames" + } + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "storedProcedure", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName", "query", "query.ngram"], + "shouldMatch": [], + "highlightFields": [], + "aggregations": [ + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "apiEndpoint", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName", "query", "query.ngram"], + "shouldMatch": ["responseSchema.schemaFields.name.keyword", "responseSchema.schemaFields.description", + "responseSchema.schemaFields.children.name", "responseSchema.schemaFields.children.keyword"], + "highlightFields": [], + "aggregations": [ + { + "name": "responseSchema.schemaFields.name", + "type": "terms", + "field": "responseSchema.schemaFields.name.keyword" + }, + { + "name": "fieldNames", + "type": "terms", + "field": "fieldNames" + } + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "query", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName", "query", "query.ngram"], + "highlightFields": [], + "aggregations": [ + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "dataAsset", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": [ + "displayName", + "description", + "charts.name", + "charts.displayName" + ], + "highlightFields": [], + "aggregations": [ + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "glossaryTerm", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": [ + "displayName", + "description", + "synonyms", + "synonyms.ngram", + "glossary.name", + "glossary.displayName" + ], + "highlightFields": ["synonyms"], + "aggregations": [ + { + "name": "glossary.name.keyword", + "type": "terms", + "field": "glossary.name.keyword" + }, + { + "name": "fqnParts_agg", + "type": "terms", + "field": "fqnParts" + } + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "tag", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": [ + "classfiication.name", + "description" + ], + "highlightFields": [], + "aggregations": [ + { + "name": "classification.name.keyword", + "type": "terms", + "field": "classification.name.keyword" + } + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + }, + { + "assetType": "testCase", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName", "query", "query.ngram"], + "shouldMatch": ["testSuite.fullyQualifiedName", "testSuite.name", "testSuite.description", "entityLink", "entityFQN"], + "highlightFields": ["testSuite.name", "testSuite.description"], + "aggregations": [ + ], + "boosts": [ + ], + "scoreMode": "sum", + "boostMode": "multiply" + } + ], + "defaultConfiguration": { + "assetType": "default", + "fields": { + "name": 10.0, + "displayName": 10.0, + "description": 2.0, + "fullyQualifiedName": 5.0, + "fullyQualifiedNameParts": 5.0 + }, + "mustMatch": ["name", "fullyQualifiedName"], + "shouldMatch": ["displayName", "description"], + "highlightFields": [], + "aggregations": [], + "scoreMode": "sum", + "boostMode": "multiply" + } +} \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index 1cf64c31e5ea..0a64a05d79d2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -1,6 +1,8 @@ package org.openmetadata.service.resources.system; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS; @@ -14,6 +16,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.validation.Validator; import javax.ws.rs.client.WebTarget; @@ -25,7 +28,10 @@ import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.openmetadata.api.configuration.LogoConfiguration; import org.openmetadata.api.configuration.ThemeConfiguration; import org.openmetadata.api.configuration.UiThemePreference; @@ -34,6 +40,9 @@ import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; import org.openmetadata.schema.api.data.*; import org.openmetadata.schema.api.lineage.LineageSettings; +import org.openmetadata.schema.api.search.Aggregation; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.Fields; import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.api.services.CreateDashboardService; import org.openmetadata.schema.api.services.CreateDatabaseService; @@ -86,6 +95,8 @@ @Slf4j @TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.CONCURRENT) public class SystemResourceTest extends OpenMetadataApplicationTest { static OpenMetadataApplicationConfig config; @@ -462,8 +473,9 @@ void testLoginConfigurationSettings() throws HttpResponseException { assertEquals(7200, updatedLoginConfig.getJwtTokenExpiryTime()); } + @Order(1) @Test - void testSearchSettings() throws HttpResponseException { + void testGetDefaultSearchSettings() throws HttpResponseException { // Retrieve the default search settings Settings searchSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); SearchSettings searchConfig = @@ -472,21 +484,208 @@ void testSearchSettings() throws HttpResponseException { // Assert default values assertEquals(false, searchConfig.getEnableAccessControl()); - // Update search settings + // Verify that global settings are loaded + assertNotNull(searchConfig.getGlobalSettings()); + assertEquals(10000, searchConfig.getGlobalSettings().getMaxAggregateSize()); + assertEquals(10000, searchConfig.getGlobalSettings().getMaxResultHits()); + assertEquals(1000, searchConfig.getGlobalSettings().getMaxAnalyzedOffset()); + + // Verify that aggregations are loaded + assertNotNull(searchConfig.getGlobalSettings().getAggregations()); + assertFalse(searchConfig.getGlobalSettings().getAggregations().isEmpty()); + + // Verify that highlight fields are loaded + assertNotNull(searchConfig.getGlobalSettings().getHighlightFields()); + assertFalse(searchConfig.getGlobalSettings().getHighlightFields().isEmpty()); + + // Verify that asset type configurations are loaded + assertNotNull(searchConfig.getAssetTypeConfigurations()); + assertFalse(searchConfig.getAssetTypeConfigurations().isEmpty()); + + // Check if 'table' asset type configuration exists + boolean tableConfigExists = + searchConfig.getAssetTypeConfigurations().stream() + .anyMatch(config -> "table".equalsIgnoreCase(config.getAssetType())); + assertTrue(tableConfigExists); + + // Verify default configuration is loaded + assertNotNull(searchConfig.getDefaultConfiguration()); + } + + @Test + void testResetSearchSettingsToDefault() throws HttpResponseException { + Settings searchSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings searchConfig = + JsonUtils.convertValue(searchSettings.getConfigValue(), SearchSettings.class); + searchConfig.setEnableAccessControl(true); + searchConfig.getGlobalSettings().setMaxAggregateSize(5000); + + AssetTypeConfiguration tableConfig = + searchConfig.getAssetTypeConfigurations().stream() + .filter(config -> "table".equalsIgnoreCase(config.getAssetType())) + .findFirst() + .orElseThrow(() -> new AssertionError("Table configuration not found")); - Settings updatedSearchSettings = + tableConfig.getFields().getAdditionalProperties().put("name", 20.0); + + Settings updatedSettings = new Settings().withConfigType(SettingsType.SEARCH_SETTINGS).withConfigValue(searchConfig); - updateSystemConfig(updatedSearchSettings); + updateSystemConfig(updatedSettings); - // Retrieve the updated settings - Settings updatedSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + Settings modifiedSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings modifiedSearchConfig = + JsonUtils.convertValue(modifiedSettings.getConfigValue(), SearchSettings.class); + + assertEquals(true, modifiedSearchConfig.getEnableAccessControl()); + assertEquals(5000, modifiedSearchConfig.getGlobalSettings().getMaxAggregateSize()); + assertEquals( + 20.0, + modifiedSearchConfig.getAssetTypeConfigurations().stream() + .filter(config -> "table".equalsIgnoreCase(config.getAssetType())) + .findFirst() + .get() + .getFields() + .getAdditionalProperties() + .get("name")); + + // Step 2: Reset the settings to default + resetSystemConfig(SettingsType.SEARCH_SETTINGS); + + // Retrieve the settings after reset + Settings resetSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings resetSearchConfig = + JsonUtils.convertValue(resetSettings.getConfigValue(), SearchSettings.class); + + // Verify that the settings are reset to default values + assertEquals(false, resetSearchConfig.getEnableAccessControl()); + assertEquals(10000, resetSearchConfig.getGlobalSettings().getMaxAggregateSize()); + assertEquals( + 10.0, + resetSearchConfig.getAssetTypeConfigurations().stream() + .filter(config -> "table".equalsIgnoreCase(config.getAssetType())) + .findFirst() + .get() + .getFields() + .getAdditionalProperties() + .get("name")); + } + + @Test + void testGlobalSettingsModification() throws HttpResponseException { + // Step 1: Retrieve current settings + Settings searchSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings searchConfig = + JsonUtils.convertValue(searchSettings.getConfigValue(), SearchSettings.class); + SystemResource systemResource = new SystemResource(null); + SearchSettings defaultSearchSettings = systemResource.readDefaultSearchSettings(); + + // Step 2: Modify allowed fields in globalSettings + searchConfig.getGlobalSettings().setMaxAggregateSize(5000); + searchConfig.getGlobalSettings().setMaxResultHits(8000); + searchConfig.getGlobalSettings().setMaxAnalyzedOffset(2000); + + // Step 3: Attempt to modify restricted fields + List originalAggregations = searchConfig.getGlobalSettings().getAggregations(); + List originalHighlightFields = searchConfig.getGlobalSettings().getHighlightFields(); + + // Modify restricted fields + searchConfig + .getGlobalSettings() + .setAggregations(new ArrayList<>()); // Attempt to clear aggregations + searchConfig.getGlobalSettings().setHighlightFields(Arrays.asList("modifiedField")); + + // Step 4: Save the updated settings + Settings updatedSettings = + new Settings().withConfigType(SettingsType.SEARCH_SETTINGS).withConfigValue(searchConfig); + updateSystemConfig(updatedSettings); + + // Step 5: Retrieve the settings after update + Settings retrievedSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); SearchSettings updatedSearchConfig = - JsonUtils.convertValue(updatedSettings.getConfigValue(), SearchSettings.class); + JsonUtils.convertValue(retrievedSettings.getConfigValue(), SearchSettings.class); - // Assert updated values - assertEquals(true, updatedSearchConfig.getEnableAccessControl()); + // Step 6: Verify that allowed fields have been updated + assertEquals(5000, updatedSearchConfig.getGlobalSettings().getMaxAggregateSize()); + assertEquals(8000, updatedSearchConfig.getGlobalSettings().getMaxResultHits()); + assertEquals(2000, updatedSearchConfig.getGlobalSettings().getMaxAnalyzedOffset()); + + // Step 7: Verify that restricted fields have not changed + assertEquals( + defaultSearchSettings.getGlobalSettings().getAggregations(), + updatedSearchConfig.getGlobalSettings().getAggregations()); + assertEquals( + defaultSearchSettings.getGlobalSettings().getHighlightFields(), + updatedSearchConfig.getGlobalSettings().getHighlightFields()); + } + + @Test + void testCannotDeleteAssetType() throws HttpResponseException { + // Retrieve current settings + Settings searchSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings searchConfig = + JsonUtils.convertValue(searchSettings.getConfigValue(), SearchSettings.class); + + // Remove an asset type, e.g., 'table' + searchConfig + .getAssetTypeConfigurations() + .removeIf(config -> "table".equalsIgnoreCase(config.getAssetType())); + + // Save the updated settings + Settings updatedSettings = + new Settings().withConfigType(SettingsType.SEARCH_SETTINGS).withConfigValue(searchConfig); + updateSystemConfig(updatedSettings); + + // Retrieve the settings after update + Settings retrievedSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings updatedSearchConfig = + JsonUtils.convertValue(retrievedSettings.getConfigValue(), SearchSettings.class); + + // Verify that 'table' asset type still exists + boolean tableConfigExists = + updatedSearchConfig.getAssetTypeConfigurations().stream() + .anyMatch(config -> "table".equalsIgnoreCase(config.getAssetType())); + assertTrue(tableConfigExists); + } + + @Test + void testCanAddNewAssetType() throws HttpResponseException { + // Retrieve current settings + Settings searchSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings searchConfig = + JsonUtils.convertValue(searchSettings.getConfigValue(), SearchSettings.class); + + // Create a new asset type configuration + AssetTypeConfiguration newAssetType = + new AssetTypeConfiguration() + .withAssetType("newAsset") + .withFields(new Fields()) + .withShouldMatch(Arrays.asList("description")) + .withHighlightFields(Arrays.asList("name", "description")) + .withAggregations(new ArrayList<>()) + .withBoosts(new ArrayList<>()) + .withScoreMode(AssetTypeConfiguration.ScoreMode.MULTIPLY) + .withBoostMode(AssetTypeConfiguration.BoostMode.MULTIPLY); + + // Add the new asset type + searchConfig.getAssetTypeConfigurations().add(newAssetType); + + // Save the updated settings + Settings updatedSettings = + new Settings().withConfigType(SettingsType.SEARCH_SETTINGS).withConfigValue(searchConfig); + updateSystemConfig(updatedSettings); + + // Retrieve the settings after update + Settings retrievedSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS); + SearchSettings updatedSearchConfig = + JsonUtils.convertValue(retrievedSettings.getConfigValue(), SearchSettings.class); + + // Verify that the new asset type is added + boolean newAssetTypeExists = + updatedSearchConfig.getAssetTypeConfigurations().stream() + .anyMatch(config -> "newAsset".equalsIgnoreCase(config.getAssetType())); + assertTrue(newAssetTypeExists); } @Test @@ -680,4 +879,9 @@ private static void createSystemConfig(Settings updatedSetting) throws HttpRespo WebTarget target = getResource("system/settings"); TestUtils.put(target, updatedSetting, Response.Status.CREATED, ADMIN_AUTH_HEADERS); } + + private void resetSystemConfig(SettingsType settingsType) throws HttpResponseException { + WebTarget target = getResource("system/settings/reset/" + settingsType.value()); + TestUtils.put(target, Response.Status.OK, ADMIN_AUTH_HEADERS); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index e39f2d05a6e5..b16714d646d2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -50,6 +50,7 @@ import javax.json.JsonObject; import javax.validation.constraints.Size; import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -355,6 +356,13 @@ public static void put( readResponse(response, expectedStatus.getStatusCode()); } + public static void put(WebTarget target, Status expectedStatus, Map headers) + throws HttpResponseException { + Invocation.Builder builder = SecurityUtil.addHeaders(target, headers); + Response response = builder.method("PUT"); + readResponse(response, expectedStatus.getStatusCode()); + } + public static T put( WebTarget target, K request, Class clz, Status expectedStatus, Map headers) throws HttpResponseException { diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json b/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json index 1441f4f472b9..a0d8267d4f1b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json @@ -2,7 +2,7 @@ "$id": "https://open-metadata.org/schema/entity/configuration/searchSettings.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "SearchSettings", - "description": "This schema defines the Rbac Search Configuration.", + "description": "This schema defines the Search Configuration, including ranking logic and other settings per asset type.", "type": "object", "javaType": "org.openmetadata.schema.api.search.SearchSettings", "properties": { @@ -10,6 +10,203 @@ "type": "boolean", "description": "Flag to enable or disable the RBAC Search Configuration.", "default": false + }, + "globalSettings": { + "type": "object", + "description": "Global settings for search.", + "properties": { + "maxAggregateSize": { + "type": "integer", + "default": 10000 + }, + "maxResultHits": { + "type": "integer", + "default": 10000 + }, + "maxAnalyzedOffset": { + "type": "integer", + "default": 1000 + }, + "aggregations": { + "type": "array", + "description": "List of aggregations to include in the search query.", + "items": { + "$ref": "#/definitions/aggregation" + } + }, + "highlightFields": { + "type": "array", + "items": { "type": "string" }, + "description": "Fields to include in the highlights." + } + }, + "additionalProperties": false + }, + "assetTypeConfigurations": { + "type": "array", + "description": "List of search configurations for each asset type.", + "items": { + "$ref": "#/definitions/assetTypeConfiguration" + } + }, + "defaultConfiguration": { + "$ref": "#/definitions/assetTypeConfiguration", + "description": "Default search configuration when an entity doesn't match." + } + }, + "definitions": { + "assetTypeConfiguration": { + "type": "object", + "description": "Defines the search configuration for a specific asset type.", + "properties": { + "assetType": { + "type": "string", + "description": "The type of data asset this configuration applies to." + }, + "fields": { + "type": "object", + "description": "Fields to search with their boosts.", + "additionalProperties": { + "type": "number" + } + }, + "mustMatch": { + "type": "array", + "items": { "type": "string" }, + "description": "Fields that must match in the search query." + }, + "shouldMatch": { + "type": "array", + "items": { "type": "string" }, + "description": "Fields that should match in the search query." + }, + "mustNotMatch": { + "type": "array", + "items": { "type": "string" }, + "description": "Fields that must not match." + }, + "boosts": { + "type": "array", + "items": { + "$ref": "#/definitions/fieldBoost" + }, + "description": "Boost factors for specific fields." + }, + "tagBoosts": { + "type": "array", + "items": { + "$ref": "#/definitions/tagBoost" + }, + "description": "Boost factors for specific tags." + }, + "fieldValueBoosts": { + "type": "array", + "items": { + "$ref": "#/definitions/fieldValueBoost" + }, + "description": "Boost factors based on field values with function modifiers." + }, + "highlightFields": { + "type": "array", + "items": { "type": "string" }, + "description": "Fields to include in the highlights." + }, + "aggregations": { + "type": "array", + "description": "List of aggregations to include in the search query.", + "items": { + "$ref": "#/definitions/aggregation" + } + }, + "scoreMode": { + "type": "string", + "enum": ["multiply", "sum", "avg", "first", "max", "min"], + "description": "Determines how the computed scores are combined." + }, + "boostMode": { + "type": "string", + "enum": ["multiply", "replace", "sum", "avg", "max", "min"], + "description": "Determines how the combined score and the query score are combined." + }, + "additionalSettings": { + "type": "object", + "description": "Additional settings specific to the asset type.", + "additionalProperties": true + } + }, + "required": ["assetType", "fields"], + "additionalProperties": false + }, + "fieldBoost": { + "type": "object", + "properties": { + "field": { "type": "string", "description": "Field name to boost." }, + "boost": { "type": "number", "description": "Boost factor for the field." } + }, + "required": ["field", "boost"], + "additionalProperties": false + }, + "tagBoost": { + "type": "object", + "properties": { + "tagFQN": { "type": "string", "description": "Fully Qualified Name of the tag." }, + "boost": { "type": "number", "description": "Boost factor for the tag." } + }, + "required": ["tagFQN", "boost"], + "additionalProperties": false + }, + "fieldValueBoost": { + "type": "object", + "properties": { + "field": { "type": "string", "description": "Field name whose value is used for boosting." }, + "factor": { "type": "number", "description": "Factor by which to multiply the field value." }, + "modifier": { + "type": "string", + "enum": ["none", "log", "log1p", "log2p", "ln", "ln1p", "ln2p", "square", "sqrt", "reciprocal"], + "description": "Modifier function to apply to the field value." + }, + "missing": { "type": "number", "description": "Value to use if the field is missing." }, + "condition": { + "type": "object", + "description": "Condition to apply the boost.", + "properties": { + "range": { + "type": "object", + "properties": { + "gt": { "type": "number", "description": "Greater than value." }, + "gte": { "type": "number", "description": "Greater than or equal to value." }, + "lt": { "type": "number", "description": "Less than value." }, + "lte": { "type": "number", "description": "Less than or equal to value." } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": ["field", "factor"], + "additionalProperties": false + }, + "aggregation": { + "type": "object", + "description": "Defines an aggregation for the search query.", + "properties": { + "name": { + "type": "string", + "description": "The name of the aggregation." + }, + "type": { + "type": "string", + "enum": ["terms", "range", "histogram", "date_histogram", "filters", "missing", "nested", "reverse_nested", "top_hits", "max", "min", "avg", "sum", "stats"], + "description": "The type of aggregation." + }, + "field": { + "type": "string", + "description": "The field to aggregate on." + } + }, + "required": ["name", "type", "field"], + "additionalProperties": false } }, "additionalProperties": false