Add LineageSettings to control Lineage Default depth, layers (#18498)

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -59,7 +59,8 @@
 import org.openmetadata.schema.analytics.WebAnalyticEvent;
 import org.openmetadata.schema.api.configuration.LoginConfiguration;
 import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration;
-import org.openmetadata.schema.api.searcg.SearchSettings;
+import org.openmetadata.schema.api.lineage.LineageSettings;
+import org.openmetadata.schema.api.search.SearchSettings;
 import org.openmetadata.schema.auth.EmailVerificationToken;
 import org.openmetadata.schema.auth.PasswordResetToken;
 import org.openmetadata.schema.auth.PersonalAccessToken;
@@ -4967,6 +4968,7 @@ public static Settings getSettings(SettingsType configType, String json) {
             case SEARCH_SETTINGS -> JsonUtils.readValue(json, SearchSettings.class);
             case ASSET_CERTIFICATION_SETTINGS -> JsonUtils.readValue(
                 json, AssetCertificationSettings.class);
+            case LINEAGE_SETTINGS -> JsonUtils.readValue(json, LineageSettings.class);
             default -> throw new IllegalArgumentException("Invalid Settings Type " + configType);
           };
       settings.setConfigValue(value);

openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java

@@ -16,6 +16,7 @@
 import static org.openmetadata.schema.settings.SettingsType.ASSET_CERTIFICATION_SETTINGS;
 import static org.openmetadata.schema.settings.SettingsType.CUSTOM_UI_THEME_PREFERENCE;
 import static org.openmetadata.schema.settings.SettingsType.EMAIL_CONFIGURATION;
+import static org.openmetadata.schema.settings.SettingsType.LINEAGE_SETTINGS;
 import static org.openmetadata.schema.settings.SettingsType.LOGIN_CONFIGURATION;
 import static org.openmetadata.schema.settings.SettingsType.SEARCH_SETTINGS;
 
@@ -30,7 +31,9 @@
 import org.openmetadata.api.configuration.ThemeConfiguration;
 import org.openmetadata.api.configuration.UiThemePreference;
 import org.openmetadata.schema.api.configuration.LoginConfiguration;
-import org.openmetadata.schema.api.searcg.SearchSettings;
+import org.openmetadata.schema.api.lineage.LineageLayer;
+import org.openmetadata.schema.api.lineage.LineageSettings;
+import org.openmetadata.schema.api.search.SearchSettings;
 import org.openmetadata.schema.configuration.AssetCertificationSettings;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.settings.Settings;
@@ -140,6 +143,20 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app
                       .withValidityPeriod("P30D"));
       systemRepository.createNewSetting(setting);
     }
+
+    Settings lineageSettings = systemRepository.getConfigWithKey(LINEAGE_SETTINGS.toString());
+    if (lineageSettings == null) {
+      // Only in case a config doesn't exist in DB we insert it
+      Settings setting =
+          new Settings()
+              .withConfigType(LINEAGE_SETTINGS)
+              .withConfigValue(
+                  new LineageSettings()
+                      .withDownstreamDepth(2)
+                      .withUpstreamDepth(2)
+                      .withLineageLayer(LineageLayer.ENTITY_LINEAGE));
+      systemRepository.createNewSetting(setting);
+    }
   }
 
   public static <T> T getSetting(SettingsType settingName, Class<T> clazz) {

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java

@@ -15,7 +15,7 @@
 import javax.ws.rs.core.Response;
 import lombok.Getter;
 import org.apache.commons.lang3.tuple.Pair;
-import org.openmetadata.schema.api.searcg.SearchSettings;
+import org.openmetadata.schema.api.search.SearchSettings;
 import org.openmetadata.schema.dataInsight.DataInsightChartResult;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList;

openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java

@@ -28,9 +28,12 @@
 import org.openmetadata.api.configuration.LogoConfiguration;
 import org.openmetadata.api.configuration.ThemeConfiguration;
 import org.openmetadata.api.configuration.UiThemePreference;
+import org.openmetadata.schema.api.configuration.LoginConfiguration;
 import org.openmetadata.schema.api.configuration.profiler.MetricConfigurationDefinition;
 import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration;
 import org.openmetadata.schema.api.data.*;
+import org.openmetadata.schema.api.lineage.LineageSettings;
+import org.openmetadata.schema.api.search.SearchSettings;
 import org.openmetadata.schema.api.services.CreateDashboardService;
 import org.openmetadata.schema.api.services.CreateDatabaseService;
 import org.openmetadata.schema.api.services.CreateMessagingService;
@@ -42,6 +45,7 @@
 import org.openmetadata.schema.api.tests.CreateTestSuite;
 import org.openmetadata.schema.auth.JWTAuthMechanism;
 import org.openmetadata.schema.auth.JWTTokenExpiry;
+import org.openmetadata.schema.configuration.AssetCertificationSettings;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
@@ -70,6 +74,7 @@
 import org.openmetadata.service.resources.services.MlModelServiceResourceTest;
 import org.openmetadata.service.resources.services.PipelineServiceResourceTest;
 import org.openmetadata.service.resources.services.StorageServiceResourceTest;
+import org.openmetadata.service.resources.settings.SettingsCache;
 import org.openmetadata.service.resources.storages.ContainerResourceTest;
 import org.openmetadata.service.resources.teams.TeamResourceTest;
 import org.openmetadata.service.resources.teams.UserResourceTest;
@@ -341,6 +346,210 @@ void validate_test() throws HttpResponseException {
     assertEquals(Boolean.TRUE, response.getMigrations().getPassed());
   }
 
+  @Test
+  void testDefaultSettingsInitialization() throws HttpResponseException {
+    SettingsCache.initialize(config);
+    Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
+    Settings uiThemeSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE);
+    SmtpSettings smtpSettings =
+        JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class);
+    assertEquals(config.getSmtpSettings().getUsername(), smtpSettings.getUsername());
+    assertEquals(config.getSmtpSettings().getEmailingEntity(), smtpSettings.getEmailingEntity());
+    UiThemePreference uiThemePreference =
+        JsonUtils.convertValue(uiThemeSettings.getConfigValue(), UiThemePreference.class);
+    assertEquals("", uiThemePreference.getCustomTheme().getPrimaryColor());
+    assertEquals("", uiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath());
+  }
+
+  @Test
+  void testEmailConfigurationSettings() throws HttpResponseException {
+    Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
+    SmtpSettings smtpSettings =
+        JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class);
+    SmtpSettings expectedSmtpSettings = config.getSmtpSettings();
+    expectedSmtpSettings.setPassword(
+        smtpSettings.getPassword()); // Password is encrypted, so we use the stored one
+    assertEquals(expectedSmtpSettings, smtpSettings);
+    smtpSettings.setUsername("updatedUsername");
+    smtpSettings.setEmailingEntity("updatedEntity");
+
+    Settings updatedEmailSettings =
+        new Settings()
+            .withConfigType(SettingsType.EMAIL_CONFIGURATION)
+            .withConfigValue(smtpSettings);
+
+    updateSystemConfig(updatedEmailSettings);
+
+    Settings updatedSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION);
+    SmtpSettings updatedSmtpSettings =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), SmtpSettings.class);
+
+    assertEquals("updatedUsername", updatedSmtpSettings.getUsername());
+    assertEquals("updatedEntity", updatedSmtpSettings.getEmailingEntity());
+  }
+
+  @Order(3)
+  @Test
+  void testUiThemePreferenceSettings() throws HttpResponseException {
+    Settings uiThemeSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE);
+    UiThemePreference uiThemePreference =
+        JsonUtils.convertValue(uiThemeSettings.getConfigValue(), UiThemePreference.class);
+    assertEquals("", uiThemePreference.getCustomTheme().getPrimaryColor());
+    assertEquals("", uiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath());
+
+    uiThemePreference.getCustomTheme().setPrimaryColor("#FFFFFF");
+    uiThemePreference.getCustomLogoConfig().setCustomLogoUrlPath("http://example.com/logo.png");
+
+    Settings updatedUiThemeSettings =
+        new Settings()
+            .withConfigType(SettingsType.CUSTOM_UI_THEME_PREFERENCE)
+            .withConfigValue(uiThemePreference);
+
+    updateSystemConfig(updatedUiThemeSettings);
+
+    Settings updatedSettings = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE);
+    UiThemePreference updatedUiThemePreference =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), UiThemePreference.class);
+
+    assertEquals("#FFFFFF", updatedUiThemePreference.getCustomTheme().getPrimaryColor());
+    assertEquals(
+        "http://example.com/logo.png",
+        updatedUiThemePreference.getCustomLogoConfig().getCustomLogoUrlPath());
+    // reset to default
+    uiThemePreference.getCustomTheme().setPrimaryColor("");
+    uiThemePreference.getCustomLogoConfig().setCustomLogoUrlPath("");
+    updatedUiThemeSettings =
+        new Settings()
+            .withConfigType(SettingsType.CUSTOM_UI_THEME_PREFERENCE)
+            .withConfigValue(uiThemePreference);
+    updateSystemConfig(updatedUiThemeSettings);
+  }
+
+  @Test
+  void testLoginConfigurationSettings() throws HttpResponseException {
+    // Retrieve the default login configuration settings
+    Settings loginSettings = getSystemConfig(SettingsType.LOGIN_CONFIGURATION);
+    LoginConfiguration loginConfig =
+        JsonUtils.convertValue(loginSettings.getConfigValue(), LoginConfiguration.class);
+
+    // Assert default values
+    assertEquals(3, loginConfig.getMaxLoginFailAttempts());
+    assertEquals(600, loginConfig.getAccessBlockTime());
+    assertEquals(3600, loginConfig.getJwtTokenExpiryTime());
+
+    // Update login configuration
+    loginConfig.setMaxLoginFailAttempts(5);
+    loginConfig.setAccessBlockTime(300);
+    loginConfig.setJwtTokenExpiryTime(7200);
+
+    Settings updatedLoginSettings =
+        new Settings()
+            .withConfigType(SettingsType.LOGIN_CONFIGURATION)
+            .withConfigValue(loginConfig);
+
+    updateSystemConfig(updatedLoginSettings);
+
+    // Retrieve the updated settings
+    Settings updatedSettings = getSystemConfig(SettingsType.LOGIN_CONFIGURATION);
+    LoginConfiguration updatedLoginConfig =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), LoginConfiguration.class);
+
+    // Assert updated values
+    assertEquals(5, updatedLoginConfig.getMaxLoginFailAttempts());
+    assertEquals(300, updatedLoginConfig.getAccessBlockTime());
+    assertEquals(7200, updatedLoginConfig.getJwtTokenExpiryTime());
+  }
+
+  @Test
+  void testSearchSettings() throws HttpResponseException {
+    // Retrieve the default search settings
+    Settings searchSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS);
+    SearchSettings searchConfig =
+        JsonUtils.convertValue(searchSettings.getConfigValue(), SearchSettings.class);
+
+    // Assert default values
+    assertEquals(false, searchConfig.getEnableAccessControl());
+
+    // Update search settings
+    searchConfig.setEnableAccessControl(true);
+
+    Settings updatedSearchSettings =
+        new Settings().withConfigType(SettingsType.SEARCH_SETTINGS).withConfigValue(searchConfig);
+
+    updateSystemConfig(updatedSearchSettings);
+
+    // Retrieve the updated settings
+    Settings updatedSettings = getSystemConfig(SettingsType.SEARCH_SETTINGS);
+    SearchSettings updatedSearchConfig =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), SearchSettings.class);
+
+    // Assert updated values
+    assertEquals(true, updatedSearchConfig.getEnableAccessControl());
+  }
+
+  @Test
+  void testAssetCertificationSettings() throws HttpResponseException {
+    // Retrieve the default asset certification settings
+    Settings certificationSettings = getSystemConfig(SettingsType.ASSET_CERTIFICATION_SETTINGS);
+    AssetCertificationSettings certificationConfig =
+        JsonUtils.convertValue(
+            certificationSettings.getConfigValue(), AssetCertificationSettings.class);
+
+    // Assert default values
+    assertEquals("Certification", certificationConfig.getAllowedClassification());
+    assertEquals("P30D", certificationConfig.getValidityPeriod());
+
+    // Update asset certification settings
+    certificationConfig.setAllowedClassification("NewCertification");
+    certificationConfig.setValidityPeriod("P60D");
+
+    Settings updatedCertificationSettings =
+        new Settings()
+            .withConfigType(SettingsType.ASSET_CERTIFICATION_SETTINGS)
+            .withConfigValue(certificationConfig);
+
+    updateSystemConfig(updatedCertificationSettings);
+
+    // Retrieve the updated settings
+    Settings updatedSettings = getSystemConfig(SettingsType.ASSET_CERTIFICATION_SETTINGS);
+    AssetCertificationSettings updatedCertificationConfig =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), AssetCertificationSettings.class);
+
+    // Assert updated values
+    assertEquals("NewCertification", updatedCertificationConfig.getAllowedClassification());
+    assertEquals("P60D", updatedCertificationConfig.getValidityPeriod());
+  }
+
+  @Test
+  void testLineageSettings() throws HttpResponseException {
+    // Retrieve the default lineage settings
+    Settings lineageSettings = getSystemConfig(SettingsType.LINEAGE_SETTINGS);
+    LineageSettings lineageConfig =
+        JsonUtils.convertValue(lineageSettings.getConfigValue(), LineageSettings.class);
+
+    // Assert default values
+    assertEquals(2, lineageConfig.getUpstreamDepth());
+    assertEquals(2, lineageConfig.getDownstreamDepth());
+
+    // Update lineage settings
+    lineageConfig.setUpstreamDepth(3);
+    lineageConfig.setDownstreamDepth(4);
+
+    Settings updatedLineageSettings =
+        new Settings().withConfigType(SettingsType.LINEAGE_SETTINGS).withConfigValue(lineageConfig);
+
+    updateSystemConfig(updatedLineageSettings);
+
+    // Retrieve the updated settings
+    Settings updatedSettings = getSystemConfig(SettingsType.LINEAGE_SETTINGS);
+    LineageSettings updatedLineageConfig =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), LineageSettings.class);
+
+    // Assert updated values
+    assertEquals(3, updatedLineageConfig.getUpstreamDepth());
+    assertEquals(4, updatedLineageConfig.getDownstreamDepth());
+  }
+
   @Test
   void globalProfilerConfig(TestInfo test) throws HttpResponseException {
     // Create a profiler config

openmetadata-spec/src/main/resources/json/schema/configuration/limitsConfiguration.json

@@ -1,7 +1,7 @@
 {
   "$id": "https://open-metadata.org/schema/entity/configuration/limitsConfiguration.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "FernetConfiguration",
+  "title": "LimitsConfiguration",
   "description": "This schema defines the Limits Configuration.",
   "type": "object",
   "javaType": "org.openmetadata.schema.configuration.LimitsConfiguration",

openmetadata-spec/src/main/resources/json/schema/configuration/lineageSettings.json

@@ -0,0 +1,43 @@
+{
+  "$id": "https://open-metadata.org/schema/entity/configuration/lineageSettings.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "LineageSettings",
+  "description": "This schema defines the Lineage Settings.",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.api.lineage.LineageSettings",
+  "definitions": {
+    "lineageLayer": {
+      "javaType": "org.openmetadata.schema.api.lineage.LineageLayer",
+      "description": "Lineage Layers",
+      "type": "string",
+      "enum": [
+        "EntityLineage",
+        "ColumnLevelLineage",
+        "DataObservability"
+      ],
+      "default": "EntityLineage"
+    }
+  },
+  "properties": {
+    "upstreamDepth": {
+      "description": "Upstream Depth for Lineage.",
+      "type": "integer",
+      "default": 2,
+      "minimum": 1,
+      "maximum": 5
+    },
+    "downstreamDepth": {
+      "description": "DownStream Depth for Lineage.",
+      "type": "integer",
+      "default": 2,
+      "minimum": 1,
+      "maximum": 5
+    },
+    "lineageLayer": {
+      "description": "Lineage Layer.",
+      "$ref": "#/definitions/lineageLayer"
+    }
+  },
+  "required": ["upstreamDepth", "downstreamDepth", "lineageLayer"],
+  "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/configuration/searchSettings.json

@@ -4,7 +4,7 @@
   "title": "SearchSettings",
   "description": "This schema defines the Rbac Search Configuration.",
   "type": "object",
-  "javaType": "org.openmetadata.schema.api.searcg.SearchSettings",
+  "javaType": "org.openmetadata.schema.api.search.SearchSettings",
   "properties": {
     "enableAccessControl": {
       "type": "boolean",

openmetadata-spec/src/main/resources/json/schema/settings/settings.json

@@ -31,7 +31,8 @@
         "slackState",
         "profilerConfiguration",
         "searchSettings",
-        "assetCertificationSettings"
+        "assetCertificationSettings",
+        "lineageSettings"
       ]
     }
   },
@@ -80,6 +81,9 @@
         },
         {
           "$ref": "../configuration/assetCertificationSettings.json"
+        },
+        {
+          "$ref": "../configuration/lineageSettings.json"
         }
       ]
     }