From 350e0f70cbb92f9ddff13a90ed6efce815996cd7 Mon Sep 17 00:00:00 2001 From: Imri Paran Date: Tue, 24 Sep 2024 16:21:50 +0200 Subject: [PATCH] MNIOR: feat(apps): support config file (#17872) * feat(apps): support config file - added support for app config files - removed AppPrivateConfig from the OpenMetadata server configuration * use dorpwizard utility classes for resolving environment variables in the config * moved fields to class level * format --- .../OpenMetadataApplicationConfig.java | 4 -- .../service/apps/ApplicationHandler.java | 41 +++++++------ .../service/apps/ConfigurationReader.java | 57 +++++++++++++++++++ .../apps/ConfigurationReaderTest.java | 57 +++++++++++++++++++ .../applications/InvalidConfig/config.yaml | 3 + .../applications/TestApplication/config.yaml | 10 ++++ 6 files changed, 147 insertions(+), 25 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java create mode 100644 openmetadata-service/src/test/resources/applications/InvalidConfig/config.yaml create mode 100644 openmetadata-service/src/test/resources/applications/TestApplication/config.yaml diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java index 31f0b7a3c510..39153d16c591 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java @@ -23,7 +23,6 @@ import javax.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.configuration.dataQuality.DataQualityConfiguration; import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration; import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; @@ -114,9 +113,6 @@ public PipelineServiceClientConfiguration getPipelineServiceClientConfiguration( @JsonProperty("dataQualityConfiguration") private DataQualityConfiguration dataQualityConfiguration; - @JsonProperty("applications") - private AppsPrivateConfiguration appsPrivateConfiguration; - @JsonProperty("limits") private LimitsConfiguration limitsConfiguration; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 80643df4a194..1d42626762d4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -1,17 +1,17 @@ package org.openmetadata.service.apps; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_JOB_GROUP; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; +import io.dropwizard.configuration.ConfigurationException; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Collection; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.apps.scheduler.AppScheduler; @@ -33,12 +33,11 @@ public class ApplicationHandler { @Getter private static ApplicationHandler instance; private final OpenMetadataApplicationConfig config; - private final AppsPrivateConfiguration privateConfiguration; private final AppRepository appRepository; + private final ConfigurationReader configReader = new ConfigurationReader(); private ApplicationHandler(OpenMetadataApplicationConfig config) { this.config = config; - this.privateConfiguration = config.getAppsPrivateConfiguration(); this.appRepository = new AppRepository(); } @@ -55,28 +54,28 @@ public static void initialize(OpenMetadataApplicationConfig config) { public void setAppRuntimeProperties(App app) { app.setOpenMetadataServerConnection( new OpenMetadataConnectionBuilder(config, app.getBot().getName()).build()); - - if (privateConfiguration != null - && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (app.getName().equals(appPrivateConfig.getName())) { - app.setPreview(appPrivateConfig.getPreview()); - app.setPrivateConfiguration(appPrivateConfig.getParameters()); - } - } + try { + AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(app.getName()); + app.setPreview(appPrivateConfig.getPreview()); + app.setPrivateConfiguration(appPrivateConfig.getParameters()); + } catch (IOException e) { + LOG.debug("Config file for app {} not found: ", app.getName(), e); + } catch (ConfigurationException e) { + LOG.error("Error reading config file for app {}", app.getName(), e); } } public Boolean isPreview(String appName) { - if (privateConfiguration != null - && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (appName.equals(appPrivateConfig.getName())) { - return appPrivateConfig.getPreview(); - } - } + try { + AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(appName); + return appPrivateConfig.getPreview(); + } catch (IOException e) { + LOG.debug("Config file for app {} not found: ", appName, e); + return false; + } catch (ConfigurationException e) { + LOG.error("Error reading config file for app {}", appName, e); + return false; } - return false; } public void triggerApplicationOnDemand( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java new file mode 100644 index 000000000000..ba680d1a347f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java @@ -0,0 +1,57 @@ +package org.openmetadata.service.apps; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.dropwizard.configuration.ConfigurationException; +import io.dropwizard.configuration.EnvironmentVariableSubstitutor; +import io.dropwizard.configuration.FileConfigurationSourceProvider; +import io.dropwizard.configuration.SubstitutingSourceProvider; +import io.dropwizard.configuration.YamlConfigurationFactory; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import org.apache.commons.text.StringSubstitutor; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; +import org.openmetadata.service.util.JsonUtils; + +public class ConfigurationReader { + private final StringSubstitutor substitutor; + private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + private final YamlConfigurationFactory factory = + new YamlConfigurationFactory<>(Object.class, null, mapper, "dw"); + + public ConfigurationReader(Map envMap) { + // envMap is for custom environment variables (e.g., for testing), defaulting to the system + // environment. + substitutor = + envMap == null ? new EnvironmentVariableSubstitutor(false) : new StringSubstitutor(envMap); + } + + public ConfigurationReader() { + this(System.getenv()); + } + + public AppPrivateConfig readConfigFromResource(String appName) + throws IOException, ConfigurationException { + String configFilePath = "applications/" + appName + "/config.yaml"; + URL resource = ConfigurationReader.class.getClassLoader().getResource(configFilePath); + if (resource == null) { + throw new IOException("Configuration file not found: " + configFilePath); + } + File configFile = new File(resource.getFile()); + return JsonUtils.convertValue(readConfigFile(configFile), AppPrivateConfig.class); + } + + public Map readConfigFile(File configFile) + throws IOException, ConfigurationException { + try { + return (Map) + factory.build( + new SubstitutingSourceProvider(new FileConfigurationSourceProvider(), substitutor), + configFile.getAbsolutePath()); + } catch (ClassCastException e) { + throw new RuntimeException("Configuration file is not a valid YAML file", e); + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java new file mode 100644 index 000000000000..22ade489254b --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java @@ -0,0 +1,57 @@ +package org.openmetadata.service.resources.apps; + +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.dropwizard.configuration.ConfigurationException; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; +import org.openmetadata.service.apps.ConfigurationReader; + +public class ConfigurationReaderTest { + + @Test + public void testReadConfigFile() throws IOException, ConfigurationException { + ConfigurationReader reader = + new ConfigurationReader( + Map.of( + "ENV_VAR", + "resolvedValue", + "NESTED_ENV_VAR", + "nestedValue", + "LIST_ENV_VAR", + "value1")); + AppPrivateConfig appConfig = reader.readConfigFromResource("TestApplication"); + assertNotNull(appConfig); + assertEquals("value1", appConfig.getParameters().getAdditionalProperties().get("key1")); + assertEquals("resolvedValue", appConfig.getParameters().getAdditionalProperties().get("key2")); + assertEquals("", appConfig.getParameters().getAdditionalProperties().get("emptyKey")); + assertEquals("default", appConfig.getParameters().getAdditionalProperties().get("defaultKey")); + Map nested = + (Map) appConfig.getParameters().getAdditionalProperties().get("nested"); + assertEquals("nestedValue", nested.get("nestedKey")); + List list = + (List) appConfig.getParameters().getAdditionalProperties().get("list"); + assertEquals("value1", list.get(1)); + } + + @Test + public void testInvalidConfig() { + ConfigurationReader reader = new ConfigurationReader(); + assertThrows(RuntimeException.class, () -> reader.readConfigFromResource("InvalidConfig")); + } + + @Test + public void missingConfig() { + ConfigurationReader reader = new ConfigurationReader(); + assertThrows( + IOException.class, + () -> { + reader.readConfigFromResource("missing"); + }); + } +} diff --git a/openmetadata-service/src/test/resources/applications/InvalidConfig/config.yaml b/openmetadata-service/src/test/resources/applications/InvalidConfig/config.yaml new file mode 100644 index 000000000000..655347f45614 --- /dev/null +++ b/openmetadata-service/src/test/resources/applications/InvalidConfig/config.yaml @@ -0,0 +1,3 @@ +--- +- a +- b \ No newline at end of file diff --git a/openmetadata-service/src/test/resources/applications/TestApplication/config.yaml b/openmetadata-service/src/test/resources/applications/TestApplication/config.yaml new file mode 100644 index 000000000000..933ef8649887 --- /dev/null +++ b/openmetadata-service/src/test/resources/applications/TestApplication/config.yaml @@ -0,0 +1,10 @@ +parameters: + key1: value1 + key2: ${ENV_VAR} + emptyKey: ${UNDEFINED_ENV_VAR:-""} + defaultKey: ${UNDEFINED_ENV_VAR:-default} + nested: + nestedKey: ${NESTED_ENV_VAR} + list: + - elem1 + - ${LIST_ENV_VAR} \ No newline at end of file