diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java index 31f0b7a3c510..39153d16c591 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java @@ -23,7 +23,6 @@ import javax.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.api.configuration.dataQuality.DataQualityConfiguration; import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration; import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; @@ -114,9 +113,6 @@ public PipelineServiceClientConfiguration getPipelineServiceClientConfiguration( @JsonProperty("dataQualityConfiguration") private DataQualityConfiguration dataQualityConfiguration; - @JsonProperty("applications") - private AppsPrivateConfiguration appsPrivateConfiguration; - @JsonProperty("limits") private LimitsConfiguration limitsConfiguration; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 80643df4a194..9a8c321207de 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -1,17 +1,16 @@ package org.openmetadata.service.apps; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_JOB_GROUP; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Collection; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; -import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration; import org.openmetadata.schema.entity.app.App; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.apps.scheduler.AppScheduler; @@ -33,12 +32,11 @@ public class ApplicationHandler { @Getter private static ApplicationHandler instance; private final OpenMetadataApplicationConfig config; - private final AppsPrivateConfiguration privateConfiguration; private final AppRepository appRepository; + private final ConfigurationReader configReader = new ConfigurationReader(); private ApplicationHandler(OpenMetadataApplicationConfig config) { this.config = config; - this.privateConfiguration = config.getAppsPrivateConfiguration(); this.appRepository = new AppRepository(); } @@ -55,28 +53,23 @@ public static void initialize(OpenMetadataApplicationConfig config) { public void setAppRuntimeProperties(App app) { app.setOpenMetadataServerConnection( new OpenMetadataConnectionBuilder(config, app.getBot().getName()).build()); - - if (privateConfiguration != null - && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (app.getName().equals(appPrivateConfig.getName())) { - app.setPreview(appPrivateConfig.getPreview()); - app.setPrivateConfiguration(appPrivateConfig.getParameters()); - } - } + try { + AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(app.getName()); + app.setPreview(appPrivateConfig.getPreview()); + app.setPrivateConfiguration(appPrivateConfig.getParameters()); + } catch (IOException e) { + LOG.debug("Config file for app {} not found: ", app.getName(), e); } } public Boolean isPreview(String appName) { - if (privateConfiguration != null - && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) { - for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) { - if (appName.equals(appPrivateConfig.getName())) { - return appPrivateConfig.getPreview(); - } - } + try { + AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(appName); + return appPrivateConfig.getPreview(); + } catch (IOException e) { + LOG.debug("Config file for app {} not found: ", appName, e); + return false; } - return false; } public void triggerApplicationOnDemand( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java new file mode 100644 index 000000000000..ee54c7339de9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java @@ -0,0 +1,74 @@ +package org.openmetadata.service.apps; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import joptsimple.internal.Strings; +import org.apache.commons.text.StringSubstitutor; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; +import org.openmetadata.service.util.JsonUtils; + +public class ConfigurationReader { + + private final Map envMap; + + public ConfigurationReader(Map envMap) { + this.envMap = envMap; + } + + public ConfigurationReader() { + this.envMap = System.getenv(); + } + + public AppPrivateConfig readConfigFromResource(String appName) throws IOException { + String configFilePath = "applications/" + appName + "/config.yaml"; + try (InputStream inputStream = + ConfigurationReader.class.getClassLoader().getResourceAsStream(configFilePath)) { + if (inputStream == null) { + throw new IOException("Configuration file not found: " + configFilePath); + } + return JsonUtils.convertValue(readConfigFile(inputStream), AppPrivateConfig.class); + } + } + + public Map readConfigFile(InputStream configStream) throws IOException { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + Map config = mapper.readValue(configStream, Map.class); + resolveEnvVariablesInMap(config); + return mapper.convertValue(config, Map.class); + } + + private void resolveEnvVariablesInMap(Map map) { + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() instanceof String) { + map.put(entry.getKey(), resolveEnvVariables((String) entry.getValue(), envMap)); + } else if (entry.getValue() instanceof Map) { + resolveEnvVariablesInMap((Map) entry.getValue()); + } else if (entry.getValue() instanceof List) { + resolveEnvVariablesInList((List) entry.getValue()); + } + } + } + + private void resolveEnvVariablesInList(List list) { + for (int i = 0; i < list.size(); i++) { + Object element = list.get(i); + if (element instanceof String) { + list.set(i, resolveEnvVariables((String) element, envMap)); + } else if (element instanceof Map) { + resolveEnvVariablesInMap((Map) element); + } else if (element instanceof List) { + resolveEnvVariablesInList((List) element); + } + } + } + + public static String resolveEnvVariables(String value, Map envMap) { + StringSubstitutor substitutor = new StringSubstitutor(envMap); + String resolved = substitutor.replace(value); + return resolved.equals("\"\"") ? Strings.EMPTY : resolved; + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java new file mode 100644 index 000000000000..157507a86516 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java @@ -0,0 +1,39 @@ +package org.openmetadata.service.resources.apps; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; +import org.openmetadata.service.apps.ConfigurationReader; + +public class ConfigurationReaderTest { + + @Test + public void testReadConfigFile() throws IOException { + ConfigurationReader reader = + new ConfigurationReader( + Map.of( + "ENV_VAR", + "resolvedValue", + "NESTED_ENV_VAR", + "nestedValue", + "LIST_ENV_VAR", + "value1")); + AppPrivateConfig appConfig = reader.readConfigFromResource("TestApplication"); + assertNotNull(appConfig); + assertEquals("value1", appConfig.getParameters().getAdditionalProperties().get("key1")); + assertEquals("resolvedValue", appConfig.getParameters().getAdditionalProperties().get("key2")); + assertEquals("", appConfig.getParameters().getAdditionalProperties().get("emptyKey")); + assertEquals("default", appConfig.getParameters().getAdditionalProperties().get("defaultKey")); + Map nested = + (Map) appConfig.getParameters().getAdditionalProperties().get("nested"); + assertEquals("nestedValue", nested.get("nestedKey")); + List list = + (List) appConfig.getParameters().getAdditionalProperties().get("list"); + assertEquals("value1", list.get(1)); + } +} diff --git a/openmetadata-service/src/test/resources/applications/TestApplication/config.yaml b/openmetadata-service/src/test/resources/applications/TestApplication/config.yaml new file mode 100644 index 000000000000..933ef8649887 --- /dev/null +++ b/openmetadata-service/src/test/resources/applications/TestApplication/config.yaml @@ -0,0 +1,10 @@ +parameters: + key1: value1 + key2: ${ENV_VAR} + emptyKey: ${UNDEFINED_ENV_VAR:-""} + defaultKey: ${UNDEFINED_ENV_VAR:-default} + nested: + nestedKey: ${NESTED_ENV_VAR} + list: + - elem1 + - ${LIST_ENV_VAR} \ No newline at end of file