MNIOR: feat(apps): support config file

openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplicationConfig.java

@@ -23,7 +23,6 @@
 import javax.validation.constraints.NotNull;
 import lombok.Getter;
 import lombok.Setter;
-import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration;
 import org.openmetadata.schema.api.configuration.dataQuality.DataQualityConfiguration;
 import org.openmetadata.schema.api.configuration.events.EventHandlerConfiguration;
 import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration;
@@ -114,9 +113,6 @@ public PipelineServiceClientConfiguration getPipelineServiceClientConfiguration(
  @JsonProperty("dataQualityConfiguration")
  private DataQualityConfiguration dataQualityConfiguration;
 
- @JsonProperty("applications")
- private AppsPrivateConfiguration appsPrivateConfiguration;
-
  @JsonProperty("limits")
  private LimitsConfiguration limitsConfiguration;
 

openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java

@@ -1,17 +1,17 @@
 package org.openmetadata.service.apps;
 
-import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_JOB_GROUP;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME;
 
+import io.dropwizard.configuration.ConfigurationException;
+import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Collection;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
-import org.openmetadata.schema.api.configuration.apps.AppsPrivateConfiguration;
 import org.openmetadata.schema.entity.app.App;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.apps.scheduler.AppScheduler;
@@ -33,12 +33,11 @@ public class ApplicationHandler {
 
  @Getter private static ApplicationHandler instance;
  private final OpenMetadataApplicationConfig config;
- private final AppsPrivateConfiguration privateConfiguration;
  private final AppRepository appRepository;
+ private final ConfigurationReader configReader = new ConfigurationReader();
 
  private ApplicationHandler(OpenMetadataApplicationConfig config) {
  this.config = config;
- this.privateConfiguration = config.getAppsPrivateConfiguration();
  this.appRepository = new AppRepository();
  }
 
@@ -55,28 +54,28 @@ public static void initialize(OpenMetadataApplicationConfig config) {
  public void setAppRuntimeProperties(App app) {
  app.setOpenMetadataServerConnection(
  new OpenMetadataConnectionBuilder(config, app.getBot().getName()).build());
-
- if (privateConfiguration != null
- && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) {
- for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) {
- if (app.getName().equals(appPrivateConfig.getName())) {
- app.setPreview(appPrivateConfig.getPreview());
- app.setPrivateConfiguration(appPrivateConfig.getParameters());
- }
- }
+ try {
+ AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(app.getName());
+ app.setPreview(appPrivateConfig.getPreview());
+ app.setPrivateConfiguration(appPrivateConfig.getParameters());
+ } catch (IOException e) {
+ LOG.debug("Config file for app {} not found: ", app.getName(), e);
+ } catch (ConfigurationException e) {
+ LOG.error("Error reading config file for app {}", app.getName(), e);
  }
  }
 
  public Boolean isPreview(String appName) {
- if (privateConfiguration != null
- && !nullOrEmpty(privateConfiguration.getAppsPrivateConfiguration())) {
- for (AppPrivateConfig appPrivateConfig : privateConfiguration.getAppsPrivateConfiguration()) {
- if (appName.equals(appPrivateConfig.getName())) {
- return appPrivateConfig.getPreview();
- }
- }
+ try {
+ AppPrivateConfig appPrivateConfig = configReader.readConfigFromResource(appName);
+ return appPrivateConfig.getPreview();
+ } catch (IOException e) {
+ LOG.debug("Config file for app {} not found: ", appName, e);
+ return false;
+ } catch (ConfigurationException e) {
+ LOG.error("Error reading config file for app {}", appName, e);
+ return false;
  }
- return false;
  }
 
  public void triggerApplicationOnDemand(

openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java

@@ -0,0 +1,57 @@
+package org.openmetadata.service.apps;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import io.dropwizard.configuration.ConfigurationException;
+import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
+import io.dropwizard.configuration.FileConfigurationSourceProvider;
+import io.dropwizard.configuration.SubstitutingSourceProvider;
+import io.dropwizard.configuration.YamlConfigurationFactory;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+import org.apache.commons.text.StringSubstitutor;
+import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
+import org.openmetadata.service.util.JsonUtils;
+
+public class ConfigurationReader {
+ private final StringSubstitutor substitutor;
+ private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ private final YamlConfigurationFactory<Object> factory =
+ new YamlConfigurationFactory<>(Object.class, null, mapper, "dw");
+
+ public ConfigurationReader(Map<String, String> envMap) {
+ // envMap is for custom environment variables (e.g., for testing), defaulting to the system
+ // environment.
+ substitutor =
+ envMap == null ? new EnvironmentVariableSubstitutor(false) : new StringSubstitutor(envMap);
+ }
+
+ public ConfigurationReader() {
+ this(System.getenv());
+ }
+
+ public AppPrivateConfig readConfigFromResource(String appName)
+ throws IOException, ConfigurationException {
+ String configFilePath = "applications/" + appName + "/config.yaml";
+ URL resource = ConfigurationReader.class.getClassLoader().getResource(configFilePath);
+ if (resource == null) {
+ throw new IOException("Configuration file not found: " + configFilePath);
+ }
+ File configFile = new File(resource.getFile());
+ return JsonUtils.convertValue(readConfigFile(configFile), AppPrivateConfig.class);
+ }
+
+ public Map<String, Object> readConfigFile(File configFile)
+ throws IOException, ConfigurationException {
+ try {
+ return (Map<String, Object>)
+ factory.build(
+ new SubstitutingSourceProvider(new FileConfigurationSourceProvider(), substitutor),
+ configFile.getAbsolutePath());
+ } catch (ClassCastException e) {
+ throw new RuntimeException("Configuration file is not a valid YAML file", e);
+ }
+ }
+}

openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/ConfigurationReaderTest.java

@@ -0,0 +1,57 @@
+package org.openmetadata.service.resources.apps;
+
+import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.dropwizard.configuration.ConfigurationException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
+import org.openmetadata.service.apps.ConfigurationReader;
+
+public class ConfigurationReaderTest {
+
+ @Test
+ public void testReadConfigFile() throws IOException, ConfigurationException {
+ ConfigurationReader reader =
+ new ConfigurationReader(
+ Map.of(
+ "ENV_VAR",
+ "resolvedValue",
+ "NESTED_ENV_VAR",
+ "nestedValue",
+ "LIST_ENV_VAR",
+ "value1"));
+ AppPrivateConfig appConfig = reader.readConfigFromResource("TestApplication");
+ assertNotNull(appConfig);
+ assertEquals("value1", appConfig.getParameters().getAdditionalProperties().get("key1"));
+ assertEquals("resolvedValue", appConfig.getParameters().getAdditionalProperties().get("key2"));
+ assertEquals("", appConfig.getParameters().getAdditionalProperties().get("emptyKey"));
+ assertEquals("default", appConfig.getParameters().getAdditionalProperties().get("defaultKey"));
+ Map<String, String> nested =
+ (Map<String, String>) appConfig.getParameters().getAdditionalProperties().get("nested");
+ assertEquals("nestedValue", nested.get("nestedKey"));
+ List<String> list =
+ (List<String>) appConfig.getParameters().getAdditionalProperties().get("list");
+ assertEquals("value1", list.get(1));
+ }
+
+ @Test
+ public void testInvalidConfig() {
+ ConfigurationReader reader = new ConfigurationReader();
+ assertThrows(RuntimeException.class, () -> reader.readConfigFromResource("InvalidConfig"));
+ }
+
+ @Test
+ public void missingConfig() {
+ ConfigurationReader reader = new ConfigurationReader();
+ assertThrows(
+ IOException.class,
+ () -> {
+ reader.readConfigFromResource("missing");
+ });
+ }
+}

openmetadata-service/src/test/resources/applications/InvalidConfig/config.yaml

@@ -0,0 +1,3 @@
+---
+- a
+- b

openmetadata-service/src/test/resources/applications/TestApplication/config.yaml

@@ -0,0 +1,10 @@
+parameters:
+ key1: value1
+ key2: ${ENV_VAR}
+ emptyKey: ${UNDEFINED_ENV_VAR:-""}
+ defaultKey: ${UNDEFINED_ENV_VAR:-default}
+ nested:
+ nestedKey: ${NESTED_ENV_VAR}
+ list:
+ - elem1
+ - ${LIST_ENV_VAR}