Skip to content

Commit

Permalink
Merge pull request hpcc-systems#19299 from jakesmith/HPCC-32991-globa…
Browse files Browse the repository at this point in the history 
…l-yaml-defaults

HPCC-32991 Add ability to defined global config defaults

Reviewed-By: Jack Del Vecchio
Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>
Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>
  • Loading branch information
@ghalliday
ghalliday authored Nov 15, 2024
2 parents 143a000 + 7fff875 commit 42e8bcb
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 15 deletions.
2 changes: 1 addition & 1 deletion esp/platform/application_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ IPropertyTree *loadApplicationConfig(const char *application, const char* argv[]
appendPTreeFromYamlFile(defaultConfig, application_dir->query().queryFilename(), true);

//apply provided config to the application
Owned<IPropertyTree> config = loadConfiguration(defaultConfig, argv, "esp", "ESP", nullptr, nullptr);
Owned<IPropertyTree> config = loadConfiguration(defaultConfig, nullptr, argv, "esp", "ESP", nullptr, nullptr);

return config.getClear();
}
Expand Down
36 changes: 24 additions & 12 deletions system/jlib/jptree.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8696,13 +8696,13 @@ static void holdLoop()
}
#endif

static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfiguration(IPropertyTree *componentDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute);
static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfiguration(IPropertyTree *componentDefault, IPropertyTree *globalDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute);

class CConfigUpdater : public CInterface
{
StringAttr absoluteConfigFilename;
StringAttr configFilename;
Linked<IPropertyTree> componentDefault;
Linked<IPropertyTree> componentDefault, globalDefault;
StringArray args;
StringAttr componentTag, envPrefix, legacyFilename;
IPropertyTree * (*mapper)(IPropertyTree *);
Expand All @@ -8721,11 +8721,12 @@ class CConfigUpdater : public CInterface
{
return args.ordinality(); // NB: null terminated, so always >=1 if initialized
}
void init(const char *_absoluteConfigFilename, IPropertyTree *_componentDefault, const char * * argv, const char * _componentTag, const char * _envPrefix, const char *_legacyFilename, IPropertyTree * (_mapper)(IPropertyTree *), const char *_altNameAttribute)
void init(const char *_absoluteConfigFilename, IPropertyTree *_componentDefault, IPropertyTree *_globalDefault, const char * * argv, const char * _componentTag, const char * _envPrefix, const char *_legacyFilename, IPropertyTree * (_mapper)(IPropertyTree *), const char *_altNameAttribute)
{
dbgassertex(!isInitialized());
absoluteConfigFilename.set(_absoluteConfigFilename);
componentDefault.set(_componentDefault);
globalDefault.set(_globalDefault);
componentTag.set(_componentTag);
envPrefix.set(_envPrefix);
legacyFilename.set(_legacyFilename);
Expand Down Expand Up @@ -8761,7 +8762,7 @@ class CConfigUpdater : public CInterface
#endif
if (changed)
{
auto result = doLoadConfiguration(componentDefault, args.getArray(), componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);
auto result = doLoadConfiguration(componentDefault, globalDefault, args.getArray(), componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);

// NB: block calls to get*Config*() until callbacks notified and new swapped in
CriticalBlock b(configCS);
Expand Down Expand Up @@ -8913,10 +8914,10 @@ void CConfigUpdateHook::installOnce(ConfigUpdateFunc callbackFunc, bool callWhen
}


static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfiguration(IPropertyTree *componentDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute)
static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfiguration(IPropertyTree *componentDefault, IPropertyTree *globalDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute)
{
Owned<IPropertyTree> newComponentConfig = createPTreeFromIPT(componentDefault);
Owned<IPropertyTree> newGlobalConfig;
Linked<IPropertyTree> newGlobalConfig = globalDefault ? createPTreeFromIPT(globalDefault) : nullptr;
StringBuffer absConfigFilename;
const char * optConfig = nullptr;
bool outputConfig = false;
Expand Down Expand Up @@ -8979,6 +8980,7 @@ static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfigura
}
absConfigFilename.append(config);

Owned<IPropertyTree> configGlobal;
Owned<IPropertyTree> delta;
if (optConfig)
{
Expand All @@ -8989,21 +8991,29 @@ static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfigura
if (!isEmptyString(optConfig))
{
delta.setown(loadConfiguration(absConfigFilename, componentTag, true, altNameAttribute));
newGlobalConfig.setown(loadConfiguration(absConfigFilename, "global", false, altNameAttribute));
configGlobal.setown(loadConfiguration(absConfigFilename, "global", false, altNameAttribute));
}
}
else
{
if (legacyFilename && checkFileExists(legacyFilename))
{
delta.setown(createPTreeFromXMLFile(legacyFilename, ipt_caseInsensitive));
newGlobalConfig.set(delta->queryPropTree("global"));
configGlobal.set(delta->queryPropTree("global"));
}

if (delta && mapper)
delta.setown(mapper(delta));
}

if (configGlobal)
{
if (newGlobalConfig)
mergeConfiguration(*newGlobalConfig, *configGlobal);
else
newGlobalConfig.setown(configGlobal.getClear());
}

if (delta)
mergeConfiguration(*newComponentConfig, *delta, altNameAttribute);

Expand Down Expand Up @@ -9046,13 +9056,13 @@ static std::tuple<std::string, IPropertyTree *, IPropertyTree *> doLoadConfigura
return std::make_tuple(std::string(absConfigFilename.str()), newComponentConfig.getClear(), newGlobalConfig.getClear());
}

jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute, bool monitor)
jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, IPropertyTree *globalDefault, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute, bool monitor)
{
assertex(configFileUpdater); // NB: loadConfiguration should always be called after configFileUpdater is initialized
if (configFileUpdater->isInitialized())
throw makeStringExceptionV(99, "Configuration for component %s has already been initialised", componentTag);

auto result = doLoadConfiguration(componentDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);
auto result = doLoadConfiguration(componentDefault, globalDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);

componentConfiguration.setown(std::get<1>(result));
globalConfiguration.setown(std::get<2>(result));
Expand All @@ -9072,7 +9082,7 @@ jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, con
* installed config hooks to be called when an environment change is detected e.g when pushed to Dali)
*/

configFileUpdater->init(std::get<0>(result).c_str(), componentDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);
configFileUpdater->init(std::get<0>(result).c_str(), componentDefault, globalDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute);
if (monitor)
configFileUpdater->startMonitoring();

Expand All @@ -9086,17 +9096,19 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
throw makeStringExceptionV(99, "Configuration for component %s has already been initialised", componentTag);

Owned<IPropertyTree> componentDefault;
Owned<IPropertyTree> defaultGlobalConfig;
if (defaultYaml)
{
Owned<IPropertyTree> defaultConfig = createPTreeFromYAML(defaultYaml);
componentDefault.set(defaultConfig->queryPropTree(componentTag));
if (!componentDefault)
throw makeStringExceptionV(99, "Default configuration does not contain the tag %s", componentTag);
defaultGlobalConfig.set(defaultConfig->queryPropTree("global"));
}
else
componentDefault.setown(createPTree(componentTag));

return loadConfiguration(componentDefault, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute, monitor);
return loadConfiguration(componentDefault, defaultGlobalConfig, argv, componentTag, envPrefix, legacyFilename, mapper, altNameAttribute, monitor);
}

void replaceComponentConfig(IPropertyTree *newComponentConfig, IPropertyTree *newGlobalConfig)
Expand Down
2 changes: 1 addition & 1 deletion system/jlib/jptree.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@ inline static bool isValidXPathChr(char c)
jlib_decl void mergeConfiguration(IPropertyTree & target, const IPropertyTree & source, const char *altNameAttribute=nullptr, bool overwriteAttr=true);

jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const std::string> ignoreOptions = {});
jlib_decl IPropertyTree * loadConfiguration(IPropertyTree * defaultConfig, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute=nullptr, bool monitor=true);
jlib_decl IPropertyTree * loadConfiguration(IPropertyTree * defaultConfig, IPropertyTree * globalConfig, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute=nullptr, bool monitor=true);
jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute=nullptr, bool monitor=true);
jlib_decl void replaceComponentConfig(IPropertyTree *newComponentConfig, IPropertyTree *newGlobalConfig);
jlib_decl void initNullConfiguration();
Expand Down
2 changes: 1 addition & 1 deletion thorlcr/slave/thslavemain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ int main( int argc, const char *argv[] )
#ifdef _CONTAINERIZED
globals.setown(loadConfiguration(thorDefaultConfigYaml, argv, "thor", "THOR", nullptr, nullptr, nullptr, false));
#else
globals.setown(loadConfiguration(globals, argv, "thor", "THOR", nullptr, nullptr, nullptr, false));
globals.setown(loadConfiguration(globals, nullptr, argv, "thor", "THOR", nullptr, nullptr, nullptr, false));
#endif

// NB: the thor configuration is serialized from the manager and only available after RegisterSelf
Expand Down

0 comments on commit 42e8bcb

Please sign in to comment.