From cb82acc94afab0502edbb68acc7130116ab48d9c Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 24 May 2024 18:42:04 +0100 Subject: [PATCH 1/7] Address issue #113 by importing componenets as well as runs --- cosmosis/campaign.py | 14 ++++++++++---- cosmosis/test/campaign.yml | 22 +++++++++++++++++++++- cosmosis/test/test_campaign.py | 26 ++++++++++++++++++++------ 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/cosmosis/campaign.py b/cosmosis/campaign.py index b549baad..bfc90171 100644 --- a/cosmosis/campaign.py +++ b/cosmosis/campaign.py @@ -575,6 +575,9 @@ def parse_yaml_run_file(run_config): ------- runs : dict A dictionary of runs, keyed by name + + components : dict + A dictionary of components, keyed by name """ if isinstance(run_config, dict): info = run_config @@ -592,8 +595,11 @@ def parse_yaml_run_file(run_config): # Can include another run file, which we deal with # recursively. runs = {} + components = {} for include_file in include: - runs.update(parse_yaml_run_file(include_file)) + inc_runs, inc_comps = parse_yaml_run_file(include_file) + components.update(inc_comps) + runs.update(inc_runs) # But we override the output directory # of any imported runs with the one we have here @@ -601,7 +607,7 @@ def parse_yaml_run_file(run_config): set_output_dir(run["params"], name, output_dir, output_name) # deal with re-usable components - components = info.get("components", {}) + components.update(info.get("components", {})) submission_info = info.get("submission", {}) @@ -614,7 +620,7 @@ def parse_yaml_run_file(run_config): # a chance to override the environment variables of their parents. expand_environment_variables(runs) - return runs + return runs, components def show_run(run): @@ -799,7 +805,7 @@ def submit_run(run_file, run): def main(args): - runs = parse_yaml_run_file(args.run_config) + runs, _ = parse_yaml_run_file(args.run_config) if args.mpi and not args.run: raise ValueError("MPI can only be used when running a single run") diff --git a/cosmosis/test/campaign.yml b/cosmosis/test/campaign.yml index 4643cb00..9369af46 100644 --- a/cosmosis/test/campaign.yml +++ b/cosmosis/test/campaign.yml @@ -1,6 +1,11 @@ output_dir: output/campaign-test output_name: my_project_{name}_suite1 -include: [] +include: [cosmosis/test/included.yml] + +components: + test_component_1: + params: + - emcee.walkers = 100 submission: # These parameters can be overridden per run. @@ -65,3 +70,18 @@ runs: parent: env-test-1 env: TEST : yyy + + - name: component-test + parent: v1 + components: + - test_component_1 + + - name: include-test-1 + parent: imported-run + params: + - emcee.walkers = 755 + + - name: include-test-2 + parent: v1 + components: + - test_component_2 diff --git a/cosmosis/test/test_campaign.py b/cosmosis/test/test_campaign.py index 0bcd7deb..6a721857 100644 --- a/cosmosis/test/test_campaign.py +++ b/cosmosis/test/test_campaign.py @@ -70,9 +70,9 @@ def test_pipeline_prepend(): def test_campaign_functions(): with run_from_source_dir(): - runs = parse_yaml_run_file("cosmosis/test/campaign.yml") + runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml") - assert len(runs) == 6 + assert len(runs) == 10 assert "v1" in runs assert runs["v2"]["values"].get("parameters", "p1") == "-2.0 0.0 2.0" assert runs["v2"]["priors"].get("parameters", "p2") == "gaussian 0.0 1.0" @@ -108,13 +108,13 @@ def test_campaign_functions2(): with tempfile.TemporaryDirectory() as dirname: runs_config['output_dir'] = dirname - runs = parse_yaml_run_file(runs_config) + runs, _ = parse_yaml_run_file(runs_config) for name in runs: print(name) - assert len(runs) == 6 + assert len(runs) == 10 assert "v1" in runs assert runs["v2"]["values"].get("parameters", "p1") == "-2.0 0.0 2.0" assert runs["v2"]["priors"].get("parameters", "p2") == "gaussian 0.0 1.0" @@ -142,11 +142,25 @@ def test_campaign_functions2(): def test_campaign_env(): os.environ["TEST"] = "aaa" with run_from_source_dir(): - runs = parse_yaml_run_file("cosmosis/test/campaign.yml") + runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml") assert runs["env-test-1"]["params"].get("test1", "env_test_var") == "xxx" assert runs["env-test-2"]["params"].get("test1", "env_test_var") == "yyy" def test_campaign_duplicate_keys(): with pytest.raises(ValueError): with run_from_source_dir(): - runs = parse_yaml_run_file("cosmosis/test/bad-campaign.yml") + runs, _ = parse_yaml_run_file("cosmosis/test/bad-campaign.yml") + +def test_component(): + with run_from_source_dir(): + runs, components = parse_yaml_run_file("cosmosis/test/campaign.yml") + assert "test_component_1" in components + assert runs["component-test"]["params"].get("emcee", "walkers") == "100" + +def test_include(): + runs, components = parse_yaml_run_file("cosmosis/test/campaign.yml") + assert "imported-run" in runs + assert "include-test-1" in runs + assert runs['include-test-1']['params'].get("emcee", "walkers") == "755" + assert "include-test-2" in runs + assert runs['include-test-2']['params'].get("emcee", "walkers") == "200" From b2296e368cef5f1987e0d11b53da0ca4ec3ce721 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Fri, 24 May 2024 19:26:57 +0100 Subject: [PATCH 2/7] Address issue #123 by ensuring that env vars are inherited and correctly overwritten --- cosmosis/campaign.py | 14 +++++++++++--- cosmosis/test/campaign.yml | 15 +++++++++++++++ cosmosis/test/example.ini | 1 + cosmosis/test/test_campaign.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/cosmosis/campaign.py b/cosmosis/campaign.py index bfc90171..ad5c0853 100644 --- a/cosmosis/campaign.py +++ b/cosmosis/campaign.py @@ -410,9 +410,6 @@ def build_run(name, run_info, runs, components, output_dir, submission_info, out ------- """ run = run_info.copy() - env_vars = run_info.get("env", {}) - run["env"] = env_vars - # We want to delay expanding environment variables so that child runs # have a chance to override them. So we set no_expand_vars=True on all of these @@ -428,6 +425,17 @@ def build_run(name, run_info, runs, components, output_dir, submission_info, out else: warnings.warn(f"Run {name} specifies neither 'parent' nor 'base' so is invalid") return None + + # Build environment variables + # These are inherited from the parent run, if there is one, + # and then updated with any specific to this run, which can overwrite. + # env vars are only applied right at the end when all runs are collected + if "parent" in run_info: + env_vars = parent["env"].copy() + else: + env_vars = {} + env_vars.update(run_info.get("env", {})) + run["env"] = env_vars # Build values file, which is mandatory if "parent" in run_info: diff --git a/cosmosis/test/campaign.yml b/cosmosis/test/campaign.yml index 9369af46..4809a390 100644 --- a/cosmosis/test/campaign.yml +++ b/cosmosis/test/campaign.yml @@ -71,6 +71,20 @@ runs: env: TEST : yyy + - name: env-test-3 + parent: env-test-1 + params: + - emcee.walkers = ${TEST} + + - name: env-test-4 + parent: env-test-2 + env: + TEST2 : zzz + + # Test double inheritance + - name: env-test-5 + parent: env-test-4 + - name: component-test parent: v1 components: @@ -85,3 +99,4 @@ runs: parent: v1 components: - test_component_2 + diff --git a/cosmosis/test/example.ini b/cosmosis/test/example.ini index c86c8b40..1ec4dd81 100644 --- a/cosmosis/test/example.ini +++ b/cosmosis/test/example.ini @@ -12,3 +12,4 @@ priors = cosmosis/test/example-priors.ini [test1] file = cosmosis/test/example_module.py env_test_var=${TEST} +env_test_var2=${TEST2} \ No newline at end of file diff --git a/cosmosis/test/test_campaign.py b/cosmosis/test/test_campaign.py index 6a721857..b1eb2339 100644 --- a/cosmosis/test/test_campaign.py +++ b/cosmosis/test/test_campaign.py @@ -5,6 +5,8 @@ import contextlib import pytest +NRUN = 13 + @contextlib.contextmanager def run_from_source_dir(): this_dir = os.path.split(os.path.abspath(__file__))[0] @@ -72,7 +74,7 @@ def test_campaign_functions(): with run_from_source_dir(): runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml") - assert len(runs) == 10 + assert len(runs) == NRUN assert "v1" in runs assert runs["v2"]["values"].get("parameters", "p1") == "-2.0 0.0 2.0" assert runs["v2"]["priors"].get("parameters", "p2") == "gaussian 0.0 1.0" @@ -114,7 +116,7 @@ def test_campaign_functions2(): print(name) - assert len(runs) == 10 + assert len(runs) == NRUN assert "v1" in runs assert runs["v2"]["values"].get("parameters", "p1") == "-2.0 0.0 2.0" assert runs["v2"]["priors"].get("parameters", "p2") == "gaussian 0.0 1.0" @@ -146,6 +148,15 @@ def test_campaign_env(): assert runs["env-test-1"]["params"].get("test1", "env_test_var") == "xxx" assert runs["env-test-2"]["params"].get("test1", "env_test_var") == "yyy" +def test_inherit_env(): + with run_from_source_dir(): + runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml") + assert runs["env-test-3"]["params"].get("test1", "env_test_var") == "xxx" + assert runs["env-test-3"]["params"].get("emcee", "walkers") == "xxx" + assert runs["env-test-5"]["params"].get("test1", "env_test_var") == "yyy" + assert runs["env-test-5"]["params"].get("test1", "env_test_var2") == "zzz" + + def test_campaign_duplicate_keys(): with pytest.raises(ValueError): with run_from_source_dir(): From 9dc78feeab5ab011c8660fd6ad903bffc32de27d Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 May 2024 09:32:05 +0100 Subject: [PATCH 3/7] add missing test file --- cosmosis/test/included.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cosmosis/test/included.yml diff --git a/cosmosis/test/included.yml b/cosmosis/test/included.yml new file mode 100644 index 00000000..e4af4ae4 --- /dev/null +++ b/cosmosis/test/included.yml @@ -0,0 +1,10 @@ + +components: + test_component_2: + params: + - emcee.walkers = 200 + + +runs: + - name: imported-run + base: cosmosis/test/example.ini From b59782bb5695b07f75a0a1c999e19a062db98653 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 May 2024 09:38:55 +0100 Subject: [PATCH 4/7] run test from correct dir --- cosmosis/test/test_campaign.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cosmosis/test/test_campaign.py b/cosmosis/test/test_campaign.py index b1eb2339..c4caa231 100644 --- a/cosmosis/test/test_campaign.py +++ b/cosmosis/test/test_campaign.py @@ -165,13 +165,14 @@ def test_campaign_duplicate_keys(): def test_component(): with run_from_source_dir(): runs, components = parse_yaml_run_file("cosmosis/test/campaign.yml") - assert "test_component_1" in components - assert runs["component-test"]["params"].get("emcee", "walkers") == "100" + assert "test_component_1" in components + assert runs["component-test"]["params"].get("emcee", "walkers") == "100" def test_include(): + with run_from_source_dir(): runs, components = parse_yaml_run_file("cosmosis/test/campaign.yml") - assert "imported-run" in runs - assert "include-test-1" in runs - assert runs['include-test-1']['params'].get("emcee", "walkers") == "755" - assert "include-test-2" in runs - assert runs['include-test-2']['params'].get("emcee", "walkers") == "200" + assert "imported-run" in runs + assert "include-test-1" in runs + assert runs['include-test-1']['params'].get("emcee", "walkers") == "755" + assert "include-test-2" in runs + assert runs['include-test-2']['params'].get("emcee", "walkers") == "200" From fcfac5f58643b336399a67af50e806f63fcb2935 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 May 2024 09:58:21 +0100 Subject: [PATCH 5/7] move things inside correct dir setting --- cosmosis/test/test_campaign.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cosmosis/test/test_campaign.py b/cosmosis/test/test_campaign.py index c4caa231..f6ad3a81 100644 --- a/cosmosis/test/test_campaign.py +++ b/cosmosis/test/test_campaign.py @@ -108,9 +108,12 @@ def test_campaign_functions2(): with open("cosmosis/test/campaign.yml") as f: runs_config = load_yaml(f) - with tempfile.TemporaryDirectory() as dirname: - runs_config['output_dir'] = dirname - runs, _ = parse_yaml_run_file(runs_config) + with tempfile.TemporaryDirectory() as dirname: + runs_config['output_dir'] = dirname + + with run_from_source_dir(): + with open("cosmosis/test/campaign.yml") as f: + runs, _ = parse_yaml_run_file(runs_config) for name in runs: print(name) From d1a938b808b274d2c3610f8617809c3fc3b35fae Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 May 2024 10:08:18 +0100 Subject: [PATCH 6/7] include new file in setup installation --- cosmosis/test/test_campaign.py | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cosmosis/test/test_campaign.py b/cosmosis/test/test_campaign.py index f6ad3a81..dea8f8d5 100644 --- a/cosmosis/test/test_campaign.py +++ b/cosmosis/test/test_campaign.py @@ -173,7 +173,7 @@ def test_component(): def test_include(): with run_from_source_dir(): - runs, components = parse_yaml_run_file("cosmosis/test/campaign.yml") + runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml") assert "imported-run" in runs assert "include-test-1" in runs assert runs['include-test-1']['params'].get("emcee", "walkers") == "755" diff --git a/setup.py b/setup.py index 554f5957..f5943ccc 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ "test/libtest/test_c_datablock_scalars.template", "test/libtest/Makefile", "test/campaign.yml", + "test/included.yml", "test/bad-campaign.yml", "test/example-priors.ini", "test/example-values.ini", From d698709bddfaf4b61028d302cee1651be0c16c27 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 27 May 2024 10:34:00 +0100 Subject: [PATCH 7/7] version bump --- cosmosis/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmosis/version.py b/cosmosis/version.py index bf5afe7f..f1d93208 100644 --- a/cosmosis/version.py +++ b/cosmosis/version.py @@ -1 +1 @@ -__version__ = '3.5.1' +__version__ = '3.6'