Merge pull request #125 from joezuntz/campaign-fixes

Campaign issues

cosmosis/campaign.py

@@ -410,9 +410,6 @@ def build_run(name, run_info, runs, components, output_dir, submission_info, out
     -------
     """
     run = run_info.copy()
-    env_vars = run_info.get("env", {})
-    run["env"] = env_vars
-
 
     # We want to delay expanding environment variables so that child runs
     # have a chance to override them. So we set no_expand_vars=True on all of these
@@ -428,6 +425,17 @@ def build_run(name, run_info, runs, components, output_dir, submission_info, out
     else:
         warnings.warn(f"Run {name} specifies neither 'parent' nor 'base' so is invalid")
         return None
+
+    # Build environment variables
+    # These are inherited from the parent run, if there is one,
+    # and then updated with any specific to this run, which can overwrite.
+    # env vars are only applied right at the end when all runs are collected
+    if "parent" in run_info:
+        env_vars = parent["env"].copy()
+    else:
+        env_vars = {}
+    env_vars.update(run_info.get("env", {}))
+    run["env"] = env_vars
 
     # Build values file, which is mandatory
     if "parent" in run_info:
@@ -575,6 +583,9 @@ def parse_yaml_run_file(run_config):
     -------
     runs : dict
         A dictionary of runs, keyed by name
+    
+    components : dict
+        A dictionary of components, keyed by name
     """
     if isinstance(run_config, dict):
         info = run_config
@@ -592,16 +603,19 @@ def parse_yaml_run_file(run_config):
     # Can include another run file, which we deal with
     # recursively.  
     runs = {}
+    components = {}
     for include_file in include:
-        runs.update(parse_yaml_run_file(include_file))
+        inc_runs, inc_comps = parse_yaml_run_file(include_file)
+        components.update(inc_comps)
+        runs.update(inc_runs)
 
     # But we override the output directory
     # of any imported runs with the one we have here   
     for name, run in runs.items():
         set_output_dir(run["params"], name, output_dir, output_name)
 
     # deal with re-usable components
-    components = info.get("components", {})
+    components.update(info.get("components", {}))
 
     submission_info = info.get("submission", {})
 
@@ -614,7 +628,7 @@ def parse_yaml_run_file(run_config):
     # a chance to override the environment variables of their parents.
     expand_environment_variables(runs)
 
-    return runs
+    return runs, components
 
 
 def show_run(run):
@@ -799,7 +813,7 @@ def submit_run(run_file, run):
 
 
 def main(args):
-    runs = parse_yaml_run_file(args.run_config)
+    runs, _ = parse_yaml_run_file(args.run_config)
 
     if args.mpi and not args.run:
         raise ValueError("MPI can only be used when running a single run")

cosmosis/test/campaign.yml

@@ -1,6 +1,11 @@
 output_dir: output/campaign-test
 output_name: my_project_{name}_suite1
-include: []
+include: [cosmosis/test/included.yml]
+
+components: 
+  test_component_1:
+    params:
+    - emcee.walkers = 100
 
 submission:
   # These parameters can be overridden per run.
@@ -65,3 +70,33 @@ runs:
     parent: env-test-1
     env:
       TEST : yyy
+
+  - name: env-test-3
+    parent: env-test-1
+    params:
+    - emcee.walkers = ${TEST}
+
+  - name: env-test-4
+    parent: env-test-2
+    env:
+      TEST2 : zzz
+
+  # Test double inheritance
+  - name: env-test-5
+    parent: env-test-4
+
+  - name: component-test
+    parent: v1
+    components:
+    - test_component_1
+
+  - name: include-test-1
+    parent: imported-run
+    params:
+    - emcee.walkers = 755
+
+  - name: include-test-2
+    parent: v1
+    components:
+    - test_component_2
+

cosmosis/test/example.ini

@@ -12,3 +12,4 @@ priors = cosmosis/test/example-priors.ini
 [test1]
 file = cosmosis/test/example_module.py
 env_test_var=${TEST}
+env_test_var2=${TEST2}

cosmosis/test/included.yml

@@ -0,0 +1,10 @@
+
+components: 
+  test_component_2:
+    params:
+    - emcee.walkers = 200
+
+
+runs:
+  - name: imported-run
+    base: cosmosis/test/example.ini

cosmosis/test/test_campaign.py

@@ -5,6 +5,8 @@
 import contextlib
 import pytest
 
+NRUN = 13
+
 @contextlib.contextmanager
 def run_from_source_dir():
     this_dir = os.path.split(os.path.abspath(__file__))[0]
@@ -70,9 +72,9 @@ def test_pipeline_prepend():
 
 def test_campaign_functions():
     with run_from_source_dir():
-        runs = parse_yaml_run_file("cosmosis/test/campaign.yml")
+        runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml")
 
-        assert len(runs) == 6
+        assert len(runs) == NRUN
         assert "v1" in runs
         assert runs["v2"]["values"].get("parameters", "p1") == "-2.0 0.0 2.0"
         assert runs["v2"]["priors"].get("parameters", "p2") == "gaussian 0.0 1.0"
@@ -106,15 +108,18 @@ def test_campaign_functions2():
         with open("cosmosis/test/campaign.yml") as f:
             runs_config = load_yaml(f)
 
-        with tempfile.TemporaryDirectory() as dirname:
-            runs_config['output_dir']  = dirname
-            runs = parse_yaml_run_file(runs_config)
+    with tempfile.TemporaryDirectory() as dirname:
+        runs_config['output_dir']  = dirname
+
+        with run_from_source_dir():
+            with open("cosmosis/test/campaign.yml") as f:
+                runs, _ = parse_yaml_run_file(runs_config)
 
             for name in runs:
                 print(name)
 
 
-            assert len(runs) == 6
+            assert len(runs) == NRUN
             assert "v1" in runs
             assert runs["v2"]["values"].get("parameters", "p1") == "-2.0 0.0 2.0"
             assert runs["v2"]["priors"].get("parameters", "p2") == "gaussian 0.0 1.0"
@@ -142,11 +147,35 @@ def test_campaign_functions2():
 def test_campaign_env():
     os.environ["TEST"] = "aaa"
     with run_from_source_dir():
-        runs = parse_yaml_run_file("cosmosis/test/campaign.yml")
+        runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml")
     assert runs["env-test-1"]["params"].get("test1", "env_test_var")  == "xxx"
     assert runs["env-test-2"]["params"].get("test1", "env_test_var")  == "yyy"
 
+def test_inherit_env():
+    with run_from_source_dir():
+        runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml")
+    assert runs["env-test-3"]["params"].get("test1", "env_test_var")  == "xxx"
+    assert runs["env-test-3"]["params"].get("emcee", "walkers")  == "xxx"
+    assert runs["env-test-5"]["params"].get("test1", "env_test_var")  == "yyy"
+    assert runs["env-test-5"]["params"].get("test1", "env_test_var2")  == "zzz"
+
+
 def test_campaign_duplicate_keys():
     with pytest.raises(ValueError):
         with run_from_source_dir():
-            runs = parse_yaml_run_file("cosmosis/test/bad-campaign.yml")
+            runs, _ = parse_yaml_run_file("cosmosis/test/bad-campaign.yml")
+
+def test_component():
+    with run_from_source_dir():
+        runs, components = parse_yaml_run_file("cosmosis/test/campaign.yml")
+    assert "test_component_1" in components
+    assert runs["component-test"]["params"].get("emcee", "walkers") == "100"
+
+def test_include():
+    with run_from_source_dir():
+        runs, _ = parse_yaml_run_file("cosmosis/test/campaign.yml")
+    assert "imported-run" in runs
+    assert "include-test-1" in runs
+    assert runs['include-test-1']['params'].get("emcee", "walkers") == "755"
+    assert "include-test-2" in runs
+    assert runs['include-test-2']['params'].get("emcee", "walkers") == "200"

cosmosis/version.py

@@ -1 +1 @@
-__version__ = '3.5.1'
+__version__ = '3.6'

setup.py

@@ -71,6 +71,7 @@
     "test/libtest/test_c_datablock_scalars.template",
     "test/libtest/Makefile",
     "test/campaign.yml",
+    "test/included.yml",
     "test/bad-campaign.yml",
     "test/example-priors.ini",
     "test/example-values.ini",