Add surfacefluxes test to CI (#66)

Add `calibrate` func, surfacefluxes test for CI

.github/workflows/ci.yml

@@ -8,12 +8,18 @@ on:
     tags: '*'
   pull_request:
 
+# Needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
+
 jobs:
   test:
     name: ci ${{ matrix.version }} - ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         version:
           - '1.10'
@@ -23,24 +29,20 @@ jobs:
         arch:
           - x64
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: julia-actions/cache@v1
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v1
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
+      - name: Surface Fluxes Perfect Model Test
+        run: |
+          julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.instantiate(;verbose=true)'
+          julia -t 10 --project=experiments/surface_fluxes_perfect_model test/test_sf.jl
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
-          file: lcov.info
+          file: lcov.info
+          token: ${{secrets.CODECOV_TOKEN}}

experiments/pipeline.jl

@@ -9,35 +9,7 @@ pkg_dir = pkgdir(CalibrateAtmos)
 experiment_path = "$pkg_dir/experiments/$experiment_id"
 include("$experiment_path/model_interface.jl")
 include("$experiment_path/generate_truth.jl")
-ekp_config =
-    YAML.load_file(joinpath("experiments", experiment_id, "ekp_config.yml"))
 
-# initialize the CalibrateAtmos
-CalibrateAtmos.initialize(experiment_id)
-
-# run experiment with CalibrateAtmos for N_iter iterations
-N_iter = ekp_config["n_iterations"]
-N_mem = ekp_config["ensemble_size"]
-for i in 0:(N_iter - 1)
-    # run G model to produce output from N_mem ensemble members
-    physical_model =
-        CalibrateAtmos.get_forward_model(Val(Symbol(experiment_id)))
-
-    for m in 1:N_mem # TODO: parallelize with threads!
-        # model run for each ensemble member
-        model_config =
-            CalibrateAtmos.get_config(physical_model, m, i, experiment_id)
-        CalibrateAtmos.run_forward_model(physical_model, model_config)
-        @info "Finished model run for member $m at iteration $i"
-    end
-
-    # update EKP with the ensemble output and update calibrated parameters
-    G_ensemble = CalibrateAtmos.observation_map(Val(Symbol(experiment_id)), i)
-    output_dir = ekp_config["output_dir"]
-    iter_path = CalibrateAtmos.path_to_iteration(output_dir, i)
-    JLD2.save_object(joinpath(iter_path, "observation_map.jld2"), G_ensemble)
-    CalibrateAtmos.update_ensemble(experiment_id, i)
-
-end
+CalibrateAtmos.calibrate(experiment_id)
 
 include("$pkg_dir/plot/convergence_$experiment_id.jl")

experiments/surface_fluxes_perfect_model/Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.0"
+julia_version = "1.10.1"
 manifest_format = "2.0"
-project_hash = "649ec5152c3ae3d47ed46495072e82cd9bdd7d79"
+project_hash = "8b52a1f87337958a3f0ec6e731cccb862b78de20"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "41c37aa88889c171f1300ceac1313c06e891d245"
@@ -44,9 +44,9 @@ version = "0.4.5"
 
 [[deps.Adapt]]
 deps = ["LinearAlgebra", "Requires"]
-git-tree-sha1 = "cea4ac3f5b4bc4b3000aa55afb6e5626518948fa"
+git-tree-sha1 = "e2a9873379849ce2ac9f9fa34b0e37bde5d5fe0a"
 uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-version = "4.0.3"
+version = "4.0.2"
 weakdeps = ["StaticArrays"]
 
     [deps.Adapt.extensions]
@@ -92,10 +92,10 @@ uuid = "68821587-b530-5797-8361-c406ea357684"
 version = "3.5.1+1"
 
 [[deps.ArrayInterface]]
-deps = ["Adapt", "LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "c85c172841acde8dcdf5d77967b14c89c33d65cc"
+deps = ["Adapt", "LinearAlgebra", "Requires", "SparseArrays", "SuiteSparse"]
+git-tree-sha1 = "881e43f1aa014a6f75c8fc0847860e00a1500846"
 uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
-version = "7.8.1"
+version = "7.8.0"
 
     [deps.ArrayInterface.extensions]
     ArrayInterfaceBandedMatricesExt = "BandedMatrices"
@@ -294,13 +294,13 @@ uuid = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 version = "0.11.9"
 
 [[deps.Cairo_jll]]
-deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
-git-tree-sha1 = "a4c43f59baa34011e303e76f5c8c91bf58415aaf"
+deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
+git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2"
 uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
-version = "1.18.0+1"
+version = "1.16.1+1"
 
 [[deps.CalibrateAtmos]]
-deps = ["CalibrateEmulateSample", "ClimaComms", "ClimaCore", "ClimaParams", "Conda", "Distributions", "EnsembleKalmanProcesses", "JLD2", "PrecompileTools", "Random", "SciMLBase", "TOML", "YAML"]
+deps = ["CalibrateEmulateSample", "ClimaComms", "ClimaCore", "ClimaParams", "Distributions", "EnsembleKalmanProcesses", "JLD2", "PrecompileTools", "Random", "SciMLBase", "TOML", "YAML"]
 path = "../.."
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
 version = "0.1.0"
@@ -426,7 +426,7 @@ weakdeps = ["Dates", "LinearAlgebra"]
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "1.0.5+1"
+version = "1.1.0+0"
 
 [[deps.CompositionsBase]]
 git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"
@@ -851,9 +851,9 @@ version = "0.1.0"
 
 [[deps.Glib_jll]]
 deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"]
-git-tree-sha1 = "359a1ba2e320790ddbe4ee8b4d54a305c0ea2aff"
+git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17"
 uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
-version = "2.80.0+0"
+version = "2.76.5+0"
 
 [[deps.Graphics]]
 deps = ["Colors", "LinearAlgebra", "NaNMath"]
@@ -1233,16 +1233,16 @@ uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
 version = "1.17.0+0"
 
 [[deps.Libmount_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "dae976433497a2f841baadea93d27e68f1a12a97"
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73"
 uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9"
-version = "2.39.3+0"
+version = "2.35.0+0"
 
 [[deps.Libuuid_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "0a04a1318df1bf510beb2562cf90fb0c386f58c4"
+git-tree-sha1 = "e5edc369a598dfde567269dc6add5812cfa10cd5"
 uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
-version = "2.39.3+1"
+version = "2.39.3+0"
 
 [[deps.LightXML]]
 deps = ["Libdl", "XML2_jll"]
@@ -1528,7 +1528,7 @@ version = "0.3.24+0"
 [[deps.OpenBLAS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
 uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.23+2"
+version = "0.3.23+4"
 
 [[deps.OpenEXR]]
 deps = ["Colors", "FileIO", "OpenEXR_jll"]
@@ -1613,9 +1613,9 @@ version = "0.5.12"
 
 [[deps.Pango_jll]]
 deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "526f5a03792669e4187e584e8ec9d534248ca765"
+git-tree-sha1 = "4745216e94f71cb768d58330b059c9b76f32cb66"
 uuid = "36c8627f-9965-5494-a995-c6b170f724f3"
-version = "1.52.1+0"
+version = "1.50.14+0"
 
 [[deps.Parameters]]
 deps = ["OrderedCollections", "UnPack"]
@@ -1817,9 +1817,9 @@ version = "1.3.4"
 
 [[deps.RecursiveArrayTools]]
 deps = ["Adapt", "ArrayInterface", "DocStringExtensions", "GPUArraysCore", "IteratorInterfaceExtensions", "LinearAlgebra", "RecipesBase", "SparseArrays", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"]
-git-tree-sha1 = "b81c1728477663d4104910f448770f448f946d11"
+git-tree-sha1 = "dc428bb59c20dafd1ec500c3432b9e3d7e78e7f3"
 uuid = "731186ca-8d62-57ce-b412-fbd966d074cd"
-version = "3.11.0"
+version = "3.10.1"
 
     [deps.RecursiveArrayTools.extensions]
     RecursiveArrayToolsFastBroadcastExt = "FastBroadcast"
@@ -2211,9 +2211,9 @@ weakdeps = ["ClimaParams"]
     CreateParametersExt = "ClimaParams"
 
 [[deps.SymbolicIndexingInterface]]
-git-tree-sha1 = "7303000b4a1d5348c8c26fc4c856f5f4982885b1"
+git-tree-sha1 = "251bb311585143931a306175c3b7ced220300578"
 uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5"
-version = "0.3.9"
+version = "0.3.8"
 
 [[deps.TOML]]
 deps = ["Dates"]
@@ -2508,9 +2508,9 @@ version = "2.0.2+0"
 
 [[deps.libpng_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"]
-git-tree-sha1 = "d7015d2e18a5fd9a4f47de711837e980519781a4"
+git-tree-sha1 = "1ea2ebe8ffa31f9c324e8c1d6e86b4165b84a024"
 uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f"
-version = "1.6.43+1"
+version = "1.6.43+0"
 
 [[deps.libsixel_jll]]
 deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "libpng_jll"]

experiments/surface_fluxes_perfect_model/model_interface.jl

@@ -33,7 +33,7 @@ struct SurfaceFluxModel <: AbstractPhysicalModel end
 include("sf_model.jl")
 include("observation_map.jl")
 
-function get_forward_model(experiment_id::Val{:surface_fluxes_perfect_model})
+function get_forward_model(::Val{:surface_fluxes_perfect_model})
     return SurfaceFluxModel()
 end
 
@@ -84,8 +84,18 @@ end
 Runs the model with the given an AbstractDict object.
 """
 
-function run_forward_model(::SurfaceFluxModel, config::AbstractDict)
-    x_inputs = load_profiles(config["x_data_file"])
+function run_forward_model(
+    ::SurfaceFluxModel,
+    config::AbstractDict;
+    lk = nothing,
+)
+    x_inputs = if isnothing(lk)
+        load_profiles(config["x_data_file"])
+    else
+        lock(lk) do
+            load_profiles(config["x_data_file"])
+        end
+    end
     FT = typeof(x_inputs.profiles_int[1].T)
     obtain_ustar(FT, x_inputs, config)
 end

experiments/surface_fluxes_perfect_model/prior.toml

@@ -1,9 +1,7 @@
 ["coefficient_a_m_businger"]
 prior = "constrained_gaussian(coefficient_a_m_businger, 4.7, 0.5, 2, 6)"
 type = "float"
-alias = "businger_a_m"
 
 ["coefficient_a_h_businger"]
 prior = "constrained_gaussian(coefficient_a_h_businger, 4.6, 3, 0, 10)"
 type = "float"
-alias = "businger_a_h"

src/ekp_interface.jl

@@ -5,6 +5,7 @@ using Distributions
 import EnsembleKalmanProcesses as EKP
 using EnsembleKalmanProcesses.ParameterDistributions
 using EnsembleKalmanProcesses.TOMLInterface
+import ClimaComms
 
 """
     path_to_iteration(output_dir, iteration)
@@ -131,4 +132,55 @@ function update_ensemble(
     iter_path = path_to_iteration(output_dir, iteration)
     eki_path = joinpath(iter_path, "eki_file.jld2")
     JLD2.save_object(eki_path, eki)
+    return eki
+end
+
+"""
+    calibrate(experiment_id)
+
+Convenience function for running a full calibration experiment for the given
+`experiment_id`. 
+This function requires the relevant experiment project and model interface to be loaded.
+
+```julia
+import CalibrateAtmos
+
+experiment_id = "surface_fluxes_perfect_model"
+experiment_path = joinpath(pkgdir(CalibrateAtmos), "experiments", experiment_id)
+include(joinpath(experiment_path, "model_interface.jl"))
+include(joinpath(experiment_path, "generate_truth.jl"))
+
+eki = CalibrateAtmos.calibrate(experiment_id)
+```
+"""
+function calibrate(experiment_id; device = ClimaComms.device())
+    ekp_config =
+        YAML.load_file(joinpath("experiments", experiment_id, "ekp_config.yml"))
+    # initialize the CalibrateAtmos
+    initialize(experiment_id)
+
+    # run experiment with CalibrateAtmos for N_iter iterations
+    N_iter = ekp_config["n_iterations"]
+    N_mem = ekp_config["ensemble_size"]
+    output_dir = ekp_config["output_dir"]
+    eki = nothing
+    physical_model = get_forward_model(Val(Symbol(experiment_id)))
+    lk = ReentrantLock()
+    for i in 0:(N_iter - 1)
+        ClimaComms.@threaded device for m in 1:N_mem
+            # model run for each ensemble member
+            model_config = get_config(physical_model, m, i, experiment_id)
+            run_forward_model(physical_model, model_config; lk)
+            @info "Finished model run for member $m at iteration $i"
+        end
+
+        # update EKP with the ensemble output and update calibrated parameters
+        G_ensemble = observation_map(Val(Symbol(experiment_id)), i)
+        JLD2.save_object(
+            joinpath(path_to_iteration(output_dir, i), "observation_map.jld2"),
+            G_ensemble,
+        )
+        eki = update_ensemble(experiment_id, i)
+    end
+    return eki
 end