From e159fd6d5ede47903c91a84d8c97e270835808a6 Mon Sep 17 00:00:00 2001 From: Nat Efrat-Henrici <60049837+nefrathenrici@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:21:02 -0700 Subject: [PATCH] Add surfacefluxes test to CI (#66) Add `calibrate` func, surfacefluxes test for CI --- .github/workflows/ci.yml | 30 +++++----- experiments/pipeline.jl | 30 +--------- .../Manifest.toml | 56 +++++++++---------- .../model_interface.jl | 16 +++++- .../surface_fluxes_perfect_model/prior.toml | 2 - src/ekp_interface.jl | 52 +++++++++++++++++ test/test_emulate_sample.jl | 33 ++++++----- test/test_model_interface.jl | 10 ++++ test/test_sf.jl | 21 +++++++ 9 files changed, 159 insertions(+), 91 deletions(-) create mode 100644 test/test_sf.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9704be6..e83910a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,12 +8,18 @@ on: tags: '*' pull_request: +# Needed to allow julia-actions/cache to delete old caches that it has created +permissions: + actions: write + contents: read + jobs: test: name: ci ${{ matrix.version }} - ${{ matrix.os }} runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: - fail-fast: false + fail-fast: true matrix: version: - '1.10' @@ -23,24 +29,20 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - uses: julia-actions/cache@v1 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 + - name: Surface Fluxes Perfect Model Test + run: | + julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.instantiate(;verbose=true)' + julia -t 10 --project=experiments/surface_fluxes_perfect_model test/test_sf.jl - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v3 with: - file: lcov.info \ No newline at end of file + file: lcov.info + token: ${{secrets.CODECOV_TOKEN}} \ No newline at end of file diff --git a/experiments/pipeline.jl b/experiments/pipeline.jl index cd9e45a3..1f7880ad 100644 --- a/experiments/pipeline.jl +++ b/experiments/pipeline.jl @@ -9,35 +9,7 @@ pkg_dir = pkgdir(CalibrateAtmos) experiment_path = "$pkg_dir/experiments/$experiment_id" include("$experiment_path/model_interface.jl") include("$experiment_path/generate_truth.jl") -ekp_config = - YAML.load_file(joinpath("experiments", experiment_id, "ekp_config.yml")) -# initialize the CalibrateAtmos -CalibrateAtmos.initialize(experiment_id) - -# run experiment with CalibrateAtmos for N_iter iterations -N_iter = ekp_config["n_iterations"] -N_mem = ekp_config["ensemble_size"] -for i in 0:(N_iter - 1) - # run G model to produce output from N_mem ensemble members - physical_model = - CalibrateAtmos.get_forward_model(Val(Symbol(experiment_id))) - - for m in 1:N_mem # TODO: parallelize with threads! - # model run for each ensemble member - model_config = - CalibrateAtmos.get_config(physical_model, m, i, experiment_id) - CalibrateAtmos.run_forward_model(physical_model, model_config) - @info "Finished model run for member $m at iteration $i" - end - - # update EKP with the ensemble output and update calibrated parameters - G_ensemble = CalibrateAtmos.observation_map(Val(Symbol(experiment_id)), i) - output_dir = ekp_config["output_dir"] - iter_path = CalibrateAtmos.path_to_iteration(output_dir, i) - JLD2.save_object(joinpath(iter_path, "observation_map.jld2"), G_ensemble) - CalibrateAtmos.update_ensemble(experiment_id, i) - -end +CalibrateAtmos.calibrate(experiment_id) include("$pkg_dir/plot/convergence_$experiment_id.jl") diff --git a/experiments/surface_fluxes_perfect_model/Manifest.toml b/experiments/surface_fluxes_perfect_model/Manifest.toml index af62d9c2..5dab021d 100644 --- a/experiments/surface_fluxes_perfect_model/Manifest.toml +++ b/experiments/surface_fluxes_perfect_model/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.0" +julia_version = "1.10.1" manifest_format = "2.0" -project_hash = "649ec5152c3ae3d47ed46495072e82cd9bdd7d79" +project_hash = "8b52a1f87337958a3f0ec6e731cccb862b78de20" [[deps.ADTypes]] git-tree-sha1 = "41c37aa88889c171f1300ceac1313c06e891d245" @@ -44,9 +44,9 @@ version = "0.4.5" [[deps.Adapt]] deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "cea4ac3f5b4bc4b3000aa55afb6e5626518948fa" +git-tree-sha1 = "e2a9873379849ce2ac9f9fa34b0e37bde5d5fe0a" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "4.0.3" +version = "4.0.2" weakdeps = ["StaticArrays"] [deps.Adapt.extensions] @@ -92,10 +92,10 @@ uuid = "68821587-b530-5797-8361-c406ea357684" version = "3.5.1+1" [[deps.ArrayInterface]] -deps = ["Adapt", "LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "c85c172841acde8dcdf5d77967b14c89c33d65cc" +deps = ["Adapt", "LinearAlgebra", "Requires", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "881e43f1aa014a6f75c8fc0847860e00a1500846" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "7.8.1" +version = "7.8.0" [deps.ArrayInterface.extensions] ArrayInterfaceBandedMatricesExt = "BandedMatrices" @@ -294,13 +294,13 @@ uuid = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" version = "0.11.9" [[deps.Cairo_jll]] -deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "a4c43f59baa34011e303e76f5c8c91bf58415aaf" +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2" uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" -version = "1.18.0+1" +version = "1.16.1+1" [[deps.CalibrateAtmos]] -deps = ["CalibrateEmulateSample", "ClimaComms", "ClimaCore", "ClimaParams", "Conda", "Distributions", "EnsembleKalmanProcesses", "JLD2", "PrecompileTools", "Random", "SciMLBase", "TOML", "YAML"] +deps = ["CalibrateEmulateSample", "ClimaComms", "ClimaCore", "ClimaParams", "Distributions", "EnsembleKalmanProcesses", "JLD2", "PrecompileTools", "Random", "SciMLBase", "TOML", "YAML"] path = "../.." uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2" version = "0.1.0" @@ -426,7 +426,7 @@ weakdeps = ["Dates", "LinearAlgebra"] [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.5+1" +version = "1.1.0+0" [[deps.CompositionsBase]] git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" @@ -851,9 +851,9 @@ version = "0.1.0" [[deps.Glib_jll]] deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "359a1ba2e320790ddbe4ee8b4d54a305c0ea2aff" +git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17" uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" -version = "2.80.0+0" +version = "2.76.5+0" [[deps.Graphics]] deps = ["Colors", "LinearAlgebra", "NaNMath"] @@ -1233,16 +1233,16 @@ uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" version = "1.17.0+0" [[deps.Libmount_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "dae976433497a2f841baadea93d27e68f1a12a97" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" -version = "2.39.3+0" +version = "2.35.0+0" [[deps.Libuuid_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "0a04a1318df1bf510beb2562cf90fb0c386f58c4" +git-tree-sha1 = "e5edc369a598dfde567269dc6add5812cfa10cd5" uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" -version = "2.39.3+1" +version = "2.39.3+0" [[deps.LightXML]] deps = ["Libdl", "XML2_jll"] @@ -1528,7 +1528,7 @@ version = "0.3.24+0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.23+2" +version = "0.3.23+4" [[deps.OpenEXR]] deps = ["Colors", "FileIO", "OpenEXR_jll"] @@ -1613,9 +1613,9 @@ version = "0.5.12" [[deps.Pango_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl"] -git-tree-sha1 = "526f5a03792669e4187e584e8ec9d534248ca765" +git-tree-sha1 = "4745216e94f71cb768d58330b059c9b76f32cb66" uuid = "36c8627f-9965-5494-a995-c6b170f724f3" -version = "1.52.1+0" +version = "1.50.14+0" [[deps.Parameters]] deps = ["OrderedCollections", "UnPack"] @@ -1817,9 +1817,9 @@ version = "1.3.4" [[deps.RecursiveArrayTools]] deps = ["Adapt", "ArrayInterface", "DocStringExtensions", "GPUArraysCore", "IteratorInterfaceExtensions", "LinearAlgebra", "RecipesBase", "SparseArrays", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"] -git-tree-sha1 = "b81c1728477663d4104910f448770f448f946d11" +git-tree-sha1 = "dc428bb59c20dafd1ec500c3432b9e3d7e78e7f3" uuid = "731186ca-8d62-57ce-b412-fbd966d074cd" -version = "3.11.0" +version = "3.10.1" [deps.RecursiveArrayTools.extensions] RecursiveArrayToolsFastBroadcastExt = "FastBroadcast" @@ -2211,9 +2211,9 @@ weakdeps = ["ClimaParams"] CreateParametersExt = "ClimaParams" [[deps.SymbolicIndexingInterface]] -git-tree-sha1 = "7303000b4a1d5348c8c26fc4c856f5f4982885b1" +git-tree-sha1 = "251bb311585143931a306175c3b7ced220300578" uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5" -version = "0.3.9" +version = "0.3.8" [[deps.TOML]] deps = ["Dates"] @@ -2508,9 +2508,9 @@ version = "2.0.2+0" [[deps.libpng_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "d7015d2e18a5fd9a4f47de711837e980519781a4" +git-tree-sha1 = "1ea2ebe8ffa31f9c324e8c1d6e86b4165b84a024" uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" -version = "1.6.43+1" +version = "1.6.43+0" [[deps.libsixel_jll]] deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "libpng_jll"] diff --git a/experiments/surface_fluxes_perfect_model/model_interface.jl b/experiments/surface_fluxes_perfect_model/model_interface.jl index 1fefb239..37dc3d30 100644 --- a/experiments/surface_fluxes_perfect_model/model_interface.jl +++ b/experiments/surface_fluxes_perfect_model/model_interface.jl @@ -33,7 +33,7 @@ struct SurfaceFluxModel <: AbstractPhysicalModel end include("sf_model.jl") include("observation_map.jl") -function get_forward_model(experiment_id::Val{:surface_fluxes_perfect_model}) +function get_forward_model(::Val{:surface_fluxes_perfect_model}) return SurfaceFluxModel() end @@ -84,8 +84,18 @@ end Runs the model with the given an AbstractDict object. """ -function run_forward_model(::SurfaceFluxModel, config::AbstractDict) - x_inputs = load_profiles(config["x_data_file"]) +function run_forward_model( + ::SurfaceFluxModel, + config::AbstractDict; + lk = nothing, +) + x_inputs = if isnothing(lk) + load_profiles(config["x_data_file"]) + else + lock(lk) do + load_profiles(config["x_data_file"]) + end + end FT = typeof(x_inputs.profiles_int[1].T) obtain_ustar(FT, x_inputs, config) end diff --git a/experiments/surface_fluxes_perfect_model/prior.toml b/experiments/surface_fluxes_perfect_model/prior.toml index 5671848b..e431729f 100644 --- a/experiments/surface_fluxes_perfect_model/prior.toml +++ b/experiments/surface_fluxes_perfect_model/prior.toml @@ -1,9 +1,7 @@ ["coefficient_a_m_businger"] prior = "constrained_gaussian(coefficient_a_m_businger, 4.7, 0.5, 2, 6)" type = "float" -alias = "businger_a_m" ["coefficient_a_h_businger"] prior = "constrained_gaussian(coefficient_a_h_businger, 4.6, 3, 0, 10)" type = "float" -alias = "businger_a_h" \ No newline at end of file diff --git a/src/ekp_interface.jl b/src/ekp_interface.jl index 21dbff30..54914ef6 100644 --- a/src/ekp_interface.jl +++ b/src/ekp_interface.jl @@ -5,6 +5,7 @@ using Distributions import EnsembleKalmanProcesses as EKP using EnsembleKalmanProcesses.ParameterDistributions using EnsembleKalmanProcesses.TOMLInterface +import ClimaComms """ path_to_iteration(output_dir, iteration) @@ -131,4 +132,55 @@ function update_ensemble( iter_path = path_to_iteration(output_dir, iteration) eki_path = joinpath(iter_path, "eki_file.jld2") JLD2.save_object(eki_path, eki) + return eki +end + +""" + calibrate(experiment_id) + +Convenience function for running a full calibration experiment for the given +`experiment_id`. +This function requires the relevant experiment project and model interface to be loaded. + +```julia +import CalibrateAtmos + +experiment_id = "surface_fluxes_perfect_model" +experiment_path = joinpath(pkgdir(CalibrateAtmos), "experiments", experiment_id) +include(joinpath(experiment_path, "model_interface.jl")) +include(joinpath(experiment_path, "generate_truth.jl")) + +eki = CalibrateAtmos.calibrate(experiment_id) +``` +""" +function calibrate(experiment_id; device = ClimaComms.device()) + ekp_config = + YAML.load_file(joinpath("experiments", experiment_id, "ekp_config.yml")) + # initialize the CalibrateAtmos + initialize(experiment_id) + + # run experiment with CalibrateAtmos for N_iter iterations + N_iter = ekp_config["n_iterations"] + N_mem = ekp_config["ensemble_size"] + output_dir = ekp_config["output_dir"] + eki = nothing + physical_model = get_forward_model(Val(Symbol(experiment_id))) + lk = ReentrantLock() + for i in 0:(N_iter - 1) + ClimaComms.@threaded device for m in 1:N_mem + # model run for each ensemble member + model_config = get_config(physical_model, m, i, experiment_id) + run_forward_model(physical_model, model_config; lk) + @info "Finished model run for member $m at iteration $i" + end + + # update EKP with the ensemble output and update calibrated parameters + G_ensemble = observation_map(Val(Symbol(experiment_id)), i) + JLD2.save_object( + joinpath(path_to_iteration(output_dir, i), "observation_map.jld2"), + G_ensemble, + ) + eki = update_ensemble(experiment_id, i) + end + return eki end diff --git a/test/test_emulate_sample.jl b/test/test_emulate_sample.jl index 308c7412..00eb3b1d 100644 --- a/test/test_emulate_sample.jl +++ b/test/test_emulate_sample.jl @@ -1,3 +1,4 @@ +using Test import JLD2 import Statistics: mean @@ -9,26 +10,28 @@ using EnsembleKalmanProcesses.TOMLInterface import CalibrateAtmos as CAL -y_obs = [261.5493] -y_noise_cov = [0.02619;;] -ekp = JLD2.load_object(joinpath("test_case_inputs", "eki_test.jld2")) -init_params = [EKP.get_u_final(ekp)[1]] +@testset "Emulate and Sample tests" begin + y_obs = [261.5493] + y_noise_cov = [0.02619;;] + ekp = JLD2.load_object(joinpath("test_case_inputs", "eki_test.jld2")) + init_params = [EKP.get_u_final(ekp)[1]] -prior_path = joinpath("test_case_inputs", "sphere_hs_rhoe.toml") + prior_path = joinpath("test_case_inputs", "sphere_hs_rhoe.toml") -prior = CAL.get_prior(prior_path) + prior = CAL.get_prior(prior_path) -input_output_pairs = CAL.get_input_output_pairs(ekp) + input_output_pairs = CAL.get_input_output_pairs(ekp) -@test input_output_pairs.inputs.stored_data == - hcat([ekp.u[i].stored_data for i in 1:(length(ekp.u) - 1)]...) -@test input_output_pairs.outputs.stored_data == - hcat([ekp.g[i].stored_data for i in 1:length(ekp.g)]...) + @test input_output_pairs.inputs.stored_data == + hcat([ekp.u[i].stored_data for i in 1:(length(ekp.u) - 1)]...) + @test input_output_pairs.outputs.stored_data == + hcat([ekp.g[i].stored_data for i in 1:length(ekp.g)]...) -emulator = CAL.gp_emulator(input_output_pairs, y_noise_cov) + emulator = CAL.gp_emulator(input_output_pairs, y_noise_cov) -(; mcmc, chain) = CAL.sample(emulator, y_obs, prior, init_params) -@test mean(chain.value[1:100000]) ≈ 4.19035299 rtol = 0.0001 + (; mcmc, chain) = CAL.sample(emulator, y_obs, prior, init_params) + @test mean(chain.value[1:100000]) ≈ 4.19035299 rtol = 0.0001 -constrained_posterior = CAL.save_posterior(mcmc, chain) + constrained_posterior = CAL.save_posterior(mcmc, chain) +end diff --git a/test/test_model_interface.jl b/test/test_model_interface.jl index 3a1b063d..3ce8f863 100644 --- a/test/test_model_interface.jl +++ b/test/test_model_interface.jl @@ -29,3 +29,13 @@ end "observation_map not implemented for experiment Val{:test}() at iteration 1", ) CalibrateAtmos.observation_map(Val(:test), 1) end + +# This test depends on `surface_fluxes_perfect_model` in the `experiments/` folder +@testset "calibrate func" begin + dir = pwd() + cd(pkgdir(CalibrateAtmos)) + @test_throws ErrorException CalibrateAtmos.calibrate( + "surface_fluxes_perfect_model", + ) + cd(dir) +end diff --git a/test/test_sf.jl b/test/test_sf.jl new file mode 100644 index 00000000..082259b3 --- /dev/null +++ b/test/test_sf.jl @@ -0,0 +1,21 @@ +import CalibrateAtmos +import EnsembleKalmanProcesses: get_ϕ_mean_final, get_g_mean_final +using Test + +experiment_id = "surface_fluxes_perfect_model" +experiment_path = joinpath(pkgdir(CalibrateAtmos), "experiments", experiment_id) +include(joinpath(experiment_path, "model_interface.jl")) +include(joinpath(experiment_path, "generate_truth.jl")) + +prior = CalibrateAtmos.get_prior(joinpath(experiment_path, "prior.toml")) +eki = CalibrateAtmos.calibrate(experiment_id) + +@testset "Test pure Julia calibration (surface fluxes perfect model)" begin + parameter_values = get_ϕ_mean_final(prior, eki) + test_parameter_values = [4.8684152849621976, 5.2022848059758875] + @test all(isapprox.(parameter_values, test_parameter_values; rtol = 1e-3)) + + forward_model_output = get_g_mean_final(eki) + test_model_output = [0.04734060615301132] + @test all(isapprox.(forward_model_output, test_model_output; rtol = 1e-3)) +end