diff --git a/.buildkite/clima_server_test/pipeline.yml b/.buildkite/clima_server_test/pipeline.yml index 02faaaad..90c24eb1 100644 --- a/.buildkite/clima_server_test/pipeline.yml +++ b/.buildkite/clima_server_test/pipeline.yml @@ -17,7 +17,7 @@ steps: key: "init_cpu_env" command: - echo "--- Instantiate SurfaceFluxes calibration project" - - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.develop(;path="."); Pkg.precompile()' + - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.precompile()' - wait - label: "SurfaceFluxes perfect model calibration" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7288490b..085008e6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -13,7 +13,7 @@ steps: key: "init_cpu_env" command: - echo "--- Instantiate SurfaceFluxes calibration project" - - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.develop(;path="."); Pkg.precompile()' + - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.precompile()' - wait - label: "SurfaceFluxes perfect model calibration" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f26ab0cd..aed69dbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: - uses: actions/cache@v1 env: cache-name: cache-artifacts + PYTHON: "" with: path: ~/.julia/artifacts key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} @@ -44,6 +45,17 @@ jobs: with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} + # CalibrateEmulateSample relies on Python modules + # We retry this step because the first install is flaky + - uses: nick-fields/retry@v3 + with: + max_attempts: 2 + timeout_minutes: 10 + command: | + julia --color=yes --project=test -e 'using Pkg; Pkg.instantiate()' + julia --project=test -e 'using Conda; Conda.add("scipy=1.14.1"); Conda.add("scikit-learn=1.5.1")' + env: + PYTHON: "" - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 diff --git a/Project.toml b/Project.toml index 95d0fe19..bbadcc44 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ClimaCalibrate" uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2" authors = ["Climate Modeling Alliance"] -version = "0.0.3" +version = "0.0.4" [deps] Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" @@ -22,8 +22,8 @@ CESExt = "CalibrateEmulateSample" CalibrateEmulateSample = "0.5" Distributed = "1" Distributions = "0.25" -EnsembleKalmanProcesses = "1" -JLD2 = "0.4" +EnsembleKalmanProcesses = "1, 2" +JLD2 = "0.4, 0.5" Random = "1" TOML = "1" YAML = "0.4" diff --git a/docs/Project.toml b/docs/Project.toml index dfb9f086..0be233e7 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,8 +1,6 @@ [deps] ClimaCalibrate = "4347a170-ebd6-470c-89d3-5c705c0cacc2" -CalibrateEmulateSample = "95e48a1f-0bec-4818-9538-3db4340308e3" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" EnsembleKalmanProcesses = "aa8a2aa5-91d8-4396-bcef-d4f2ec43552d" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" - diff --git a/experiments/surface_fluxes_perfect_model/Manifest.toml b/experiments/surface_fluxes_perfect_model/Manifest.toml index f6e2cc6e..29b8841c 100644 --- a/experiments/surface_fluxes_perfect_model/Manifest.toml +++ b/experiments/surface_fluxes_perfect_model/Manifest.toml @@ -183,10 +183,10 @@ weakdeps = ["SparseArrays"] ChainRulesCoreSparseArraysExt = "SparseArrays" [[deps.ClimaCalibrate]] -deps = ["Distributions", "EnsembleKalmanProcesses", "JLD2", "Random", "TOML", "YAML"] +deps = ["Distributed", "Distributions", "EnsembleKalmanProcesses", "JLD2", "Random", "TOML", "YAML"] path = "../.." uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2" -version = "0.0.2" +version = "0.0.4" [deps.ClimaCalibrate.extensions] CESExt = "CalibrateEmulateSample" diff --git a/test/Project.toml b/test/Project.toml index 895336b4..0b7abcc4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -3,6 +3,7 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CalibrateEmulateSample = "95e48a1f-0bec-4818-9538-3db4340308e3" ClimaCalibrate = "4347a170-ebd6-470c-89d3-5c705c0cacc2" ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c" +Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" EnsembleKalmanProcesses = "aa8a2aa5-91d8-4396-bcef-d4f2ec43552d" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" @@ -11,6 +12,3 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[compat] -EnsembleKalmanProcesses = "< 1.1.6" diff --git a/test/emulate_sample.jl b/test/emulate_sample.jl index 84b8e6c9..beae2365 100644 --- a/test/emulate_sample.jl +++ b/test/emulate_sample.jl @@ -13,23 +13,24 @@ import ClimaCalibrate as CAL @testset "Emulate and Sample tests" begin y_obs = [261.5493] y_noise_cov = [0.02619;;] - ekp = JLD2.load_object(joinpath("test_case_inputs", "eki_test.jld2")) - init_params = [EKP.get_u_final(ekp)[1]] - + init_params = [4.1928479297330306] prior_path = joinpath("test_case_inputs", "sphere_hs_rhoe.toml") - prior = CAL.get_prior(prior_path) - input_output_pairs = CAL.get_input_output_pairs(ekp) - - @test input_output_pairs.inputs.stored_data == - hcat([ekp.u[i].stored_data for i in 1:(length(ekp.u) - 1)]...) - @test input_output_pairs.outputs.stored_data == - hcat([ekp.g[i].stored_data for i in 1:length(ekp.g)]...) + # Sample data + input_output_pairs = EKP.DataContainers.PairedDataContainer( + # Parameter values + EKP.DataContainers.DataContainer( + [5.05038266176406 4.49801357065531 4.625175276903062 4.497648108467572 5.049462857273453 5.470189766650297 4.945930038719361 4.694723166004012 4.936413547464582 4.618148508404665 4.041965675651152 4.121910454258125 4.077521148463588 4.119541027138994 4.043163793654862 4.21813038496636 4.0249164182511485 4.0292005257357895 4.026103892229238 4.0880286181841585 4.1861745817794205 4.20048531589655 4.187992300461601 4.19870114405988 4.209232786450678 4.191675948737242 4.187239734960614 4.186875241183792 4.189230960868622 4.19193208121736], + ), + # Observation map output values + EKP.DataContainers.DataContainer( + [229.28858947753906 249.54794311523438 244.09906005859375 249.78701782226562 229.31289672851562 221.62457275390625 232.126953125 240.37799072265625 232.520751953125 244.3943634033203 266.1481628417969 263.6910705566406 265.0572509765625 263.7556457519531 266.0339660644531 260.7401428222656 266.4653015136719 266.42529296875 266.4148864746094 264.5828552246094 261.7020568847656 261.2137451171875 261.7446594238281 261.2926025390625 260.7995910644531 261.4737548828125 261.597900390625 261.69354248046875 261.52777099609375 261.3088684082031], + ), + ) emulator = CAL.gp_emulator(input_output_pairs, y_noise_cov) - (; mcmc, chain) = CAL.sample(emulator, y_obs, prior, init_params) @test mean(chain.value[1:100000]) ≈ 4.19035299 rtol = 0.0001 diff --git a/test/pure_julia_e2e.jl b/test/pure_julia_e2e.jl index 12718e26..3c10a19b 100644 --- a/test/pure_julia_e2e.jl +++ b/test/pure_julia_e2e.jl @@ -19,7 +19,7 @@ import JLD2 output_file = "model_output.jld2" prior = constrained_gaussian("test_param", 10, 5, 0, Inf) n_iterations = 1 -ensemble_size = 10 +ensemble_size = 20 observations = [20.0] noise = [0.01;;] output_dir = joinpath("test", "e2e_test_output") @@ -34,6 +34,8 @@ experiment_config = ExperimentConfig( ) # Model interface +# This "model" just samples parameters and returns them, we are checking that the +# results are reproducible. function set_up_forward_model( member, iteration, @@ -55,9 +57,7 @@ function run_forward_model(config) JLD2.save_object(joinpath(config["output_dir"], output_file), output) end -# Observation map function observation_map(iteration) - (; ensemble_size) = experiment_config dims = 1 G_ensemble = Array{Float64}(undef, dims..., ensemble_size) @@ -76,8 +76,8 @@ ekp = calibrate(JuliaBackend, experiment_config) @testset "Test end-to-end calibration" begin parameter_values = [EKP.get_ϕ_mean(prior, ekp, it) for it in 1:(n_iterations + 1)] - @test parameter_values[1][1] ≈ 9.779 rtol = 0.01 - @test parameter_values[end][1] ≈ 19.63 rtol = 0.01 + @test parameter_values[1][1] ≈ 9.215 rtol = 0.01 + @test parameter_values[end][1] ≈ 20.224 rtol = 0.01 end rm(output_dir; recursive = true) diff --git a/test/test_case_inputs/eki_test.jld2 b/test/test_case_inputs/eki_test.jld2 deleted file mode 100644 index 924eb89e..00000000 Binary files a/test/test_case_inputs/eki_test.jld2 and /dev/null differ