Merge pull request #110 from CliMA/ne/up_ekp

Bump compat for EKP, JLD2

.buildkite/clima_server_test/pipeline.yml

@@ -17,7 +17,7 @@ steps:
     key: "init_cpu_env"
     command:
       - echo "--- Instantiate SurfaceFluxes calibration project"
-      - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.develop(;path="."); Pkg.precompile()'
+      - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.precompile()'
 
   - wait
   - label: "SurfaceFluxes perfect model calibration"

.buildkite/pipeline.yml

@@ -13,7 +13,7 @@ steps:
     key: "init_cpu_env"
     command:
       - echo "--- Instantiate SurfaceFluxes calibration project"
-      - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.develop(;path="."); Pkg.precompile()'
+      - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.precompile()'
 
   - wait
   - label: "SurfaceFluxes perfect model calibration"

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ jobs:
       - uses: actions/cache@v1
         env:
           cache-name: cache-artifacts
+          PYTHON: ""
         with:
           path: ~/.julia/artifacts
           key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
@@ -44,6 +45,17 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
+      # CalibrateEmulateSample relies on Python modules
+      # We retry this step because the first install is flaky
+      - uses: nick-fields/retry@v3
+        with:
+          max_attempts: 2
+          timeout_minutes: 10
+          command: |
+            julia --color=yes --project=test -e 'using Pkg; Pkg.instantiate()'
+            julia --project=test -e 'using Conda; Conda.add("scipy=1.14.1"); Conda.add("scikit-learn=1.5.1")'
+        env: 
+          PYTHON: ""
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1

Project.toml

@@ -1,7 +1,7 @@
 name = "ClimaCalibrate"
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
 authors = ["Climate Modeling Alliance"]
-version = "0.0.3"
+version = "0.0.4"
 
 [deps]
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
@@ -22,8 +22,8 @@ CESExt = "CalibrateEmulateSample"
 CalibrateEmulateSample = "0.5"
 Distributed = "1"
 Distributions = "0.25"
-EnsembleKalmanProcesses = "1"
-JLD2 = "0.4"
+EnsembleKalmanProcesses = "1, 2"
+JLD2 = "0.4, 0.5"
 Random = "1"
 TOML = "1"
 YAML = "0.4"

docs/Project.toml

@@ -1,8 +1,6 @@
 [deps]
 ClimaCalibrate = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
-CalibrateEmulateSample = "95e48a1f-0bec-4818-9538-3db4340308e3"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244"
 EnsembleKalmanProcesses = "aa8a2aa5-91d8-4396-bcef-d4f2ec43552d"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-

experiments/surface_fluxes_perfect_model/Manifest.toml

@@ -183,10 +183,10 @@ weakdeps = ["SparseArrays"]
     ChainRulesCoreSparseArraysExt = "SparseArrays"
 
 [[deps.ClimaCalibrate]]
-deps = ["Distributions", "EnsembleKalmanProcesses", "JLD2", "Random", "TOML", "YAML"]
+deps = ["Distributed", "Distributions", "EnsembleKalmanProcesses", "JLD2", "Random", "TOML", "YAML"]
 path = "../.."
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
-version = "0.0.2"
+version = "0.0.4"
 
     [deps.ClimaCalibrate.extensions]
     CESExt = "CalibrateEmulateSample"

test/Project.toml

@@ -3,6 +3,7 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 CalibrateEmulateSample = "95e48a1f-0bec-4818-9538-3db4340308e3"
 ClimaCalibrate = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
 ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
+Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 EnsembleKalmanProcesses = "aa8a2aa5-91d8-4396-bcef-d4f2ec43552d"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
@@ -11,6 +12,3 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[compat]
-EnsembleKalmanProcesses = "< 1.1.6"

test/emulate_sample.jl

@@ -13,23 +13,24 @@ import ClimaCalibrate as CAL
 @testset "Emulate and Sample tests" begin
     y_obs = [261.5493]
     y_noise_cov = [0.02619;;]
-    ekp = JLD2.load_object(joinpath("test_case_inputs", "eki_test.jld2"))
-    init_params = [EKP.get_u_final(ekp)[1]]
-
+    init_params = [4.1928479297330306]
     prior_path = joinpath("test_case_inputs", "sphere_hs_rhoe.toml")
-
     prior = CAL.get_prior(prior_path)
 
-    input_output_pairs = CAL.get_input_output_pairs(ekp)
-
-    @test input_output_pairs.inputs.stored_data ==
-          hcat([ekp.u[i].stored_data for i in 1:(length(ekp.u) - 1)]...)
-    @test input_output_pairs.outputs.stored_data ==
-          hcat([ekp.g[i].stored_data for i in 1:length(ekp.g)]...)
+    # Sample data
+    input_output_pairs = EKP.DataContainers.PairedDataContainer(
+        # Parameter values
+        EKP.DataContainers.DataContainer(
+            [5.05038266176406 4.49801357065531 4.625175276903062 4.497648108467572 5.049462857273453 5.470189766650297 4.945930038719361 4.694723166004012 4.936413547464582 4.618148508404665 4.041965675651152 4.121910454258125 4.077521148463588 4.119541027138994 4.043163793654862 4.21813038496636 4.0249164182511485 4.0292005257357895 4.026103892229238 4.0880286181841585 4.1861745817794205 4.20048531589655 4.187992300461601 4.19870114405988 4.209232786450678 4.191675948737242 4.187239734960614 4.186875241183792 4.189230960868622 4.19193208121736],
+        ),
+        # Observation map output values
+        EKP.DataContainers.DataContainer(
+            [229.28858947753906 249.54794311523438 244.09906005859375 249.78701782226562 229.31289672851562 221.62457275390625 232.126953125 240.37799072265625 232.520751953125 244.3943634033203 266.1481628417969 263.6910705566406 265.0572509765625 263.7556457519531 266.0339660644531 260.7401428222656 266.4653015136719 266.42529296875 266.4148864746094 264.5828552246094 261.7020568847656 261.2137451171875 261.7446594238281 261.2926025390625 260.7995910644531 261.4737548828125 261.597900390625 261.69354248046875 261.52777099609375 261.3088684082031],
+        ),
+    )
 
     emulator = CAL.gp_emulator(input_output_pairs, y_noise_cov)
 
-
     (; mcmc, chain) = CAL.sample(emulator, y_obs, prior, init_params)
     @test mean(chain.value[1:100000]) ≈ 4.19035299 rtol = 0.0001
 

test/pure_julia_e2e.jl

@@ -19,7 +19,7 @@ import JLD2
 output_file = "model_output.jld2"
 prior = constrained_gaussian("test_param", 10, 5, 0, Inf)
 n_iterations = 1
-ensemble_size = 10
+ensemble_size = 20
 observations = [20.0]
 noise = [0.01;;]
 output_dir = joinpath("test", "e2e_test_output")
@@ -34,6 +34,8 @@ experiment_config = ExperimentConfig(
 )
 
 # Model interface
+# This "model" just samples parameters and returns them, we are checking that the 
+# results are reproducible.
 function set_up_forward_model(
     member,
     iteration,
@@ -55,9 +57,7 @@ function run_forward_model(config)
     JLD2.save_object(joinpath(config["output_dir"], output_file), output)
 end
 
-# Observation map
 function observation_map(iteration)
-
     (; ensemble_size) = experiment_config
     dims = 1
     G_ensemble = Array{Float64}(undef, dims..., ensemble_size)
@@ -76,8 +76,8 @@ ekp = calibrate(JuliaBackend, experiment_config)
 @testset "Test end-to-end calibration" begin
     parameter_values =
         [EKP.get_ϕ_mean(prior, ekp, it) for it in 1:(n_iterations + 1)]
-    @test parameter_values[1][1] ≈ 9.779 rtol = 0.01
-    @test parameter_values[end][1] ≈ 19.63 rtol = 0.01
+    @test parameter_values[1][1] ≈ 9.215 rtol = 0.01
+    @test parameter_values[end][1] ≈ 20.224 rtol = 0.01
 end
 
 rm(output_dir; recursive = true)