From 8e0db6434510ed823bbb8c2bdd8b7469e723043e Mon Sep 17 00:00:00 2001 From: nefrathenrici Date: Tue, 14 May 2024 13:00:48 -0700 Subject: [PATCH] Remove experiment ID Update tests Fixes after rebase test fixes fix default output w --- docs/src/api.md | 3 +- docs/src/atmos_setup_guide.md | 9 ++-- docs/src/emulate_sample.md | 3 +- docs/src/precompilation.md | 6 +-- .../generate_data.jl | 6 +-- .../model_interface.jl | 31 +++-------- .../observation_map.jl | 2 +- .../postprocessing.jl | 10 +--- src/backends.jl | 12 ++--- src/ekp_interface.jl | 7 +-- src/model_interface.jl | 53 +++++-------------- src/slurm.jl | 16 +++--- test/ekp_interface.jl | 1 - test/model_interface.jl | 47 +++++----------- test/pure_julia_e2e.jl | 24 +++------ test/slurm_unit_tests.jl | 6 +-- 16 files changed, 68 insertions(+), 168 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 9df02173..71daebb0 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -3,9 +3,8 @@ ## Model Interface ```@docs -ClimaCalibrate.get_config +ClimaCalibrate.set_up_forward_model ClimaCalibrate.run_forward_model -ClimaCalibrate.get_forward_model ClimaCalibrate.observation_map ``` diff --git a/docs/src/atmos_setup_guide.md b/docs/src/atmos_setup_guide.md index 07edd0cd..8db8a591 100644 --- a/docs/src/atmos_setup_guide.md +++ b/docs/src/atmos_setup_guide.md @@ -16,9 +16,9 @@ To calibrate parameters, you need: - Truth and noise data - Observation map script with a function `observation_map(iteration)` -These components are detailed in the guide below. Examples of all of these can also be found in `experiments/sphere_held_suarez_rhoe_equilmoist` +These components are detailed in the guide below. Examples of all of these can also be found in ClimaAtmos in `calibration/experiments/sphere_held_suarez_rhoe_equilmoist` -First, create a folder for your experiment with a descriptive experiment id in the `experiments/` folder. All of the components described below will be stored in this folder. +First, create a folder for your experiment with a descriptive name in the `calibration/experiments/` folder. All of the components described below will be stored in this folder. ## Atmos Configuration File @@ -96,8 +96,7 @@ Constraint constructors: The observation map is applied to process model output diagnostics into the exact observable used to fit to observations. In a perfect model setting it is used also to generate the observation. -Your observation map file must export a function `observation_map(::Val{:}, iteration)`, this function is specific to each experiment, so it is dispatched on the `experiment_id`. -These requirements arise from the update step, which runs the function with your given experiment ID. +These requirements arise from the update step, which runs the `observation_map` function. This function must load in model diagnostics for each ensemble member in the iteration and construct an array `arr = Array{Float64}(undef, dims..., ensemble_size)` such that `arr[:, i]` will return the i-th ensemble member's observation map output. Note this floating point precision is required for the EKI update step. @@ -113,7 +112,7 @@ First, construct the array to store the ensemble's observations. Then, for each Pseudocode for `observation_map(iteration)`: ```julia -function observation_map(::Val{:sphere_held_suarez_rhoe_equilmoist}, iteration) +function observation_map(iteration) # Get Configuration config_file = joinpath("calibration", "sphere_held_suarez_rhoe_equilmoist") config = ExperimentConfig(config_file) diff --git a/docs/src/emulate_sample.md b/docs/src/emulate_sample.md index 4e04d7b0..aa2f7a62 100644 --- a/docs/src/emulate_sample.md +++ b/docs/src/emulate_sample.md @@ -16,7 +16,8 @@ import ClimaCalibrate as CAL ``` Next, load in the data, EKP object, and prior distribution. These values are taken -from the perfect model experiment with experiment ID `sphere_held_suarez_rhoe_equilmoist`. +from the Held-Suarez perfect model experiment in ClimaAtmos. + ```julia asset_path = joinpath( pkgdir(CAL), diff --git a/docs/src/precompilation.md b/docs/src/precompilation.md index 854d169a..f350c34c 100644 --- a/docs/src/precompilation.md +++ b/docs/src/precompilation.md @@ -7,7 +7,7 @@ For ClimaCalibrate, this is useful under certain conditions: 2. **The model runtime is short compared to the compile time.** If the model runtime is an order of magnitude or more than the compilation time, any benefit from reduced compilation time will be trivial. # How do I precompile my configuration? -The easiest way is by copying and pasting the code snippet below into `src/ClimaCalibrate.jl` and replacing the `job_id` with your experiment ID. +The easiest way is by copying and pasting the code snippet below into `src/ClimaCalibrate.jl`. This will precompile the model step and all callbacks for the given configuration. ```julia using PrecompileTools @@ -16,10 +16,8 @@ import ClimaAtmos as CA import YAML @setup_workload begin - config_file = "your experiment dir" - ExperimentConfig(config_file; output_dir = "precompilation") + config_file = Dict("FLOAT_TYPE" => "Float64") @compile_workload begin - initialize(job_id) config = CA.AtmosConfig(config_dict) simulation = CA.get_simulation(config) (; integrator) = simulation diff --git a/experiments/surface_fluxes_perfect_model/generate_data.jl b/experiments/surface_fluxes_perfect_model/generate_data.jl index 512b0cc9..4b7ccc17 100644 --- a/experiments/surface_fluxes_perfect_model/generate_data.jl +++ b/experiments/surface_fluxes_perfect_model/generate_data.jl @@ -1,5 +1,4 @@ # generate_data: generate true y, noise and x_inputs -experiment_id = "surface_fluxes_perfect_model" import SurfaceFluxes as SF import SurfaceFluxes.Parameters as SFPP @@ -10,8 +9,9 @@ import SurfaceFluxes.Parameters: SurfaceFluxesParameters using ClimaCalibrate pkg_dir = pkgdir(ClimaCalibrate) -experiment_path = "$pkg_dir/experiments/$experiment_id" -data_path = "$experiment_path/data" +experiment_path = + joinpath(pkg_dir, "experiments", "surface_fluxes_perfect_model") +data_path = joinpath(experiment_path, "data") include(joinpath(experiment_path, "model_interface.jl")) FT = Float32 diff --git a/experiments/surface_fluxes_perfect_model/model_interface.jl b/experiments/surface_fluxes_perfect_model/model_interface.jl index 0b77cead..6c9806f1 100644 --- a/experiments/surface_fluxes_perfect_model/model_interface.jl +++ b/experiments/surface_fluxes_perfect_model/model_interface.jl @@ -1,11 +1,6 @@ import EnsembleKalmanProcesses as EKP using ClimaCalibrate -import ClimaCalibrate: - AbstractPhysicalModel, - get_config, - run_forward_model, - get_forward_model, - ExperimentConfig +import ClimaCalibrate: set_up_forward_model, run_forward_model, ExperimentConfig import YAML """ @@ -32,7 +27,6 @@ We need to follow the following steps for the calibration: 4. define the prior distributions for θ (this is subjective and can be based on expert knowledge or previous studies) """ -struct SurfaceFluxModel <: AbstractPhysicalModel end experiment_dir = joinpath( pkgdir(ClimaCalibrate), @@ -42,18 +36,8 @@ experiment_dir = joinpath( include(joinpath(experiment_dir, "sf_model.jl")) include(joinpath(experiment_dir, "observation_map.jl")) -function get_forward_model(::Val{:surface_fluxes_perfect_model}) - return SurfaceFluxModel() -end - -function get_config( - model::SurfaceFluxModel, - member, - iteration, - experiment_dir::AbstractString, -) - return get_config( - model, +function set_up_forward_model(member, iteration, experiment_dir::AbstractString) + return set_up_forward_model( member, iteration, ExperimentConfig(experiment_dir), @@ -61,15 +45,14 @@ function get_config( end """ - get_config(member, iteration, experiment_dir::AbstractString) - get_config(member, iteration, experiment_config::ExperimentConfig) + set_up_forward_model(member, iteration, experiment_dir::AbstractString) + set_up_forward_model(member, iteration, experiment_config::ExperimentConfig) Returns an config dictionary object for the given member and iteration. Given an experiment dir, it will load the ExperimentConfig This assumes that the config dictionary has the `output_dir` key. """ -function get_config( - ::SurfaceFluxModel, +function set_up_forward_model( member, iteration, experiment_config::ExperimentConfig, @@ -103,7 +86,7 @@ end Runs the model with the given an AbstractDict object. """ -function run_forward_model(::SurfaceFluxModel, config::AbstractDict) +function run_forward_model(config::AbstractDict) x_inputs = load_profiles(config["x_data_file"]) FT = typeof(x_inputs.profiles_int[1].T) obtain_ustar(FT, x_inputs, config) diff --git a/experiments/surface_fluxes_perfect_model/observation_map.jl b/experiments/surface_fluxes_perfect_model/observation_map.jl index 5b49650d..6eda0070 100644 --- a/experiments/surface_fluxes_perfect_model/observation_map.jl +++ b/experiments/surface_fluxes_perfect_model/observation_map.jl @@ -16,7 +16,7 @@ experiment_dir = joinpath( Returns the observation map (from the raw model output to the observable y), as specified by process_member_data, for the given iteration. """ -function observation_map(::Val{:surface_fluxes_perfect_model}, iteration) +function observation_map(iteration) config = ExperimentConfig(experiment_dir) (; output_dir, ensemble_size) = config model_output = "model_ustar_array.jld2" diff --git a/experiments/surface_fluxes_perfect_model/postprocessing.jl b/experiments/surface_fluxes_perfect_model/postprocessing.jl index a019beb7..d5b0ba19 100644 --- a/experiments/surface_fluxes_perfect_model/postprocessing.jl +++ b/experiments/surface_fluxes_perfect_model/postprocessing.jl @@ -14,7 +14,6 @@ using ClimaCalibrate experiment_dir = dirname(Base.active_project()) experiment_config = ClimaCalibrate.ExperimentConfig(experiment_dir) output_dir = experiment_config.output_dir -experiment_id = experiment_config.id N_iter = experiment_config.n_iterations N_mem = experiment_config.ensemble_size @@ -103,9 +102,7 @@ end pkg_dir = pkgdir(ClimaCalibrate) -model_config = YAML.load_file( - joinpath(pkg_dir, "experiments", experiment_id, "model_config.yml"), -) +model_config = YAML.load_file(joinpath(experiment_dir, "model_config.yml")) eki_path = joinpath( ClimaCalibrate.path_to_iteration(output_dir, N_iter), @@ -133,10 +130,7 @@ include(joinpath(experiment_dir, "model_interface.jl")) f = Makie.Figure() ax = Makie.Axis(f[1, 1], xlabel = "Iteration", ylabel = "Model Ustar") ustar_obs = JLD2.load_object( - joinpath( - pkg_dir, - "experiments/$experiment_id/data/synthetic_ustar_array_noisy.jld2", - ), + joinpath(pkg_dir, "$experiment_dir/data/synthetic_ustar_array_noisy.jld2"), ) x_inputs = load_profiles(model_config["x_data_file"]) diff --git a/src/backends.jl b/src/backends.jl index 486084de..028fbc24 100644 --- a/src/backends.jl +++ b/src/backends.jl @@ -47,19 +47,15 @@ calibrate(b::Type{JuliaBackend}, experiment_dir::AbstractString) = function calibrate(::Type{JuliaBackend}, config::ExperimentConfig) initialize(config) - (; n_iterations, id, ensemble_size) = config + (; n_iterations, ensemble_size) = config eki = nothing - physical_model = get_forward_model(Val(Symbol(id))) for i in 0:(n_iterations - 1) @info "Running iteration $i" for m in 1:ensemble_size - run_forward_model( - physical_model, - get_config(physical_model, m, i, config), - ) + run_forward_model(set_up_forward_model(m, i, config)) @info "Completed member $m" end - G_ensemble = observation_map(Val(Symbol(id)), i) + G_ensemble = observation_map(i) save_G_ensemble(config, i, G_ensemble) eki = update_ensemble(config, i) end @@ -150,7 +146,7 @@ function calibrate( ) report_iteration_status(statuses, output_dir, iter) @info "Completed iteration $iter, updating ensemble" - G_ensemble = observation_map(Val(Symbol(config.id)), iter) + G_ensemble = observation_map(iter) save_G_ensemble(config, iter, G_ensemble) eki = update_ensemble(config, iter) end diff --git a/src/ekp_interface.jl b/src/ekp_interface.jl index 51c486e0..3036464f 100644 --- a/src/ekp_interface.jl +++ b/src/ekp_interface.jl @@ -16,7 +16,6 @@ ExperimentConfig holds the configuration for a calibration experiment. This can be constructed from a YAML configuration file or directly using individual parameters. """ struct ExperimentConfig - id::AbstractString n_iterations::Integer ensemble_size::Integer observations::Any @@ -40,10 +39,7 @@ function ExperimentConfig(filepath::AbstractString; kwargs...) error("Invalid experiment configuration filepath: `$filepath`") end - experiment_id = - get(config_dict, "experiment_id", last(splitdir(experiment_dir))) - default_output = - haskey(ENV, "CI") ? experiment_id : joinpath("output", experiment_id) + default_output = joinpath(experiment_dir, "output") output_dir = get(config_dict, "output_dir", default_output) n_iterations = config_dict["n_iterations"] @@ -65,7 +61,6 @@ function ExperimentConfig(filepath::AbstractString; kwargs...) prior = get_prior(prior_path) return ExperimentConfig( - experiment_id, n_iterations, ensemble_size, observations, diff --git a/src/model_interface.jl b/src/model_interface.jl index b2c49a6f..703a4594 100644 --- a/src/model_interface.jl +++ b/src/model_interface.jl @@ -1,57 +1,30 @@ import EnsembleKalmanProcesses as EKP import YAML -""" - AbstractPhysicalModel -Abstract type to define the interface for physical models. """ -abstract type AbstractPhysicalModel end + set_up_forward_model(member, iteration, experiment_dir::AbstractString) + set_up_forward_model(member, iteration, experiment_config::ExperimentConfig) -""" - get_config(physical_model::AbstractPhysicalModel, member, iteration, experiment_path::AbstractString) - get_config(physical_model::AbstractPhysicalModel, member, iteration, experiment_config) +Set up and configure a single member's forward model. Used in conjunction with `run_forward_model`. -Fetch the model information for a specific ensemble member and iteration based on a provided path. This function must be overriden by a component's model interface and should set things like the parameter path and other member-specific settings. """ -function get_config( - physical_model::AbstractPhysicalModel, - member, - iteration, - experiment_path::AbstractString, -) - experiment_config = ExperimentConfig(experiment_path) - return get_config(physical_model, member, iteration, experiment_config) -end +set_up_forward_model(member, iteration, experiment_dir::AbstractString) = + set_up_forward_model(member, iteration, ExperimentConfig(experiment_dir)) -get_config( - physical_model::AbstractPhysicalModel, - member, - iteration, - experiment_config, -) = error("get_config not implemented for $physical_model") +set_up_forward_model(member, iteration, experiment_config::ExperimentConfig) = + error("set_up_forward_model not implemented") """ - run_forward_model(physical_model::AbstractPhysicalModel, config) + run_forward_model(config) Executes the forward model simulation with the given configuration. -The `config` should be obtained from `get_config`. +`config` should be obtained from `set_up_forward_model`. This function should be overridden with model-specific implementation details. """ -run_forward_model(physical_model::AbstractPhysicalModel, model_config) = - error("run_forward_model not implemented for $physical_model") - -""" - get_forward_model(experiment_id::Val) - -Retrieves a custom physical model struct for the specified experiment ID. -Throws an error if the experiment ID is unrecognized. -""" -function get_forward_model(experiment_id::Val) - error("get_forward_model not implemented for $experiment_id") -end +run_forward_model(model_config) = error("run_forward_model not implemented") """ observation_map(val:Val, iteration) @@ -59,8 +32,6 @@ end Runs the observation map for the specified iteration. This function must be implemented for each calibration experiment. """ -function observation_map(val::Val, iteration) - error( - "observation_map not implemented for experiment $val at iteration $iteration", - ) +function observation_map(iteration) + error("observation_map not implemented") end diff --git a/src/slurm.jl b/src/slurm.jl index 00b4e91e..12ff63df 100644 --- a/src/slurm.jl +++ b/src/slurm.jl @@ -2,7 +2,7 @@ kwargs(; kwargs...) = Dict{Symbol, Any}(kwargs...) """ -generate_sbatch_script + generate_sbatch_script """ @@ -47,11 +47,7 @@ function generate_sbatch_script( model_interface = "$model_interface"; include(model_interface) experiment_dir = "$experiment_dir" - experiment_config = CAL.ExperimentConfig(experiment_dir) - experiment_id = experiment_config.id - physical_model = CAL.get_forward_model(Val(Symbol(experiment_id))) - CAL.run_forward_model(physical_model, CAL.get_config(physical_model, member, iteration, experiment_dir)) - @info "Forward Model Run Completed" experiment_id physical_model iteration member' + CAL.run_forward_model(CAL.set_up_forward_model(member, iteration, experiment_dir))' """ return sbatch_contents end @@ -157,7 +153,8 @@ If verbose, includes the ensemble member's output. """ function log_member_error(output_dir, iteration, member, verbose = false) member_log = path_to_model_log(output_dir, iteration, member) - warn_str = "Ensemble member $member raised an error. See model log at $(abspath(member_log)) for stacktrace" + warn_str = """Ensemble member $member raised an error. See model log at \ + $(abspath(member_log)) for stacktrace""" if verbose stacktrace = replace(readchomp(member_log), "\\n" => "\n") warn_str = warn_str * ": \n$stacktrace" @@ -167,9 +164,11 @@ end function report_iteration_status(statuses, output_dir, iter) all(job_completed.(statuses)) || error("Some jobs are not complete") + if all(job_failed, statuses) error( - "Full ensemble for iteration $iter has failed. See model logs in $(abspath(path_to_iteration(output_dir, iter))) for details.", + """Full ensemble for iteration $iter has failed. See model logs in + $(abspath(path_to_iteration(output_dir, iter)))""", ) elseif any(job_failed, statuses) @warn "Failed ensemble members: $(findall(job_failed, statuses))" @@ -246,5 +245,4 @@ function format_slurm_time(minutes::Int) ) end end - format_slurm_time(str::AbstractString) = str diff --git a/test/ekp_interface.jl b/test/ekp_interface.jl index 0f694454..b117a452 100644 --- a/test/ekp_interface.jl +++ b/test/ekp_interface.jl @@ -17,7 +17,6 @@ n_iterations = 1 ensemble_size = 10 config = CAL.ExperimentConfig( - "test", n_iterations, ensemble_size, observations, diff --git a/test/model_interface.jl b/test/model_interface.jl index 94e77cb5..3378d6c4 100644 --- a/test/model_interface.jl +++ b/test/model_interface.jl @@ -6,49 +6,30 @@ using Test # Tests for ensuring ClimaCalibrate has protected interfaces. The API tested below must be defined for each model, # otherwise ClimaCalibrate will throw an error. -struct TestPhysicalModel <: ClimaCalibrate.AbstractPhysicalModel end - @testset "Model Interface stubs" begin - @testset "get_config" begin - @test_throws ErrorException( - "get_config not implemented for TestPhysicalModel()", - ) ClimaCalibrate.get_config(TestPhysicalModel(), 1, 1, Dict{Any, Any}()) + @testset "set_up_forward_model" begin + experiment_dir = joinpath( + abspath(pkgdir(ClimaCalibrate)), + "experiments", + "surface_fluxes_perfect_model", + ) + @test_throws ErrorException("set_up_forward_model not implemented") ClimaCalibrate.set_up_forward_model( + 1, + 1, + experiment_dir, + ) end @testset "run_forward_model" begin - @test_throws ErrorException( - "run_forward_model not implemented for TestPhysicalModel()", - ) ClimaCalibrate.run_forward_model( - TestPhysicalModel(), - Dict{Any, Any}(), + @test_throws ErrorException("run_forward_model not implemented") ClimaCalibrate.run_forward_model( + nothing, ) end - @testset "get_forward_model" begin - @test_throws ErrorException( - "get_forward_model not implemented for Val{:test}()", - ) ClimaCalibrate.get_forward_model(Val(:test)) - end - @testset "observation_map" begin - @test_throws ErrorException( - "observation_map not implemented for experiment Val{:test}() at iteration 1", - ) ClimaCalibrate.observation_map(Val(:test), 1) - end - - @testset "calibrate func" begin - experiment_config = ClimaCalibrate.ExperimentConfig( - "test", + @test_throws ErrorException("observation_map not implemented") ClimaCalibrate.observation_map( 1, - 1, - [20.0], - [0.01;;], - constrained_gaussian("test_param", 10, 5, 0, Inf), - joinpath("test", "e2e_test_output"), - false, ) - @test_throws ErrorException ClimaCalibrate.calibrate(experiment_config) end - end diff --git a/test/pure_julia_e2e.jl b/test/pure_julia_e2e.jl index cedbb7db..0b5ef18d 100644 --- a/test/pure_julia_e2e.jl +++ b/test/pure_julia_e2e.jl @@ -6,10 +6,9 @@ using EnsembleKalmanProcesses.TOMLInterface import ClimaParams as CP import ClimaCalibrate: - AbstractPhysicalModel, - get_config, run_forward_model, - get_forward_model, + set_up_forward_model, + JuliaBackend, ExperimentConfig, calibrate, observation_map @@ -17,7 +16,6 @@ import ClimaCalibrate: import JLD2 # Experiment Info -id = "e2e_test" output_file = "model_output.jld2" prior = constrained_gaussian("test_param", 10, 5, 0, Inf) n_iterations = 1 @@ -27,7 +25,6 @@ noise = [0.01;;] output_dir = joinpath("test", "e2e_test_output") experiment_config = ExperimentConfig( - id, n_iterations, ensemble_size, observations, @@ -38,14 +35,7 @@ experiment_config = ExperimentConfig( ) # Model interface -struct TestModel end - -TestModel <: AbstractPhysicalModel - -get_forward_model(::Val{:e2e_test}) = TestModel() - -function get_config( - physical_model::TestModel, +function set_up_forward_model( member, iteration, experiment_config::ExperimentConfig, @@ -59,7 +49,7 @@ function get_config( return model_config end -function run_forward_model(::TestModel, config) +function run_forward_model(config) toml_dict = CP.create_toml_dict(Float64; override_file = config["toml"]) (; test_param) = CP.get_parameter_values(toml_dict, "test_param") output = test_param @@ -67,7 +57,7 @@ function run_forward_model(::TestModel, config) end # Observation map -function observation_map(::Val{:e2e_test}, iteration) +function observation_map(iteration) (; ensemble_size) = experiment_config dims = 1 @@ -82,9 +72,9 @@ function observation_map(::Val{:e2e_test}, iteration) end # Test! -ekp = calibrate(experiment_config) +ekp = calibrate(JuliaBackend, experiment_config) -@testset "Test simple end-to-end calibration" begin +@testset "Test end-to-end calibration" begin parameter_values = [EKP.get_ϕ_mean(prior, ekp, it) for it in 1:(n_iterations + 1)] @test parameter_values[1][1] ≈ 9.779 rtol = 0.01 diff --git a/test/slurm_unit_tests.jl b/test/slurm_unit_tests.jl index 715a15fc..b0cf47a1 100644 --- a/test/slurm_unit_tests.jl +++ b/test/slurm_unit_tests.jl @@ -56,11 +56,7 @@ iteration = 1; member = 1 model_interface = "model_interface.jl"; include(model_interface) experiment_dir = "exp/dir" -experiment_config = CAL.ExperimentConfig(experiment_dir) -experiment_id = experiment_config.id -physical_model = CAL.get_forward_model(Val(Symbol(experiment_id))) -CAL.run_forward_model(physical_model, CAL.get_config(physical_model, member, iteration, experiment_dir)) -@info "Forward Model Run Completed" experiment_id physical_model iteration member' +CAL.run_forward_model(CAL.set_up_forward_model(member, iteration, experiment_dir))' """ for (generated_str, test_str) in