diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index b2eff3758..b46ee7fc8 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -327,6 +327,4 @@ steps: - | find experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)" - done - - + done \ No newline at end of file diff --git a/.buildkite/longruns/pipeline_main.yml b/.buildkite/longruns/pipeline_main.yml new file mode 100644 index 000000000..9ca71b213 --- /dev/null +++ b/.buildkite/longruns/pipeline_main.yml @@ -0,0 +1,414 @@ +agents: + queue: new-central + slurm_time: 24:00:00 + modules: climacommon/2024_03_18 + +env: + JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite" + OPENBLAS_NUM_THREADS: 1 + JULIA_NVTX_CALLBACKS: gc + OMPI_MCA_opal_warn_on_missing_libcuda: 0 + JULIA_MAX_NUM_PRECOMPILE_FILES: 100 + GKSwstype: 100 + SLURM_KILL_BAD_EXIT: 1 + + CONFIG_PATH: "config/longrun_configs" + PERF_CONFIG_PATH: "config/perf_configs" + +timeout_in_minutes: 1440 + +steps: + - label: "init :computer:" + key: "init_cpu_env" + command: + - "echo $$JULIA_DEPOT_PATH" + + - echo "--- Instantiate AMIP env" + - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" + - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" + + - echo "--- Download artifacts" + - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'" + - "julia --project=artifacts -e 'using Pkg; Pkg.status()'" + - "julia --project=artifacts artifacts/download_artifacts.jl" + + - echo "--- Instantiate perf env" + - "julia --project=perf/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=perf/ -e 'using Pkg; Pkg.precompile()'" + - "julia --project=perf/ -e 'using Pkg; Pkg.status()'" + + agents: + slurm_cpus_per_task: 8 + env: + JULIA_NUM_PRECOMPILE_TASKS: 8 + JULIA_MAX_NUM_PRECOMPILE_FILES: 50 + + concurrency: 1 + concurrency_group: 'depot/climacoupler-longruns' + + - wait + + # DYAMOND AMIP: 1 day (convection resolving) - running on clima, so need to instantiate again + - label: "GPU AMIP SUPERFINE: dyamond_target" + key: "gpu_dyamond_target" + command: + - "echo $$JULIA_DEPOT_PATH" + + - echo "--- Instantiate AMIP env" + - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" + - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" + + - echo "--- Download artifacts" + - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'" + - "julia --project=artifacts -e 'using Pkg; Pkg.status()'" + - "julia --project=artifacts artifacts/download_artifacts.jl" + + - echo "--- Run simulation" + - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*" + agents: + queue: clima + slurm_mem: 20GB + slurm_gpus: 1 + modules: common + + - group: "Coupler integration and conservation tests" + + steps: + + # Integration tests - the expected results were also confirmed locally + - label: "Slabplanet_aqua: nocouple" + key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: couple" + key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: coupler fluxes" + key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: coupler fluxes, evolving ocean" + key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_terra: coupler fluxes, evolving bucket" + key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_terra.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_terra/slabplanet_terra_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet: coupler fluxes, evolving ocean and land" + key: "slabplanet_coupler_sf_evolve_ocn" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - group: "Current target tests: idealized surface" + + steps: + + - label: "ClimaAtmos standalone target" + command: + - srun julia --project=experiments/AMIP/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml + artifact_paths: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean/*" + env: + BUILD_HISTORY_HANDLE: "" + CLIMACORE_DISTRIBUTED: "MPI" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc" + key: "slabplanet_aqua_target_nocouple" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc" + key: "slabplanet_aqua_target" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T" + key: "slabplanet_aqua_target_evolve_ocn" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket" + key: "slabplanet_target" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target.yml" + artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket" + key: "slabplanet_target_evolve_ocn" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + + - group: "Current target tests: AMIP surface" + + steps: + + - label: "MPI AMIP FINE: new target amip" + key: "amip_target" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: new target amip: topo" + key: "amip_target_topo" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: new target amip: topo + diagedmf" + key: "amip_target_topo_diagedmf" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 20G + + - group: "Current target tests on GPU: AMIP surface" + + steps: + + - label: "GPU AMIP FINE: new target amip: topo" + key: "gpu_amip_target_topo" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_artifacts/*" + agents: + slurm_gpus: 1 + slurm_mem: 16GB + + - label: "GPU AMIP FINE: new target amip: topo + diagedmf" + key: "gpu_amip_target_topo_diagedmf" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*" + agents: + slurm_gpus: 1 + slurm_mem: 16GB + + - group: "Other AMIP targets" + + steps: + + # DYAMOND AMIP: 1 day (convection resolving) + - label: "MPI AMIP SUPERFINE: dyamond_target" + key: "dyamond_target" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/dyamond_target.yml" + artifact_paths: "experiments/AMIP/output/amip/dyamond_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + # mid-resolution AMIP: MPI performance scaling (10 days) + - label: "MPI AMIP FINE: n64" + key: "mpi_amip_fine_n64" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n64_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_n64_shortrun_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: n32" + key: "mpi_amip_fine_n32" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n32_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_n32_shortrun_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 8 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: n8" + key: "mpi_amip_fine_n8" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n8_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_n8_shortrun_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 8 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: n2" # 10d take 21h, so reducing to 1d + key: "mpi_amip_fine_n2" + command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n2_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_n2_shortrun_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 2 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: n1" # also reported by longruns with a flame graph; 10d take 21h, so reducing to 1d + key: "mpi_amip_fine_n1" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n1_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_n1_shortrun_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: n1 no couple" # sim time = Δt_cpl (~ benchmarking with standalone models) + key: "mpi_amip_fine_n1_nocouple" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n1_shortrun_nocouple.yml" + artifact_paths: "experiments/AMIP/output/amip/amip_n1_shortrun_nocouple_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + # mpi_amip_fine_n1 flame graph report (NB: arguments passed from the ci pipeline.yml) + - label: ":rocket: performance: flame graph diff: perf_target_amip_n1_shortrun" + command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_target_amip_n1_shortrun.yml" + artifact_paths: "perf/output/perf_diff_target_amip_n1_shortrun/*" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - wait + + # plot job performance history + - label: ":chart_with_downwards_trend: build history" + command: + - build_history main # name of branch to plot + artifact_paths: + - "build_history.html" + + - wait + + - label: ":envelope: Slack report: build_history" + command: + - slack-upload -c "#coupler-report" -f build_history.html -m html -n build_history -x "Overall job performance" + + - label: ":envelope: Slack report: Slabplanet" + command: + - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" + + - label: ":envelope: Slack report: target AMIP" + command: + - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" + - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_artifacts/biases.png -m png -n amip_fine -x "AMIP Target Longrun" + + - label: ":envelope: Slack report: Flame Diff" + command: + - slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff.html -m png -n amip_fine_flamegraphdiff -x "AMIP Longrun FlameGraphDiff" + - slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff_self_count.html -m png -n amip_fine_flamegraphdiffself -x "AMIP Longrun FlameGraphDiffSelf" diff --git a/experiments/AMIP/Manifest.toml b/experiments/AMIP/Manifest.toml index b5fd7d0fc..513c5a943 100644 --- a/experiments/AMIP/Manifest.toml +++ b/experiments/AMIP/Manifest.toml @@ -3291,4 +3291,4 @@ version = "3.5.0+0" deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] git-tree-sha1 = "9c304562909ab2bab0262639bd4f444d7bc2be37" uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" -version = "1.4.1+1" +version = "1.4.1+1" \ No newline at end of file diff --git a/experiments/AMIP/Project.toml b/experiments/AMIP/Project.toml index 3f9da3022..7ef62b409 100644 --- a/experiments/AMIP/Project.toml +++ b/experiments/AMIP/Project.toml @@ -7,6 +7,7 @@ ClimaAnalysis = "29b5916a-a76c-4e73-9657-3c8fd22e65e6" ClimaAtmos = "b2c96348-7fb7-4fe0-8da9-78d88439e717" ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" +ClimaCoreMakie = "908f55d8-4145-4867-9c14-5dad1a479e4d" ClimaCorePlots = "cf7c7e5a-b407-4c48-9047-11a94a308626" ClimaCoreSpectra = "c2caaa1d-32ae-4754-ba0d-80e7561362e9" ClimaCoreTempestRemap = "d934ef94-cdd4-4710-83d6-720549644b70" diff --git a/experiments/AMIP/components/atmosphere/climaatmos.jl b/experiments/AMIP/components/atmosphere/climaatmos.jl index c5449875f..8153cabc3 100644 --- a/experiments/AMIP/components/atmosphere/climaatmos.jl +++ b/experiments/AMIP/components/atmosphere/climaatmos.jl @@ -21,10 +21,7 @@ struct ClimaAtmosSimulation{P, Y, D, I} <: Interfacer.AtmosModelSimulation end Interfacer.name(::ClimaAtmosSimulation) = "ClimaAtmosSimulation" -function atmos_init(::Type{FT}, atmos_config_dict::Dict) where {FT} - # By passing `parsed_args` to `AtmosConfig`, `parsed_args` overwrites the default atmos config - atmos_config_dict["surface_albedo"] = "CouplerAlbedo" - atmos_config = CA.AtmosConfig(atmos_config_dict) +function atmos_init(::Type{FT}, atmos_config) where {FT} simulation = CA.get_simulation(atmos_config) (; integrator) = simulation Y = integrator.u @@ -235,12 +232,12 @@ FluxCalculator.get_surface_params(sim::ClimaAtmosSimulation) = CAP.surface_fluxe ### ClimaAtmos.jl model-specific functions (not explicitly required by ClimaCoupler.jl) ### """ - get_atmos_config(coupler_dict::Dict) + get_atmos_config_dict(coupler_dict::Dict) Returns the specified atmospheric configuration (`atmos_config_dict`) overwitten by arguments in the coupler dictionary (`config_dict`). """ -function get_atmos_config(coupler_dict) +function get_atmos_config_dict(coupler_dict) atmos_config_file = coupler_dict["atmos_config_file"] # override default or specified configs with coupler arguments, and set the correct atmos config_file if isnothing(atmos_config_file) @@ -416,4 +413,4 @@ function FluxCalculator.water_albedo_from_atmosphere!( direct_albedo .= CA.surface_albedo_direct(α_model).(λ, μ, LinearAlgebra.norm.(CC.Fields.level(Y.c.uₕ, 1))) diffuse_albedo .= CA.surface_albedo_diffuse(α_model).(λ, μ, LinearAlgebra.norm.(CC.Fields.level(Y.c.uₕ, 1))) -end +end \ No newline at end of file diff --git a/experiments/AMIP/components/atmosphere/climaatmos_extra_diags.jl b/experiments/AMIP/components/atmosphere/climaatmos_extra_diags.jl index b6a300b33..78c58dfdc 100644 --- a/experiments/AMIP/components/atmosphere/climaatmos_extra_diags.jl +++ b/experiments/AMIP/components/atmosphere/climaatmos_extra_diags.jl @@ -80,3 +80,108 @@ CAD.add_diagnostic_variable!( end, ) + +### +# Add the mass streamfunction diagnostic variable +### + +# TODO: this should be integrated from top-down +add_diagnostic_variable!( + short_name = "mass_streamfunction", + long_name = "Meridional Mass Streamfunction: vertical integral of meridional mass flux, rho v", + standard_name = "meridional_mass_streamfunction", + units = "kg m^-1 s^-1", + compute! = (out, state, cache, time) -> begin + ρv = ClimaCore.Geometry.UVVector.(state.c.uₕ).components.data.:2 .* state.c.ρ + ᶠ∫_0_z_ρv = cache.scratch.ᶠtemp_scalar + ClimaCore.Fields.bycolumn(axes(ρv)) do colidx + ClimaCore.Operators.column_integral_indefinite!(ᶠ∫_0_z_ρv[colidx], ρv[colidx]) + end + if isnothing(out) + return copy(ᶠ∫_0_z_ρv) + else + out .= ᶠ∫_0_z_ρv + end + end, +) + +add_diagnostic_variable!( + short_name = "stab", + long_name = "Static Stability: N^2 = -g/theta dtheta/dz", + standard_name = "static_stability", + units = "s^-2", + compute! = (out, state, cache, time) -> begin + dθdz = ClimaCore.Geometry.WVector.(cache.precomputed.ᶜgradᵥ_θ_virt) + thermo_params = CAP.thermodynamics_params(cache.params) + ᶜts = cache.precomputed.ᶜts + θ_virt = TD.virtual_pottemp.(thermo_params, ᶜts) + temp = cache.scratch.ᶜtemp_scalar + parent(temp) .= parent(CAP.grav(cache.params) ./ θ_virt .* dθdz) + if isnothing(out) + return copy(temp) + else + out .= temp + end + end, +) + +add_diagnostic_variable!( + short_name = "vT", + long_name = "Product of meridional wind and temperature", + standard_name = "vT", + units = "K m/s", + compute! = (out, state, cache, time) -> begin + thermo_params = CAP.thermodynamics_params(cache.params) + T = TD.air_temperature.(thermo_params, cache.precomputed.ᶜts) + v = ClimaCore.Geometry.UVVector.(state.c.uₕ).components.data.:2 + if isnothing(out) + return v .* T + else + out .= v .* T + end + end, +) + +const ᶜgradᵥ_ = ClimaCore.Operators.GradientF2C() +const ᶠinterp_ = ClimaCore.Operators.InterpolateC2F( + bottom = ClimaCore.Operators.Extrapolate(), + top = ClimaCore.Operators.Extrapolate(), +) + +add_diagnostic_variable!( + short_name = "egr", + long_name = "max. Eady growth rate (0.31 f/N du/dz)", + standard_name = "Eady_growth_rate", + units = "s^-1", + compute! = (out, state, cache, time) -> begin + FT = eltype(state) + thermo_params = CAP.thermodynamics_params(cache.params) + dθdz = cache.precomputed.ᶜgradᵥ_θ_virt + ᶜts = cache.precomputed.ᶜts + θ_virt = TD.virtual_pottemp.(thermo_params, ᶜts) + N = cache.scratch.ᶜtemp_scalar + Nsq = CAP.grav(cache.params) ./ θ_virt .* ClimaCore.Geometry.WVector.(dθdz) + mask = parent(Nsq) .> 0 + parent(N) .= sqrt.((parent(Nsq) .* mask )) + + Ω = CAP.Omega(cache.params) + φ = ClimaCore.Fields.coordinate_field(N).lat + f = 2Ω .* sin.(φ) + u = ClimaCore.Geometry.UVVector.(state.c.uₕ).components.data.:1 + grad_u = @. ᶜgradᵥ_.(ᶠinterp_(u)) + du_dz = ClimaCore.Geometry.WVector.(grad_u) + temp = cache.scratch.ᶜtemp_scalar + parent(temp) .= parent(FT(0.31) .* f ./ N .* du_dz) + if isnothing(out) + return copy(temp) + else + out .= temp + end + end, +) + +# climate diagnostics: ta, ua, (hus), *mass_streamfunction, *stab, mse + +# storm track diagnostics: [vT], [v][T], egr = f/N dTdy + +# vars = [mse, lr, ediff, mass_streamfunction, stab, vT, egr] \ No newline at end of file diff --git a/experiments/AMIP/driver_utils.jl b/experiments/AMIP/driver_utils.jl new file mode 100644 index 000000000..2b77c5d43 --- /dev/null +++ b/experiments/AMIP/driver_utils.jl @@ -0,0 +1,25 @@ +# useful functions to abstract the driver code + +function setup_output_dirs(; output_dir = nothing, regrid_dir = nothing, artifacts_dir = nothing, comms_ctx = nothing) + if output_dir === nothing + output_dir = "." + end + if regrid_dir === nothing + regrid_dir = joinpath(output_dir, "regrid_tmp/") + end + if artifacts_dir === nothing + artifacts_dir = output_dir * "_artifacts" + end + + if !isnothing(comms_ctx) && ClimaComms.iamroot(comms_ctx) + @info(output_dir) + mkpath(output_dir) + mkpath(regrid_dir) + mkpath(artifacts_dir) + end + + !isnothing(comms_ctx) ? ClimaComms.barrier(comms_ctx) : nothing + + return (; output = output_dir, artifacts = artifacts_dir, regrid = regrid_dir) + +end diff --git a/experiments/AMIP/dry_held_suarez.jl b/experiments/AMIP/dry_held_suarez.jl new file mode 100644 index 000000000..420933d21 --- /dev/null +++ b/experiments/AMIP/dry_held_suarez.jl @@ -0,0 +1,383 @@ +# # Dry Held-Suarez + +redirect_stderr(IOContext(stderr, :stacktrace_types_limited => Ref(false))) + +#= +## Configuration initialization +Here we import standard Julia packages, ClimaESM packages, parse in command-line arguments (if none are specified then the defaults in `cli_options.jl` apply). +We then specify the input data file names. If these are not already downloaded, include `artifacts/download_artifacts.jl`. +=# + +#= +### Package Import +=# + +## standard packages +using Dates +import YAML + +# ## ClimaESM packages +import ClimaAtmos as CA +using ClimaCore + +# ## Coupler specific imports +using ClimaCoupler +using ClimaCoupler.BCReader: bcfile_info_init, update_midmonth_data!, next_date_in_file, interpolate_midmonth_to_daily +using ClimaCoupler.ConservationChecker: + EnergyConservationCheck, WaterConservationCheck, check_conservation!, plot_global_conservation +using ClimaCoupler.Checkpointer: restart_model_state! +using ClimaCoupler.Diagnostics: init_diagnostics, accumulate_diagnostics!, save_diagnostics, TimeMean +using ClimaCoupler.FieldExchanger: + import_atmos_fields!, import_combined_surface_fields!, update_model_sims!, reinit_model_sims!, step_model_sims! +using ClimaCoupler.FluxCalculator: + PartitionedStateFluxes, + CombinedStateFluxes, + combined_turbulent_fluxes!, + MoninObukhovScheme, + partitioned_turbulent_fluxes!, + water_albedo_from_atmosphere! +using ClimaCoupler.Interfacer: CoupledSimulation, SurfaceStub, get_field, update_field! +using ClimaCoupler.Regridder +using ClimaCoupler.Regridder: update_surface_fractions!, combine_surfaces!, binary_mask +using ClimaCoupler.TimeManager: + current_date, Monthly, EveryTimestep, HourlyCallback, MonthlyCallback, update_firstdayofmonth!, trigger_callback! +import ClimaCoupler.Utilities: get_comms_context + +pkg_dir = pkgdir(ClimaCoupler) + +#= +### Helper Functions +These will be eventually moved to their respective component model and diagnostics packages, and so they should not +contain any internals of the ClimaCoupler source code, except extensions to the Interfacer functions. +=# + +## helpers for component models +include("components/atmosphere/climaatmos.jl") + +## helpers for user-specified IO +include("user_io/user_diagnostics.jl") +include("user_io/user_logging.jl") + +include("driver_utils.jl") +#= +### Configuration Dictionaries +Each simulation mode has its own configuration dictionary. The `config_dict` of each simulation is a merge of the default configuration +dictionary and the simulation-specific configuration dictionary, which allows the user to override the default settings. + +We can additionally pass the configuration dictionary to the component model initializers, which will then override the default settings of the component models. +=# + +## run names +run_name = "dry_held_suarez" +coupler_output_dir = "$run_name" +const FT = Float64 +restart_dir = "unspecified" +restart_t = Int(0) + +## coupler simulation specific configuration +Δt_cpl = Float64(450) +t_end = "200days" +tspan = (Float64(0.0), Float64(time_to_seconds(t_end))) +start_date = "19790301" +hourly_checkpoint = true + +## atmos arguments to override +config_dict = Dict( + # file paths + "atmos_config_file" => nothing, + "coupler_toml_file" => nothing, + "coupler_output_dir" => coupler_output_dir, + "mode_name" => "", + "run_name" => run_name, + # timestepping + "dt" => "$(Δt_cpl)secs", + "dt_save_to_sol" => "10days", + "t_end" => t_end, + "start_date" => "19790301", + # domain + # "h_elem" => 6,#16, + "z_elem" => 25,#63, + "z_max" => 45000.0, # semi-high top + "dz_bottom" => 300.0, + # "dz_top" => 3000.0, + # output + "dt_save_state_to_disk" => "1000days", + # numerics + "apply_limiter" => false, + "rayleigh_sponge" => true, + # run + "job_id" => run_name, + "surface_setup" => "PrescribedSurface", + # diagnostic (nested wirh period and short_name) + "diagnostics" => [Dict("short_name" => ["mse", "lr", "mass_streamfunction", "stab", "vT", "egr"], "period" => "1days"),], + # held-suarez specific + # "forcing" => "held_suarez", + "viscous_sponge" => true, + "rayleigh_sponge" => true, + "vert_diff" => "true", +) + + +# Alternative +# dt_save_state_to_disk: "10days" +# dt: "150secs" +# t_end: "300days" +# h_elem: 16 +# z_elem: 63 +# dz_bottom: 30.0 +# dz_top: 3000.0 +# z_max: 55000.0 +# vert_diff: "true" +# moist: "equil" +# precip_model: "0M" +# rayleigh_sponge: true +# forcing: "held_suarez" +# job_id: "longrun_hs_rhoe_equil_55km_nz63_0M" +# toml: [toml/longrun_hs_rhoe_equil_55km_nz63_0M.toml] + +# cirrently +#precip_model: "0M" +#moist: "equil" +# toml: [toml/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.toml] + + +## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries +## (if there are common keys, the last dictorionary in the `merge` arguments takes precedence) +config_dict_atmos = get_atmos_config_dict(config_dict) +config_dict = merge(config_dict_atmos, config_dict) +atmos_config_object = CA.AtmosConfig(config_dict_atmos) +# By passing `parsed_args` to `AtmosConfig`, `parsed_args` overwrites the default atmos config + +# overriding parameter values +atmos_config_object.toml_dict["zd_viscous"]["value"] = 30000.0 +atmos_config_object.toml_dict["zd_rayleigh"]["value"] = 30000.0 + +#= +## Setup Communication Context +We set up communication context for CPU single thread/CPU with MPI/GPU. If no device is passed to `ClimaComms.context()` +then `ClimaComms` automatically selects the device from which this code is called. +=# + +using ClimaComms +comms_ctx = get_comms_context(Dict("device" => "auto")) +ClimaComms.init(comms_ctx) + + +#= +### I/O Directory Setup +`coupler_output_dir` is the directory where the output of the simulation will be saved, and `COUPLER_ARTIFACTS_DIR` is the directory where +the plots (from postprocessing and the conservation checks) of the simulation will be saved. `REGRID_DIR` is the directory where the regridding +temporary files will be saved. +=# + +dir_paths = setup_output_dirs(output_dir = coupler_output_dir, comms_ctx = comms_ctx) + +ClimaComms.iamroot(comms_ctx) ? @info(config_dict) : nothing + +#= +## Data File Paths +The data files are downloaded from the `ClimaCoupler` artifacts directory. If the data files are not present, they are downloaded from the +original sources. +=# + +include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl")) +co2_data = joinpath(co2_dataset_path(), "mauna_loa_co2.nc") + +#= +## Component Model Initialization +Here we set initial and boundary conditions for each component model. Each component model is required to have an `init` function that +returns a `ComponentModelSimulation` object (see `Interfacer` docs for more details). + +### Atmosphere +This uses the `ClimaAtmos.jl` model, with parameterization options specified in the `config_dict_atmos` dictionary. +=# + +## init atmos model component +atmos_sim = atmos_init(FT, atmos_config_object); +thermo_params = get_thermo_params(atmos_sim) + +#= +### Boundary Space +We use a common `Space` for all global surfaces. This enables the MPI processes to operate on the same columns in both +the atmospheric and surface components, so exchanges are parallelized. Note this is only possible when the +atmosphere and surface are of the same horizontal resolution. +=# + +## init a 2D boundary space at the surface +boundary_space = ClimaCore.Spaces.horizontal_space(atmos_sim.domain.face_space) # TODO: specify this in the coupler and pass it to all component models #665 + +#= +## Coupler Initialization +The coupler needs to contain exchange information, manage the calendar and be able to access all component models. It can also optionally +save online diagnostics. These are all initialized here and saved in a global `CoupledSimulation` struct, `cs`. +=# + +## coupler exchange fields +coupler_field_names = ( + :T_S, + :z0m_S, + :z0b_S, + :ρ_sfc, + :q_sfc, + :surface_direct_albedo, + :surface_diffuse_albedo, + :beta, + :F_turb_energy, + :F_turb_moisture, + :F_turb_ρτxz, + :F_turb_ρτyz, + :F_radiative, + :P_liq, + :P_snow, + :radiative_energy_flux_toa, + :P_net, + :temp1, + :temp2, +) +coupler_fields = + NamedTuple{coupler_field_names}(ntuple(i -> ClimaCore.Fields.zeros(boundary_space), length(coupler_field_names))) + +## model simulations +model_sims = (atmos_sim = atmos_sim,); + +## dates +date0 = date = DateTime(start_date, dateformat"yyyymmdd") +dates = (; date = [date], date0 = [date0], date1 = [Dates.firstdayofmonth(date0)], new_month = [false]) + +#= +### Online Diagnostics +User can write custom diagnostics in the `user_diagnostics.jl`. +Note, this will be replaced by the diagnostics framework currently in ClimaAtmos, once it is abstracted +into a more general package, so we can use it to save fields from surface models. +=# + +# monthly_3d_diags = init_diagnostics( +# (:T, :u, :q_tot, :q_liq_ice), +# atmos_sim.domain.center_space; +# save = Monthly(), +# operations = (; accumulate = TimeMean([Int(0)])), +# output_dir = dir_paths.output, +# name_tag = "monthly_mean_3d_", +# ) + +# monthly_2d_diags = init_diagnostics( +# (:precipitation_rate,), +# boundary_space; +# save = Monthly(), +# operations = (; accumulate = TimeMean([Int(0)])), +# output_dir = dir_paths.output, +# name_tag = "monthly_mean_2d_", +# ) + +diagnostics = () + +#= +## Initialize Callbacks +Callbacks are used to update at a specified interval. The callbacks are initialized here and +saved in a global `Callbacks` struct, `callbacks`. The `trigger_callback!` function is used to call the callback during the simulation below. + +The frequency of the callbacks is specified in the `HourlyCallback` and `MonthlyCallback` structs. The `func` field specifies the function to be called, +the `ref_date` field specifies the reference (first) date for the callback, and the `active` field specifies whether the callback is active or not. + +The currently implemented callbacks are: +- `checkpoint_cb`: generates a checkpoint of all model states at a specified interval. This is mainly used for restarting simulations. +- `update_firstdayofmonth!_cb`: generates a callback to update the first day of the month for monthly message print (and other monthly operations). +- `albedo_cb`: for the amip mode, the water albedo is time varying (since the reflectivity of water depends on insolation and wave characteristics, with the latter + being approximated from wind speed). It is updated at the same frequency as the atmospheric radiation. + NB: Eventually, we will call all of radiation from the coupler, in addition to the albedo calculation. +=# +checkpoint_cb = + HourlyCallback(dt = FT(480), func = checkpoint_sims, ref_date = [dates.date[1]], active = hourly_checkpoint) # 20 days TODO: not GPU friendly +update_firstdayofmonth!_cb = + MonthlyCallback(dt = FT(1), func = update_firstdayofmonth!, ref_date = [dates.date1[1]], active = true) +callbacks = + (; checkpoint = checkpoint_cb, update_firstdayofmonth! = update_firstdayofmonth!_cb) + +cs = CoupledSimulation{FT}( + comms_ctx, + dates, + boundary_space, + coupler_fields, + config_dict, + nothing, # conservation checks + [tspan[1], tspan[2]], + atmos_sim.integrator.t, + Δt_cpl, + (; land = land_fraction, ocean = zeros(boundary_space), ice = zeros(boundary_space)), + model_sims, + (;), # mode_specifics + diagnostics, + callbacks, + dir_paths, + nothing, # turbulent_fluxes + thermo_params, +); + +#= +## Restart component model states if specified +If a restart directory is specified and contains output files from the `checkpoint_cb` callback, the component model states are restarted from those files. The restart directory +is specified in the `config_dict` dictionary. The `restart_t` field specifies the time step at which the restart is performed. +=# + +if restart_dir !== "unspecified" + for sim in cs.model_sims + if get_model_prog_state(sim) !== nothing + restart_model_state!(sim, comms_ctx, restart_t; input_dir = restart_dir) + end + end +end + +#= +## Coupling Loop + +The coupling loop is the main part of the simulation. It runs the component models sequentially for one coupling timestep (`Δt_cpl`), and exchanges combined fields and calculates fluxes using combined states. +Note that we want to implement this in a dispatchable function to allow for other forms of timestepping (e.g. leapfrog). (TODO: #610) +=# + +function solve_coupler!(cs) + (; model_sims, Δt_cpl, tspan, comms_ctx) = cs + (; atmos_sim) = model_sims + + ClimaComms.iamroot(comms_ctx) ? @info("Starting coupling loop") : nothing + ## step in time + walltime = @elapsed for t in ((tspan[begin] + Δt_cpl):Δt_cpl:tspan[end]) + + cs.dates.date[1] = current_date(cs, t) + + ## print date on the first of month + if cs.dates.date[1] >= cs.dates.date1[1] + ClimaComms.iamroot(comms_ctx) ? @show(cs.dates.date[1]) : nothing + end + + ## step sims + step_model_sims!(cs.model_sims, t) + + import_atmos_fields!(cs.fields, cs.model_sims, cs.boundary_space, cs.turbulent_fluxes) # radiative and/or turbulent + + ## callback to update the fist day of month if needed (for BCReader) + trigger_callback!(cs, cs.callbacks.update_firstdayofmonth!) + + ## callback to checkpoint model state + trigger_callback!(cs, cs.callbacks.checkpoint) + + end + ClimaComms.iamroot(comms_ctx) ? @show(walltime) : nothing + + return cs +end + +## run the coupled simulation +solve_coupler!(cs); + +# Postprocessing +anim = true +energy_check = false +# include("postprocessing.jl") + + +# climate diagnostics: T, u, (q), rho v, N, MSE + +# storm track diagnostics: [vT], [v][T], EGR = f/N dTdy + + + diff --git a/experiments/AMIP/dry_hs.toml b/experiments/AMIP/dry_hs.toml new file mode 100644 index 000000000..5fdedc248 --- /dev/null +++ b/experiments/AMIP/dry_hs.toml @@ -0,0 +1,5 @@ +[zd_rayleigh] +value = 35000.0 + +[alpha_rayleigh_uh] +value = 0.0 \ No newline at end of file diff --git a/experiments/AMIP/postprocessing.jl b/experiments/AMIP/postprocessing.jl new file mode 100644 index 000000000..198d990d7 --- /dev/null +++ b/experiments/AMIP/postprocessing.jl @@ -0,0 +1,108 @@ +#= +## Postprocessing +Currently all postprocessing is performed using the root process only. +=# + +if ClimaComms.iamroot(comms_ctx) + + ## energy check plots + if !isnothing(cs.conservation_checks) && cs.mode.name[1:10] == "slabplanet" + @info "Conservation Check Plots" + plot_global_conservation( + cs.conservation_checks.energy, + cs, + config_dict["conservation_softfail"], + figname1 = joinpath(COUPLER_ARTIFACTS_DIR, "total_energy_bucket.png"), + figname2 = joinpath(COUPLER_ARTIFACTS_DIR, "total_energy_log_bucket.png"), + ) + plot_global_conservation( + cs.conservation_checks.water, + cs, + config_dict["conservation_softfail"], + figname1 = joinpath(COUPLER_ARTIFACTS_DIR, "total_water_bucket.png"), + figname2 = joinpath(COUPLER_ARTIFACTS_DIR, "total_water_log_bucket.png"), + ) + end + + ## sample animations (not compatible with MPI) + if !CA.is_distributed(comms_ctx) && config_dict["anim"] + @info "Animations" + include("user_io/viz_explorer.jl") + plot_anim(cs, COUPLER_ARTIFACTS_DIR) + end + + ## plotting AMIP results + if cs.mode.name == "amip" + @info "AMIP plots" + + ## ClimaESM + include("user_io/amip_visualizer.jl") + post_spec = (; + T = (:regrid, :zonal_mean), + u = (:regrid, :zonal_mean), + q_tot = (:regrid, :zonal_mean), + toa_fluxes = (:regrid, :horizontal_slice), + precipitation_rate = (:regrid, :horizontal_slice), + T_sfc = (:regrid, :horizontal_slice), + tubulent_energy_fluxes = (:regrid, :horizontal_slice), + q_liq_ice = (:regrid, :zonal_mean), + ) + + plot_spec = (; + T = (; clims = (190, 320), units = "K"), + u = (; clims = (-50, 50), units = "m/s"), + q_tot = (; clims = (0, 30), units = "g/kg"), + toa_fluxes = (; clims = (-250, 250), units = "W/m^2"), + precipitation_rate = (clims = (0, 1e-4), units = "kg/m^2/s"), + T_sfc = (clims = (225, 310), units = "K"), + tubulent_energy_fluxes = (; clims = (-250, 250), units = "W/m^2"), + q_liq_ice = (; clims = (0, 10), units = "g/kg"), + ) + amip_data = amip_paperplots( + post_spec, + plot_spec, + COUPLER_OUTPUT_DIR, + files_root = ".monthly", + output_dir = COUPLER_ARTIFACTS_DIR, + ) + + ## NCEP reanalysis + @info "NCEP plots" + include("user_io/ncep_visualizer.jl") + ncep_post_spec = (; + T = (:zonal_mean,), + u = (:zonal_mean,), + q_tot = (:zonal_mean,), + toa_fluxes = (:horizontal_slice,), + precipitation_rate = (:horizontal_slice,), + T_sfc = (:horizontal_slice,), + tubulent_energy_fluxes = (:horizontal_slice,), + ) + ncep_plot_spec = plot_spec + ncep_data = ncep_paperplots( + ncep_post_spec, + ncep_plot_spec, + COUPLER_OUTPUT_DIR, + output_dir = COUPLER_ARTIFACTS_DIR, + month_date = cs.dates.date[1], + ) ## plot data that correspond to the model's last save_hdf5 call (i.e., last month) + + # Compare against observations + if t_end > 84600 + @info "Error against observations" + include("user_io/leaderboard.jl") + compare_vars = ["pr"] + output_path = joinpath(COUPLER_ARTIFACTS_DIR, "biases.png") + Leaderboard.plot_biases(atmos_sim.integrator.p.output_dir, compare_vars, cs.dates.date; output_path) + end + end + + if isinteractive() + ## clean up for interactive runs, retain all output otherwise + rm(COUPLER_OUTPUT_DIR; recursive = true, force = true) + + ## plot all model states and coupler fields (useful for debugging) + debug(cs) + end + +end \ No newline at end of file diff --git a/test/Project.toml b/test/Project.toml index c2ba22538..bed891c17 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -6,6 +6,7 @@ ClimaAnalysis = "29b5916a-a76c-4e73-9657-3c8fd22e65e6" ClimaAtmos = "b2c96348-7fb7-4fe0-8da9-78d88439e717" ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" +ClimaCoreMakie = "908f55d8-4145-4867-9c14-5dad1a479e4d" ClimaCorePlots = "cf7c7e5a-b407-4c48-9047-11a94a308626" ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7" ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532" diff --git a/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml b/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml index 306dd669c..c2cbfb011 100644 --- a/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml +++ b/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml @@ -1,3 +1,4 @@ +albedo_model: "RegressionFunctionAlbedo" approximate_linear_solve_iters: 2 bubble: false check_conservation: true @@ -22,4 +23,3 @@ coupler_toml_file: "test/component_model_tests/climaatmos_standalone/longrun_aqu vert_diff: "FriersonDiffusion" z_elem: 63 z_max: 55000.0 -output_default_diagnostics: false diff --git a/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean_gpu.yml b/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean_gpu.yml new file mode 100644 index 000000000..8140a4d33 --- /dev/null +++ b/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean_gpu.yml @@ -0,0 +1,25 @@ +albedo_model: "RegressionFunctionAlbedo" +approximate_linear_solve_iters: 2 +bubble: false +check_conservation: true +dt_rad: "1hours" +dt_save_state_to_disk: "10days" +dt: "150secs" +dz_bottom: 30.0 +dz_top: 3000.0 +h_elem: 16 +idealized_insolation: false +implicit_diffusion: true +job_id: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean_gpu" +moist: "equil" +precip_model: "0M" +prognostic_surface: "PrognosticSurfaceTemperature" +rad: "clearsky" +run_name: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean_gpu" +rayleigh_sponge: true +surface_setup: "DefaultMoninObukhov" +t_end: "100days" +coupler_toml_file: "test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.toml" +vert_diff: "FriersonDiffusion" +z_elem: 63 +z_max: 55000.0