diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index 351b34fe7..81f270bd0 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -18,43 +18,68 @@ env: timeout_in_minutes: 1440 steps: - - label: "init :computer:" - key: "init_cpu_env" - command: - - "echo $$JULIA_DEPOT_PATH" - - - echo "--- Instantiate AMIP env" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" - - - echo "--- Download artifacts" - - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'" - - "julia --project=artifacts -e 'using Pkg; Pkg.status()'" - - "julia --project=artifacts artifacts/download_artifacts.jl" - - - echo "--- Instantiate perf env" - - "julia --project=perf/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=perf/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=perf/ -e 'using Pkg; Pkg.status()'" - - agents: - slurm_cpus_per_task: 8 - env: - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_MAX_NUM_PRECOMPILE_FILES: 50 + # - label: "init :computer:" + # key: "init_cpu_env" + # command: + # - "echo $$JULIA_DEPOT_PATH" + + # - echo "--- Instantiate AMIP env" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" + + # - echo "--- Download artifacts" + # - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=artifacts -e 'using Pkg; Pkg.status()'" + # - "julia --project=artifacts artifacts/download_artifacts.jl" + + # - echo "--- Instantiate perf env" + # - "julia --project=perf/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=perf/ -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=perf/ -e 'using Pkg; Pkg.status()'" + + # agents: + # slurm_cpus_per_task: 8 + # env: + # JULIA_NUM_PRECOMPILE_TASKS: 8 + # JULIA_MAX_NUM_PRECOMPILE_FILES: 50 # DYAMOND AMIP: 1 day (convection resolving) - - label: "GPU AMIP SUPERFINE: dyamond_target" - key: "gpu_dyamond_target" + # - label: "GPU AMIP SUPERFINE: dyamond_target" + # key: "gpu_dyamond_target" + # command: + # - "echo $$JULIA_DEPOT_PATH" + + # - echo "--- Instantiate AMIP env" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" + + # - echo "--- Download artifacts" + # - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=artifacts -e 'using Pkg; Pkg.status()'" + # - "julia --project=artifacts artifacts/download_artifacts.jl" + + # - echo "--- Run simulation" + # - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*" + # agents: + # queue: clima + # slurm_mem: 20GB + # slurm_gpus: 1 + # modules: common + + - label: "GPU atmos-only DYAMOND without diagnostics" + key: "longrun_aquaplanet_dyamond" command: - "echo $$JULIA_DEPOT_PATH" - - echo "--- Instantiate AMIP env" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" + - echo "--- Instantiate test env" + - "julia --project=test -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=test -e 'using Pkg; Pkg.precompile()'" + - "julia --project=test -e 'using Pkg; Pkg.status()'" - echo "--- Download artifacts" - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" @@ -62,349 +87,351 @@ steps: - "julia --project=artifacts -e 'using Pkg; Pkg.status()'" - "julia --project=artifacts artifacts/download_artifacts.jl" - - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*" + - echo "--- Run simulation" + - "julia --color=yes --project=test test/component_model_tests/climaatmos_standalone/atmos_driver.jl" + artifact_paths: "test/component_model_tests/climaatmos_standalone/output/longrun_aquaplanet_dyamond_artifacts/*" agents: queue: clima slurm_mem: 20GB slurm_gpus: 1 modules: common + - wait - - group: "Coupler integration and conservation tests" - - steps: - - # Integration tests - the expected results were also confirmed locally - - label: "Slabplanet_aqua: nocouple" - key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "Slabplanet_aqua: couple" - key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "Slabplanet_aqua: coupler fluxes" - key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "Slabplanet_aqua: coupler fluxes, evolving ocean" - key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "Slabplanet_terra: coupler fluxes, evolving bucket" - key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_terra.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_terra/slabplanet_terra_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "Slabplanet: coupler fluxes, evolving ocean and land" - key: "slabplanet_coupler_sf_evolve_ocn" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - group: "Current target tests: idealized surface" - - steps: - - - label: "ClimaAtmos standalone target" - command: - - srun julia --project=experiments/AMIP/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml - artifact_paths: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean/*" - env: - BUILD_HISTORY_HANDLE: "" - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc" - key: "slabplanet_aqua_target_nocouple" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc" - key: "slabplanet_aqua_target" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T" - key: "slabplanet_aqua_target_evolve_ocn" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket" - key: "slabplanet_target" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket" - key: "slabplanet_target_evolve_ocn" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - - group: "Current target tests: AMIP surface" - - steps: - - - label: "MPI AMIP FINE: new target amip" - key: "amip_target" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_target_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: new target amip: topo" - key: "amip_target_topo" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: new target amip: topo + diagedmf" - key: "amip_target_topo_diagedmf" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 20G - - - group: "Current target tests on GPU: AMIP surface" - - steps: - - - label: "GPU AMIP FINE: new target amip: topo" - key: "gpu_amip_target_topo" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_artifacts/*" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - label: "GPU AMIP FINE: new target amip: topo + diagedmf" - key: "gpu_amip_target_topo_diagedmf" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*" - agents: - slurm_gpus: 1 - slurm_mem: 16GB - - - group: "Other AMIP targets" - - steps: - - # DYAMOND AMIP: 1 day (convection resolving) - - label: "MPI AMIP SUPERFINE: dyamond_target" - key: "dyamond_target" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/dyamond_target.yml" - artifact_paths: "experiments/AMIP/output/amip/dyamond_target_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - # mid-resolution AMIP: MPI performance scaling (10 days) - - label: "MPI AMIP FINE: n64" - key: "mpi_amip_fine_n64" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n64_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_n64_shortrun_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 16 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: n32" - key: "mpi_amip_fine_n32" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n32_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_n32_shortrun_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 8 - slurm_nodes: 4 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: n8" - key: "mpi_amip_fine_n8" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n8_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_n8_shortrun_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 8 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: n2" # 10d take 21h, so reducing to 1d - key: "mpi_amip_fine_n2" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n2_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_n2_shortrun_artifacts/*" - env: - CLIMACORE_DISTRIBUTED: "MPI" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 2 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: n1" # also reported by longruns with a flame graph; 10d take 21h, so reducing to 1d - key: "mpi_amip_fine_n1" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n1_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_n1_shortrun_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - label: "MPI AMIP FINE: n1 no couple" # sim time = Δt_cpl (~ benchmarking with standalone models) - key: "mpi_amip_fine_n1_nocouple" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n1_shortrun_nocouple.yml" - artifact_paths: "experiments/AMIP/output/amip/amip_n1_shortrun_nocouple_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - # mpi_amip_fine_n1 flame graph report (NB: arguments passed from the ci pipeline.yml) - - label: ":rocket: performance: flame graph diff: perf_target_amip_n1_shortrun" - command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_target_amip_n1_shortrun.yml" - artifact_paths: "perf/output/perf_diff_target_amip_n1_shortrun/*" - agents: - slurm_ntasks_per_node: 1 - slurm_nodes: 1 - slurm_mem_per_cpu: 16G - - - wait - - # plot job performance history - - label: ":chart_with_downwards_trend: build history" - command: - - build_history main # name of branch to plot - artifact_paths: - - "build_history.html" - - - wait - - - label: ":envelope: Slack report: build_history" - command: - - slack-upload -c "#coupler-report" -f build_history.html -m html -n build_history -x "Overall job performance" - - - label: ":envelope: Slack report: Slabplanet" - command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" - - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" - - - label: ":envelope: Slack report: target AMIP" - command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" - - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_artifacts/biases.png -m png -n amip_fine -x "AMIP Target Longrun" - - - label: ":envelope: Slack report: Flame Diff" - command: - - slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff.html -m png -n amip_fine_flamegraphdiff -x "AMIP Longrun FlameGraphDiff" - - slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff_self_count.html -m png -n amip_fine_flamegraphdiffself -x "AMIP Longrun FlameGraphDiffSelf" + # - group: "Coupler integration and conservation tests" + + # steps: + + # # Integration tests - the expected results were also confirmed locally + # - label: "Slabplanet_aqua: nocouple" + # key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "Slabplanet_aqua: couple" + # key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "Slabplanet_aqua: coupler fluxes" + # key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "Slabplanet_aqua: coupler fluxes, evolving ocean" + # key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "Slabplanet_terra: coupler fluxes, evolving bucket" + # key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_terra.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_terra/slabplanet_terra_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "Slabplanet: coupler fluxes, evolving ocean and land" + # key: "slabplanet_coupler_sf_evolve_ocn" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - group: "Current target tests: idealized surface" + + # steps: + + # - label: "ClimaAtmos standalone target" + # command: + # - srun julia --project=experiments/AMIP/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml + # artifact_paths: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc" + # key: "slabplanet_aqua_target_nocouple" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc" + # key: "slabplanet_aqua_target" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T" + # key: "slabplanet_aqua_target_evolve_ocn" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket" + # key: "slabplanet_target" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket" + # key: "slabplanet_target_evolve_ocn" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + + # - group: "Current target tests: AMIP surface" + + # steps: + + # - label: "MPI AMIP FINE: new target amip" + # key: "amip_target" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_target_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: new target amip: topo" + # key: "amip_target_topo" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: new target amip: topo + diagedmf" + # key: "amip_target_topo_diagedmf" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 20G + + # - group: "Current target tests on GPU: AMIP surface" + + # steps: + + # - label: "GPU AMIP FINE: new target amip: topo" + # key: "gpu_amip_target_topo" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_artifacts/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - label: "GPU AMIP FINE: new target amip: topo + diagedmf" + # key: "gpu_amip_target_topo_diagedmf" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16GB + + # - group: "Other AMIP targets" + + # steps: + + # # DYAMOND AMIP: 1 day (convection resolving) + # - label: "MPI AMIP SUPERFINE: dyamond_target" + # key: "dyamond_target" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/dyamond_target.yml" + # artifact_paths: "experiments/AMIP/output/amip/dyamond_target_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # # mid-resolution AMIP: MPI performance scaling (10 days) + # - label: "MPI AMIP FINE: n64" + # key: "mpi_amip_fine_n64" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n64_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_n64_shortrun_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 16 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: n32" + # key: "mpi_amip_fine_n32" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n32_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_n32_shortrun_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 8 + # slurm_nodes: 4 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: n8" + # key: "mpi_amip_fine_n8" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n8_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_n8_shortrun_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 8 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: n2" # 10d take 21h, so reducing to 1d + # key: "mpi_amip_fine_n2" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n2_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_n2_shortrun_artifacts/*" + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 2 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: n1" # also reported by longruns with a flame graph; 10d take 21h, so reducing to 1d + # key: "mpi_amip_fine_n1" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n1_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_n1_shortrun_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - label: "MPI AMIP FINE: n1 no couple" # sim time = Δt_cpl (~ benchmarking with standalone models) + # key: "mpi_amip_fine_n1_nocouple" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_n1_shortrun_nocouple.yml" + # artifact_paths: "experiments/AMIP/output/amip/amip_n1_shortrun_nocouple_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # # mpi_amip_fine_n1 flame graph report (NB: arguments passed from the ci pipeline.yml) + # - label: ":rocket: performance: flame graph diff: perf_target_amip_n1_shortrun" + # command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_target_amip_n1_shortrun.yml" + # artifact_paths: "perf/output/perf_diff_target_amip_n1_shortrun/*" + # agents: + # slurm_ntasks_per_node: 1 + # slurm_nodes: 1 + # slurm_mem_per_cpu: 16G + + # - wait + + # # plot job performance history + # - label: ":chart_with_downwards_trend: build history" + # command: + # - build_history main # name of branch to plot + # artifact_paths: + # - "build_history.html" + + # - wait + + # - label: ":envelope: Slack report: build_history" + # command: + # - slack-upload -c "#coupler-report" -f build_history.html -m html -n build_history -x "Overall job performance" + + # - label: ":envelope: Slack report: Slabplanet" + # command: + # - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" + # - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + # - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" + # - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" + + # - label: ":envelope: Slack report: target AMIP" + # command: + # - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" + # - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_artifacts/biases.png -m png -n amip_fine -x "AMIP Target Longrun" + + # - label: ":envelope: Slack report: Flame Diff" + # command: + # - slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff.html -m png -n amip_fine_flamegraphdiff -x "AMIP Longrun FlameGraphDiff" + # - slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff_self_count.html -m png -n amip_fine_flamegraphdiffself -x "AMIP Longrun FlameGraphDiffSelf" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f35cf213e..c6b667c14 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -34,25 +34,25 @@ steps: - "julia --project -e 'using Pkg; Pkg.precompile()'" - "julia --project -e 'using Pkg; Pkg.status()'" - - echo "--- Instantiate sea breeze env" - - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.precompile()'" - - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.status()'" - - - echo "--- Instantiate climacore exp env" - - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.status()'" - - - echo "--- Instantiate AMIP env" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" - - - echo "--- Instantiate perf env" - - "julia --project=perf/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=perf/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=perf/ -e 'using Pkg; Pkg.status()'" + # - echo "--- Instantiate sea breeze env" + # - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.status()'" + + # - echo "--- Instantiate climacore exp env" + # - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.status()'" + + # - echo "--- Instantiate AMIP env" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'" + + # - echo "--- Instantiate perf env" + # - "julia --project=perf/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project=perf/ -e 'using Pkg; Pkg.precompile()'" + # - "julia --project=perf/ -e 'using Pkg; Pkg.status()'" - echo "--- Instantiate test env" - "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'" @@ -75,361 +75,376 @@ steps: - wait - - group: "Unit Tests" - steps: - - label: "MPI Regridder unit tests" - key: "regridder_mpi_tests" - command: "srun julia --color=yes --project=test/ test/mpi_tests/regridder_mpi_tests.jl --config_file $MPI_CONFIG_PATH/regridder_mpi.yml" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - NPROCS: 2 - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "MPI BCReader unit tests" - key: "bcreader_mpi_tests" - command: "srun julia --color=yes --project=test/ test/mpi_tests/bcreader_mpi_tests.jl --run_name bcreader_mpi --job_id bcreader_mpi" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "MPI Checkpointer unit tests" - key: "checkpointer_mpi_tests" - command: "srun julia --color=yes --project=test/ test/mpi_tests/checkpointer_mpi_tests.jl --run_name checkpointer_mpi --job_id checkpointer_mpi" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "MPI Utilities unit tests" - key: "utilities_mpi_tests" - command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_mpi --job_id utilities_mpi" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "GPU Utilities unit tests" - key: "utilities_gpu_tests" - command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_gpu --job_id utilities_gpu" - agents: - slurm_mem: 5GB - slurm_gpus: 1 - - - label: "Perf flame graph diff tests" - command: "julia --color=yes --project=perf/ perf/flame_test.jl --run_name flame_test --job_id flame_perf_target" - timeout_in_minutes: 5 - agents: - slurm_mem: 16GB - - - group: "Integration Tests" - steps: - - # Drivers for release > - - # SLABPLANET - - # Slabplanet default: - # - this is the most lightweight example with conservation and visual checks, with CLI specification as follows - # - numerics: dt = dt_cpl = 200s, nelem = 4 - # - physics: bulk aerodynamic surface fluxes, gray radiation, idealized insolation, equil moisture model, 0-moment microphysics - # - input data: monotonous remapping (land mask, SST, SIC) - # - slurm: unthreaded, 1 ntask - # - diagnostics: check and plot energy conservation, output plots after 9 days - - label: "Slabplanet: default" - key: "slabplanet_default" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_default.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_default_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: default with Float32" - key: "slabplanet_ft32" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_ft32.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_ft32_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: partitioned turbulent fluxes" - key: "slabplanet_partitioned_fluxes" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_partitioned_fluxes.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_partitioned_fluxes_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: non-monotonous surface remap" - key: "slabplanet_non-monotonous" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_nonmono.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_nonmono_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: albedo from static map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_static_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_static_map_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: albedo from temporal map" - key: "slabplanet_albedo_temporal_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_temporal_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_temporal_map_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: albedo from function" - key: "slabplanet_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: eisenman sea ice" - key: "slabplanet_eisenman" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_eisenman.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_eisenman/slabplanet_eisenman_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: extra atmos diagnostics" - key: "slabplanet_atmos_diags" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_atmos_diags.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_atmos_diags_artifacts/*" - agents: - slurm_mem: 20GB - - # AMIP - - # ... - - - # PERFORMANCE - - # slabplanet default: track unthreaded performance (alloc tests, flame graph, flame graph diff, build history) - - label: ":rocket: Slabplanet: default (unthreaded)" - key: "slabplanet_unthreaded" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_unthreaded.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/default_unthreaded_artifacts/*" - env: - FLAME_PLOT: "" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: ":rocket: Slabplanet: default (unthreaded) - flame graph and allocation tests" - command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_default_unthreaded.yml" - artifact_paths: "perf/output/perf_default_unthreaded/*" - agents: - slurm_mem: 20GB - - - label: ":rocket: Slabplanet: default (unthreaded) - flame graph diff" - command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_default_unthreaded.yml" - artifact_paths: "perf/output/perf_diff_default_unthreaded/*" - agents: - slurm_mem: 20GB - - # < end Drivers for release - - # CLIMACORE EXPERIMENTS - - - label: "sea_breeze" - command: "julia --color=yes --project=experiments/ClimaCore/sea_breeze experiments/ClimaCore/sea_breeze/run.jl" - artifact_paths: "experiments/ClimaCore/sea_breeze/output/*" - agents: - slurm_mem: 20GB - - - label: "heat-diffusion" - command: "julia --color=yes --project=experiments/ClimaCore/ experiments/ClimaCore/heat-diffusion/run.jl" - artifact_paths: "experiments/ClimaCore/output/heat-diffusion_artifacts/*" - agents: - slurm_mem: 20GB - - # AMIP AND SLABPLANET EXPERIMENTS - - - label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_mono.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/default_mono_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - - label: "Moist earth with slab surface - notmono: monin gray no_sponge idealinsol freq_dt_cpl notmono" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_notmono.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/default_notmono_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - # - label: "Moist earth with slab surface - test: monin allsky sponge idealinsol infreq_dt_cpl" - # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab_test1 --job_id target_params_in_slab_test1" # Unconverged SF (reproduced locally); works with 200s dt_cpl - # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test1_artifacts/total_energy*.png" - - - label: "Moist earth with slab surface - test: bulk allsky sponge realinsol infreq_dt_cpl" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test2.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test2_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - - label: "Moist earth with slab surface - test: monin gray sponge realinsol infreq_dt_cpl" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test3.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test3_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - # breaking: - # - label: "Moist earth with slab surface - monin allsky no_sponge idealinsol infreq_dt_cpl" - # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge false --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --idealized_insolation true --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab1" - # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab1_artifacts/total_energy*.png" - - - label: "AMIP target: albedo from function" - key: "target_amip_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/amip/target_amip_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "AMIP - Float64 + hourly checkpoint" - key: "amip" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_artifacts/*" - env: - FLAME_PLOT: "" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "AMIP - Float64 + hourly checkpoint + co2" - key: "coarse_single_ft64_hourly_checkpoints_co2" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints_co2.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_co2_artifacts/*" - env: - FLAME_PLOT: "" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "AMIP - Float64 test" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_artifacts/*" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "AMIP - Float32 test" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft32.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft32_artifacts/*" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "MPI AMIP" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_mpi_n2.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_mpi_n2_artifacts/*" - timeout_in_minutes: 240 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "batch script" - command: "sbatch test/mpi_tests/local_checks.sh" - - # short high-res performance test - - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph - key: "unthreaded_amip_fine" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_n1_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/target_amip_n1_shortrun_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_mem: 20GB - - # PERFORMANCE RUNS: flame graphs + allocation tests - - - label: ":rocket: flame graph and allocation tests: perf_coarse_single_ft64" - command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_coarse_single_ft64.yml" - artifact_paths: "perf/output/perf_coarse_single_ft64/*" - agents: - slurm_mem: 20GB - - - label: ":rocket: performance: flame graph diff: perf_diff_coarse_single_ft64" - command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_coarse_single_ft64.yml" - artifact_paths: "perf/output/perf_diff_coarse_single_ft64/*" - agents: - slurm_mem: 20GB + # - group: "Unit Tests" + # steps: + # - label: "MPI Regridder unit tests" + # key: "regridder_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/mpi_tests/regridder_mpi_tests.jl --config_file $MPI_CONFIG_PATH/regridder_mpi.yml" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # NPROCS: 2 + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "MPI BCReader unit tests" + # key: "bcreader_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/mpi_tests/bcreader_mpi_tests.jl --run_name bcreader_mpi --job_id bcreader_mpi" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "MPI Checkpointer unit tests" + # key: "checkpointer_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/mpi_tests/checkpointer_mpi_tests.jl --run_name checkpointer_mpi --job_id checkpointer_mpi" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "MPI Utilities unit tests" + # key: "utilities_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_mpi --job_id utilities_mpi" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "GPU Utilities unit tests" + # key: "utilities_gpu_tests" + # command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_gpu --job_id utilities_gpu" + # agents: + # slurm_mem: 5GB + # slurm_gpus: 1 + + # - label: "Perf flame graph diff tests" + # command: "julia --color=yes --project=perf/ perf/flame_test.jl --run_name flame_test --job_id flame_perf_target" + # timeout_in_minutes: 5 + # agents: + # slurm_mem: 16GB + + # - group: "Integration Tests" + # steps: + + # # Drivers for release > + + # # SLABPLANET + + # # Slabplanet default: + # # - this is the most lightweight example with conservation and visual checks, with CLI specification as follows + # # - numerics: dt = dt_cpl = 200s, nelem = 4 + # # - physics: bulk aerodynamic surface fluxes, gray radiation, idealized insolation, equil moisture model, 0-moment microphysics + # # - input data: monotonous remapping (land mask, SST, SIC) + # # - slurm: unthreaded, 1 ntask + # # - diagnostics: check and plot energy conservation, output plots after 9 days + # - label: "Slabplanet: default" + # key: "slabplanet_default" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_default.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_default_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: default with Float32" + # key: "slabplanet_ft32" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_ft32.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_ft32_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: partitioned turbulent fluxes" + # key: "slabplanet_partitioned_fluxes" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_partitioned_fluxes.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_partitioned_fluxes_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: non-monotonous surface remap" + # key: "slabplanet_non-monotonous" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_nonmono.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_nonmono_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: albedo from static map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_static_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_static_map_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: albedo from temporal map" + # key: "slabplanet_albedo_temporal_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_temporal_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_temporal_map_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: albedo from function" + # key: "slabplanet_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: eisenman sea ice" + # key: "slabplanet_eisenman" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_eisenman.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_eisenman/slabplanet_eisenman_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: extra atmos diagnostics" + # key: "slabplanet_atmos_diags" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_atmos_diags.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_atmos_diags_artifacts/*" + # agents: + # slurm_mem: 20GB + + # # AMIP + + # # ... + + + # # PERFORMANCE + + # # slabplanet default: track unthreaded performance (alloc tests, flame graph, flame graph diff, build history) + # - label: ":rocket: Slabplanet: default (unthreaded)" + # key: "slabplanet_unthreaded" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_unthreaded.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/default_unthreaded_artifacts/*" + # env: + # FLAME_PLOT: "" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: ":rocket: Slabplanet: default (unthreaded) - flame graph and allocation tests" + # command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_default_unthreaded.yml" + # artifact_paths: "perf/output/perf_default_unthreaded/*" + # agents: + # slurm_mem: 20GB + + # - label: ":rocket: Slabplanet: default (unthreaded) - flame graph diff" + # command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_default_unthreaded.yml" + # artifact_paths: "perf/output/perf_diff_default_unthreaded/*" + # agents: + # slurm_mem: 20GB + + # # < end Drivers for release + + # # CLIMACORE EXPERIMENTS + + # - label: "sea_breeze" + # command: "julia --color=yes --project=experiments/ClimaCore/sea_breeze experiments/ClimaCore/sea_breeze/run.jl" + # artifact_paths: "experiments/ClimaCore/sea_breeze/output/*" + # agents: + # slurm_mem: 20GB + + # - label: "heat-diffusion" + # command: "julia --color=yes --project=experiments/ClimaCore/ experiments/ClimaCore/heat-diffusion/run.jl" + # artifact_paths: "experiments/ClimaCore/output/heat-diffusion_artifacts/*" + # agents: + # slurm_mem: 20GB + + # # AMIP AND SLABPLANET EXPERIMENTS + + # - label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_mono.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/default_mono_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # - label: "Moist earth with slab surface - notmono: monin gray no_sponge idealinsol freq_dt_cpl notmono" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_notmono.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/default_notmono_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # # - label: "Moist earth with slab surface - test: monin allsky sponge idealinsol infreq_dt_cpl" + # # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab_test1 --job_id target_params_in_slab_test1" # Unconverged SF (reproduced locally); works with 200s dt_cpl + # # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test1_artifacts/total_energy*.png" + + # - label: "Moist earth with slab surface - test: bulk allsky sponge realinsol infreq_dt_cpl" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test2.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test2_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # - label: "Moist earth with slab surface - test: monin gray sponge realinsol infreq_dt_cpl" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test3.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test3_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # # breaking: + # # - label: "Moist earth with slab surface - monin allsky no_sponge idealinsol infreq_dt_cpl" + # # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge false --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --idealized_insolation true --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab1" + # # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab1_artifacts/total_energy*.png" + + # - label: "AMIP target: albedo from function" + # key: "target_amip_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/amip/target_amip_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "AMIP - Float64 + hourly checkpoint" + # key: "amip" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_artifacts/*" + # env: + # FLAME_PLOT: "" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "AMIP - Float64 + hourly checkpoint + co2" + # key: "coarse_single_ft64_hourly_checkpoints_co2" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints_co2.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_co2_artifacts/*" + # env: + # FLAME_PLOT: "" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "AMIP - Float64 test" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_artifacts/*" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "AMIP - Float32 test" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft32.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft32_artifacts/*" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "MPI AMIP" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_mpi_n2.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_mpi_n2_artifacts/*" + # timeout_in_minutes: 240 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "batch script" + # command: "sbatch test/mpi_tests/local_checks.sh" + + # # short high-res performance test + # - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph + # key: "unthreaded_amip_fine" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_n1_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/target_amip_n1_shortrun_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_mem: 20GB + + # # PERFORMANCE RUNS: flame graphs + allocation tests + + # - label: ":rocket: flame graph and allocation tests: perf_coarse_single_ft64" + # command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_coarse_single_ft64.yml" + # artifact_paths: "perf/output/perf_coarse_single_ft64/*" + # agents: + # slurm_mem: 20GB + + # - label: ":rocket: performance: flame graph diff: perf_diff_coarse_single_ft64" + # command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_coarse_single_ft64.yml" + # artifact_paths: "perf/output/perf_diff_coarse_single_ft64/*" + # agents: + # slurm_mem: 20GB - group: "GPU integration tests" steps: - # GPU RUNS: slabplanet - - label: "GPU Slabplanet: albedo from function" - key: "gpu_slabplanet_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - label: "GPU Slabplanet: albedo from static map" - key: "gpu_slabplanet_albedo_static_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - label: "GPU Slabplanet: albedo from temporal map" - key: "gpu_slabplanet_albedo_temporal_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_temporal_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_temporal_map_artifacts/*" + # # GPU RUNS: slabplanet + # - label: "GPU Slabplanet: albedo from function" + # key: "gpu_slabplanet_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU Slabplanet: albedo from static map" + # key: "gpu_slabplanet_albedo_static_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU Slabplanet: albedo from temporal map" + # key: "gpu_slabplanet_albedo_temporal_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_temporal_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_temporal_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # # GPU RUNS: AMIP + # - label: "GPU AMIP test: albedo from function" + # key: "gpu_amip_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU AMIP target: topography and diagnostic EDMF" + # key: "gpu_amip_target_topo_diagedmf_shortrun" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU AMIP: albedo from static map" + # key: "gpu_amip_albedo_static_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_static_map.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_static_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU AMIP: albedo from temporal map" + # key: "gpu_amip_albedo_temporal_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_temporal_map.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_temporal_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU AMIP DYAMOND without diagnostics" + # key: "gpu_dyamond_nodiags" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_nodiags.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_nodiags_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + - label: "GPU atmos-only DYAMOND without diagnostics" + key: "longrun_aquaplanet_dyamond" + command: "julia --color=yes --project=test test/component_model_tests/climaatmos_standalone/atmos_driver.jl" + artifact_paths: "test/component_model_tests/climaatmos_standalone/output/longrun_aquaplanet_dyamond_artifacts/*" agents: slurm_mem: 20GB slurm_gpus: 1 - # GPU RUNS: AMIP - - label: "GPU AMIP test: albedo from function" - key: "gpu_amip_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - label: "GPU AMIP target: topography and diagnostic EDMF" - key: "gpu_amip_target_topo_diagedmf_shortrun" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - label: "GPU AMIP: albedo from static map" - key: "gpu_amip_albedo_static_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_static_map.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_static_map_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - label: "GPU AMIP: albedo from temporal map" - key: "gpu_amip_albedo_temporal_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_temporal_map.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_temporal_map_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - wait diff --git a/config/longrun_configs/gpu_dyamond_target.yml b/config/longrun_configs/gpu_dyamond_target.yml index 0c9649224..e471228f7 100644 --- a/config/longrun_configs/gpu_dyamond_target.yml +++ b/config/longrun_configs/gpu_dyamond_target.yml @@ -9,7 +9,8 @@ land_albedo_type: "map_temporal" mode_name: "amip" mono_surface: false monthly_checkpoint: false +output_default_diagnostics: false run_name: "gpu_dyamond_target" start_date: "19790301" -t_end: "1days" +t_end: "4hours" turb_flux_partition: "CombinedStateFluxes" diff --git a/config/model_configs/gpu_dyamond_nodiags.yml b/config/model_configs/gpu_dyamond_nodiags.yml new file mode 100644 index 000000000..16fdc6a53 --- /dev/null +++ b/config/model_configs/gpu_dyamond_nodiags.yml @@ -0,0 +1,13 @@ +anim: false +atmos_config_file: "config/gpu_configs/gpu_aquaplanet_dyamond.yml" +dt_cpl: 100 +energy_check: false +job_id: "gpu_dyamond_target_nodiags" +land_albedo_type: "map_temporal" +mode_name: "amip" +mono_surface: false +monthly_checkpoint: false +run_name: "gpu_dyamond_target_nodiags" +start_date: "19790301" +t_end: "12hours" +turb_flux_partition: "CombinedStateFluxes" diff --git a/experiments/AMIP/Manifest.toml b/experiments/AMIP/Manifest.toml index e1cbe1ea6..d3823d58a 100644 --- a/experiments/AMIP/Manifest.toml +++ b/experiments/AMIP/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.2" manifest_format = "2.0" -project_hash = "284db48dcd39d58ad8518daa0c1caded1552fa4a" +project_hash = "3a9507a7771542aa4f88e9c7e77e8f21b149253b" [[deps.ADTypes]] git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24" diff --git a/experiments/AMIP/Project.toml b/experiments/AMIP/Project.toml index f0c30ae8f..b15167fb4 100644 --- a/experiments/AMIP/Project.toml +++ b/experiments/AMIP/Project.toml @@ -2,6 +2,7 @@ ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" ArtifactWrappers = "a14bc488-3040-4b00-9dc1-f6467924858a" AtmosphericProfilesLibrary = "86bc3604-9858-485a-bdbe-831ec50de11d" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" ClimaAnalysis = "29b5916a-a76c-4e73-9657-3c8fd22e65e6" ClimaAtmos = "b2c96348-7fb7-4fe0-8da9-78d88439e717" diff --git a/experiments/AMIP/coupler_driver.jl b/experiments/AMIP/coupler_driver.jl index ef564b622..df2ca7250 100644 --- a/experiments/AMIP/coupler_driver.jl +++ b/experiments/AMIP/coupler_driver.jl @@ -23,6 +23,36 @@ Given that ClimaCore objects are heavily parametrized, non-abbreviated stacktrac so we force abbreviated stacktraces even in non-interactive runs. (See also `Base.type_limited_string_from_context()`) =# +using CUDA + +""" + show_memory_usage(comms_ctx, objects) + +Display the current memory footprint of the simulation, using an appropriate +method based on the device being used. + +In the GPU case, show the memory usage of the GPU. +In the CPU case, show the memory footprint of the provided object(s). +Note that these two cases provide different information, and should not be +directly compared. + +# Arguments +`comms_ctx`: the communication context being used to run the model +`objects`: Dict mapping objects whose memory footprint is displayed in the CPU case to their names +""" +function show_memory_usage(comms_ctx, objects) + if comms_ctx.device isa ClimaComms.CUDADevice + @info "Memory usage: $(CUDA.memory_status())" + elseif comms_ctx.device isa ClimaComms.AbstractCPUDevice + if ClimaComms.iamroot(comms_ctx) + for (obj, name) in objects + @info "Memory footprint of `$(name)` in bytes: $(Base.summarysize(obj))" + end + end + else + @warn "Invalid device type $device; cannot show memory usage." + end +end redirect_stderr(IOContext(stderr, :stacktrace_types_limited => Ref(false))) @@ -190,8 +220,12 @@ returns a `ComponentModelSimulation` object (see `Interfacer` docs for more deta This uses the `ClimaAtmos.jl` model, with parameterization options specified in the `config_dict_atmos` dictionary. =# +show_memory_usage(comms_ctx, Dict()) + ## init atmos model component atmos_sim = atmos_init(FT, config_dict_atmos); +show_memory_usage(comms_ctx, Dict(atmos_sim => name(atmos_sim))) + thermo_params = get_thermo_params(atmos_sim) # TODO: this should be shared by all models #610 #= @@ -222,6 +256,7 @@ land_fraction = mono = mono_surface, ) ) +show_memory_usage(comms_ctx, Dict(land_fraction => "land_fraction")) #= ### Surface Models: AMIP and SlabPlanet Modes @@ -332,6 +367,17 @@ if mode_name == "amip" update_field!(atmos_sim, Val(:co2), CO2_init) mode_specifics = (; name = mode_name, SST_info = SST_info, SIC_info = SIC_info, CO2_info = CO2_info) + show_memory_usage( + comms_ctx, + Dict( + land_sim => name(land_sim), + ocean_sim => name(ocean_sim), + ice_sim => name(ice_sim), + SST_info => "SST_info", + SIC_info => "SIC_info", + CO2_info => "CO2_info", + ), + ) elseif mode_name in ("slabplanet", "slabplanet_aqua", "slabplanet_terra") @@ -381,6 +427,10 @@ elseif mode_name in ("slabplanet", "slabplanet_aqua", "slabplanet_terra") )) mode_specifics = (; name = mode_name, SST_info = nothing, SIC_info = nothing) + show_memory_usage( + comms_ctx, + Dict(land_sim => name(land_sim), ocean_sim => name(ocean_sim), ice_sim => name(ice_sim)), + ) elseif mode_name == "slabplanet_eisenman" @@ -423,8 +473,13 @@ elseif mode_name == "slabplanet_eisenman" ) mode_specifics = (; name = mode_name, SST_info = nothing, SIC_info = nothing) + show_memory_usage( + comms_ctx, + Dict(land_sim => name(land_sim), ocean_sim => name(ocean_sim), ice_sim => name(ice_sim)), + ) end + #= ## Coupler Initialization The coupler needs to contain exchange information, manage the calendar and be able to access all component models. It can also optionally @@ -452,6 +507,7 @@ coupler_field_names = ( ) coupler_fields = NamedTuple{coupler_field_names}(ntuple(i -> ClimaCore.Fields.zeros(boundary_space), length(coupler_field_names))) +show_memory_usage(comms_ctx, Dict(coupler_fields => "coupler_fields")) ## model simulations model_sims = (atmos_sim = atmos_sim, ice_sim = ice_sim, land_sim = land_sim, ocean_sim = ocean_sim); @@ -484,7 +540,8 @@ monthly_2d_diags = init_diagnostics( name_tag = "monthly_mean_2d_", ) -diagnostics = (monthly_3d_diags, monthly_2d_diags) +diagnostics = () #(monthly_3d_diags, monthly_2d_diags) +show_memory_usage(comms_ctx, Dict(diagnostics => "diagnostics")) #= ## Initialize Conservation Checks @@ -503,6 +560,8 @@ if energy_check ) conservation_checks = (; energy = EnergyConservationCheck(model_sims), water = WaterConservationCheck(model_sims)) end +show_memory_usage(comms_ctx, Dict(conservation_checks => "conservation_checks")) + #= ## Initialize Callbacks @@ -523,6 +582,7 @@ checkpoint_cb = update_firstdayofmonth!_cb = MonthlyCallback(dt = FT(1), func = update_firstdayofmonth!, ref_date = [dates.date1[1]], active = true) callbacks = (; checkpoint = checkpoint_cb, update_firstdayofmonth! = update_firstdayofmonth!_cb) +show_memory_usage(comms_ctx, Dict(callbacks => "callbacks")) #= ## Initialize Coupled Simulation @@ -550,6 +610,7 @@ cs = CoupledSimulation{FT}( callbacks, dir_paths, ); +show_memory_usage(comms_ctx, Dict(cs => "cs")) #= ## Restart component model states if specified @@ -626,6 +687,8 @@ reinit_model_sims!(cs.model_sims) import_atmos_fields!(cs.fields, cs.model_sims, cs.boundary_space, turbulent_fluxes) update_model_sims!(cs.model_sims, cs.fields, turbulent_fluxes) +show_memory_usage(comms_ctx, Dict(cs => "cs")) + #= ## Coupling Loop @@ -726,6 +789,7 @@ end #hide ## run the coupled simulation solve_coupler!(cs); +show_memory_usage(comms_ctx, Dict(cs => "cs")) #= ## Postprocessing @@ -825,6 +889,7 @@ if ClimaComms.iamroot(comms_ctx) Leaderboard.plot_biases(atmos_sim.integrator.p.output_dir, compare_vars, cs.dates.date; output_path) end end + show_memory_usage(comms_ctx, Dict(cs => "cs")) if isinteractive() ## clean up for interactive runs, retain all output otherwise diff --git a/perf/Manifest.toml b/perf/Manifest.toml index 6515b38e0..c1285d7db 100644 --- a/perf/Manifest.toml +++ b/perf/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.2" manifest_format = "2.0" -project_hash = "ed46c461d51248cd9cbdd956368072e478bae235" +project_hash = "b45e4e9d311af3db6eb01b9c25b9517209fbcc29" [[deps.ADTypes]] git-tree-sha1 = "016833eb52ba2d6bea9fcb50ca295980e728ee24" @@ -307,7 +307,7 @@ uuid = "d934ef94-cdd4-4710-83d6-720549644b70" version = "0.3.14" [[deps.ClimaCoupler]] -deps = ["ClimaAtmos", "ClimaComms", "ClimaCore", "ClimaCoreTempestRemap", "ClimaLand", "ClimaParams", "Dates", "DocStringExtensions", "Insolation", "JLD2", "NCDatasets", "Plots", "SciMLBase", "StaticArrays", "Statistics", "SurfaceFluxes", "TempestRemap_jll", "Thermodynamics"] +deps = ["CUDA", "ClimaAtmos", "ClimaComms", "ClimaCore", "ClimaCoreTempestRemap", "ClimaLand", "ClimaParams", "Dates", "DocStringExtensions", "Insolation", "JLD2", "NCDatasets", "Plots", "SciMLBase", "StaticArrays", "Statistics", "SurfaceFluxes", "TempestRemap_jll", "Thermodynamics"] path = ".." uuid = "4ade58fe-a8da-486c-bd89-46df092ec0c7" version = "0.1.0" diff --git a/perf/Project.toml b/perf/Project.toml index e064cb0b8..be1bbf416 100644 --- a/perf/Project.toml +++ b/perf/Project.toml @@ -2,6 +2,7 @@ ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" ArtifactWrappers = "a14bc488-3040-4b00-9dc1-f6467924858a" AtmosphericProfilesLibrary = "86bc3604-9858-485a-bdbe-831ec50de11d" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" ClimaAtmos = "b2c96348-7fb7-4fe0-8da9-78d88439e717" ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d" ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" diff --git a/src/Utilities.jl b/src/Utilities.jl index 1867a76b6..df44b054b 100644 --- a/src/Utilities.jl +++ b/src/Utilities.jl @@ -25,12 +25,12 @@ function swap_space!(field_out, field_in::Fields.Field) end """ - get_device(parsed_args) + get_device(parsed_args) -Returns the device on which the model is being run +Returns the device on which the model is being run -# Arguments -- `parsed_args`: dictionary containing a "device" flag which decides which device to run on +# Arguments +- `parsed_args`: dictionary containing a "device" flag which decides which device to run on """ function get_device(parsed_args) if parsed_args["device"] == "auto" @@ -46,11 +46,11 @@ end """ - get_comms_context(parsed_args) + get_comms_context(parsed_args) -Sets up the appropriate ClimaComms context for the device the model is to be run on +Sets up the appropriate ClimaComms context for the device the model is to be run on -# Arguments +# Arguments `parsed_args`: dictionary containing a "device" flag whcih decides which device context is needed """ function get_comms_context(parsed_args) diff --git a/test/component_model_tests/climaatmos_standalone/atmos_driver.jl b/test/component_model_tests/climaatmos_standalone/atmos_driver.jl index b67dc4428..657f33369 100644 --- a/test/component_model_tests/climaatmos_standalone/atmos_driver.jl +++ b/test/component_model_tests/climaatmos_standalone/atmos_driver.jl @@ -1,12 +1,22 @@ using ClimaComms using Logging -using ClimaAtmos redirect_stderr(IOContext(stderr, :stacktrace_types_limited => Ref(false))) import ClimaAtmos as CA import Random +using ClimaCoupler Random.seed!(1234) +pkg_dir = pkgdir(ClimaCoupler) +atmos_config_file = joinpath(pkg_dir, "test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_dyamond.yml") +comms_ctx = ClimaComms.context(ClimaComms.device()) +@show typeof(comms_ctx) + +config = CA.AtmosConfig(atmos_config_file; comms_ctx = comms_ctx) + +OUTPUT_DIR = joinpath(pkg_dir, "test/component_model_tests/climaatmos_standalone/output/longrun_aquaplanet_dyamond_artifacts/") +mkpath(OUTPUT_DIR) + if !(@isdefined config) config = CA.AtmosConfig() end diff --git a/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_dyamond.yml b/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_dyamond.yml new file mode 100644 index 000000000..053590933 --- /dev/null +++ b/test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_dyamond.yml @@ -0,0 +1,19 @@ +h_elem: 60 +z_max: 55000.0 +z_elem: 63 +dz_bottom: 30.0 +dz_top: 3000.0 +moist: "equil" +precip_model: "0M" +cloud_model: "grid_scale" +rad: "allskywithclear" +idealized_insolation: false +dt_rad: "1hours" +vert_diff: "true" +surface_setup: "DefaultMoninObukhov" +rayleigh_sponge: true +dt_save_state_to_disk: "3hours" +dt: "50secs" +t_end: "4hours" +job_id: "longrun_aquaplanet_dyamond" +toml: [toml/longrun_aquaplanet_dyamond.toml] diff --git a/toml/longrun_aquaplanet_dyamond.toml b/toml/longrun_aquaplanet_dyamond.toml new file mode 100644 index 000000000..c7f080fe6 --- /dev/null +++ b/toml/longrun_aquaplanet_dyamond.toml @@ -0,0 +1,6 @@ +[zd_rayleigh] +value = 35000.0 + +[alpha_rayleigh_uh] +alias = "alpha_rayleigh_uh" +value = 0.0