Skip to content

Commit

Permalink
add dyamond
Browse files Browse the repository at this point in the history
add nocouple run

clean

rm slurm_mem_per_cpu

rm slurm_mem

fix atmos config path

add atmos toml path

incr slurm_mem

up pip

try

flag fix

rm limiter usage

clean

clean

clean

try 140d
  • Loading branch information
LenkaNovak committed Oct 6, 2023
1 parent 15314cf commit fd64e3f
Show file tree
Hide file tree
Showing 13 changed files with 92 additions and 34 deletions.
48 changes: 30 additions & 18 deletions .buildkite/longruns/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,18 +63,21 @@ steps:
slurm_nodes: 1
slurm_mem_per_cpu: 16G

# - label: "MPI AMIP FINE: longrun" # unstable after 6 months
# key: "amip_longrun_fine"
# command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_longrun_fine --coupled true --anim true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 140days --dt_save_to_sol 10days --mono_surface false --precip_model 0M"
# artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_fine_artifacts/*"
# env:
# CLIMACORE_DISTRIBUTED: "MPI"
# BUILD_HISTORY_HANDLE: ""
# agents:
# slurm_ntasks_per_node: 16
# slurm_nodes: 4
# slurm_mem_per_cpu: 16G
# DYAMOND AMIP: 1 day (convection resolving)

- label: "MPI AMIP SUPERFINE: dyamond_target"
key: "dyamond_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G

# mid-resolution AMIP: longrun (140 days)
- label: "MPI AMIP FINE: target longrun"
key: "amip_longrun_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_longrun_target.yml"
Expand All @@ -87,7 +90,7 @@ steps:
slurm_nodes: 4
slurm_mem_per_cpu: 16G

# MPI performance scaling (10 days)
# mid-resolution AMIP: MPI performance scaling (10 days)
- label: "MPI AMIP FINE: n64"
key: "mpi_amip_fine_n64"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_n64_shortrun.yml"
Expand Down Expand Up @@ -147,14 +150,25 @@ steps:
slurm_nodes: 1
slurm_mem_per_cpu: 16G

- label: "MPI AMIP FINE: n1 no couple" # sim time = Δt_cpl (~ benchmarking with standalone models)
key: "mpi_amip_fine_n1_nocouple"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_n1_shortrun_nocouple.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/amip_n1_shortrun_nocouple_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

# mpi_amip_fine_n1 flame graph report (NB: arguments passed from the ci pipeline.yml)
- label: ":rocket: performance: flame graph diff: perf_target_amip_n1_shortrun"
command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_target_amip_n1_shortrun.yml"
artifact_paths: "perf/output/perf_diff_target_amip_n1_shortrun/*"
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16GB
slurm_mem_per_cpu: 16G

- wait

Expand All @@ -175,10 +189,8 @@ steps:
command:
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"

# - label: ":envelope: Slack report: AMIP fine benchmark"
# command:
# - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_fine_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Longrun"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"

- label: ":envelope: Slack report: target AMIP"
command:
Expand All @@ -187,4 +199,4 @@ steps:
- label: ":envelope: Slack report: Flame Diff"
command:
- slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff.html -m png -n amip_fine_flamegraphdiff -x "AMIP Longrun FlameGraphDiff"
- slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff_self_count.html -m png -n amip_fine_flamegraphdiffself -x "AMIP Longrun FlameGraphDiffSelf"
- slack-upload -c "#coupler-report" -f perf/output/perf_diff_target_amip_n1_shortrun/flame_diff_self_count.html -m png -n amip_fine_flamegraphdiffself -x "AMIP Longrun FlameGraphDiffSelf"
4 changes: 2 additions & 2 deletions config/longrun_configs/amip_longrun_target.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ kappa_4: 3e16
rayleigh_sponge: true
alpha_rayleigh_uh: 0
dt: "150secs"
t_end: "100days" # TODO this has been decreased from 140 days to avoid instability
t_end: "140days" # TODO this has been decreased from 140 days to avoid instability
job_id: "amip_longrun_target"
dt_save_to_sol: "5days"
dt_save_to_disk: "1days"
apply_limiter: true
apply_limiter: false
2 changes: 1 addition & 1 deletion config/longrun_configs/amip_n1_shortrun.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ mode_name: "amip"
t_end: "1days"
dt_save_to_sol: "100days"
mono_surface: false
apply_limiter: true
apply_limiter: false
precip_model: "0M"
24 changes: 24 additions & 0 deletions config/longrun_configs/amip_n1_shortrun_nocouple.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
job_id: "amip_n1_shortrun"
run_name: "amip_n1_shortrun"
moist: "equil"
vert_diff: "true"
rad: "gray"
z_elem: 50
dz_top: 3000
dz_bottom: 30
h_elem: 16
kappa_4: 1e16
z_stretch: false
rayleigh_sponge: true
alpha_rayleigh_uh: 0
alpha_rayleigh_w: 10
dt_cpl: 86400
dt: "150secs"
dt_rad: "1hours"
energy_check: false
mode_name: "amip"
t_end: "1days"
dt_save_to_sol: "100days"
mono_surface: false
apply_limiter: false
precip_model: "0M"
2 changes: 1 addition & 1 deletion config/longrun_configs/amip_n2_shortrun.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ mode_name: "amip"
t_end: "1days"
dt_save_to_sol: "100days"
mono_surface: false
apply_limiter: true
apply_limiter: false
precip_model: "0M"
2 changes: 1 addition & 1 deletion config/longrun_configs/amip_n32_shortrun.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ mode_name: "amip"
t_end: "10days"
dt_save_to_sol: "100days"
mono_surface: false
apply_limiter: true
apply_limiter: false
precip_model: "0M"
2 changes: 1 addition & 1 deletion config/longrun_configs/amip_n64_shortrun.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ mode_name: "amip"
t_end: "10days"
dt_save_to_sol: "100days"
mono_surface: false
apply_limiter: true
apply_limiter: false
precip_model: "0M"
2 changes: 1 addition & 1 deletion config/longrun_configs/amip_n8_shortrun.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ mode_name: "amip"
t_end: "10days"
dt_save_to_sol: "100days"
mono_surface: false
apply_limiter: true
apply_limiter: false
precip_model: "0M"
14 changes: 14 additions & 0 deletions config/longrun_configs/dyamond_target.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
run_name: "dyamond_target"
anim: true
dt_cpl: 50
energy_check: false
mode_name: "amip"
mono_surface: false
t_end: "1days"
job_id: "dyamond_target"
dt_save_to_sol: "0.5days"
dt_save_to_disk: "0.5days"
turb_flux_partition: "PartitionedStateFluxes"
atmos_config_file: "config/longrun_configs/longrun_aquaplanet_dyamond.yml"
atmos_toml_file: "toml/longrun_aquaplanet_dyamond.toml"
monthly_checkpoint: false
2 changes: 1 addition & 1 deletion config/longrun_configs/slabplanet_default_longrun.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ mono_surface: true
h_elem: 6
precip_model: "0M"
anim: true
apply_limiter: true
apply_limiter: false
job_id: "slabplanet_default_longrun"
2 changes: 1 addition & 1 deletion config/model_configs/interactive_debug.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ dt_save_restart: "5days"
precip_model: "0M"
run_name: "interactive_debug_run"
job_id: "interactive_debug_run"
monthly_checkpoint: true
monthly_checkpoint: true
2 changes: 2 additions & 0 deletions experiments/AMIP/modular/cli_options.jl
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ function argparse_settings()
default = "PrescribedSurface"
"--atmos_config_file"
help = "A yaml file used to set the atmospheric model configuration. If nothing is specified, the default configuration is used."
"--atmos_toml_file"
help = "A toml file used to overwrite the atmospheric model parameters. If nothing is specified, the default parameters are used."
# ClimaLSM specific
"--land_albedo_type"
help = "Access land surface albedo information from data file. [`function`, `map_static`, `map_temporal`]"
Expand Down
20 changes: 13 additions & 7 deletions experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,25 @@ Returns the specified atmospheric configuration (`atmos_config_dict`) overwitten
in the coupler dictionary (`config_dict`).
"""
function get_atmos_config(coupler_dict)
atmos_file = coupler_dict["atmos_config_file"]
atmos_config_file = coupler_dict["atmos_config_file"]
# override default or specified configs with coupler arguments, and set the correct atmos config_file
if isnothing(atmos_file)
if isnothing(atmos_config_file)
@info "Using Atmos default configuration"
merge(CA.default_config_dict(), coupler_dict, Dict("config_file" => atmos_file))
atmos_config = merge(CA.default_config_dict(), coupler_dict, Dict("config_file" => atmos_config_file))
else
@info "Using Atmos configuration from $atmos_file"
merge(
CA.override_default_config(joinpath(pkgdir(CA), atmos_file)),
@info "Using Atmos configuration from $atmos_config_file"
atmos_config = merge(
CA.override_default_config(joinpath(pkgdir(CA), atmos_config_file)),
coupler_dict,
Dict("config_file" => atmos_file),
Dict("config_file" => atmos_config_file),
)
end
atmos_toml_file = coupler_dict["atmos_toml_file"]
if !isnothing(atmos_toml_file)
@info "Overwriting Atmos parameters from $atmos_toml_file"
atmos_config = merge(atmos_config, Dict("toml" => [joinpath(pkgdir(CA), atmos_toml_file)]))
end
return atmos_config
end

function atmos_init(::Type{FT}, atmos_config_dict::Dict) where {FT}
Expand Down

0 comments on commit fd64e3f

Please sign in to comment.