From cb05f3d06adb19a7dc95d6c681c825b7d126508d Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Mon, 18 Mar 2024 16:17:33 -0700 Subject: [PATCH 1/7] use nsys for target amip, slabplanet --- .buildkite/pipeline.yml | 12 ++++++++---- .../gpu_amip_target_topo_diagedmf_shortrun.yml | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f35cf213e..4e2fd1a37 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -380,9 +380,11 @@ steps: slurm_mem: 20GB slurm_gpus: 1 - - label: "GPU Slabplanet: albedo from static map" + - label: "GPU Slabplanet: albedo from static map (nsys)" key: "gpu_slabplanet_albedo_static_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml" + command: > + nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_slabplanet_albedo_static_map_artifacts + julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" agents: slurm_mem: 20GB @@ -405,9 +407,11 @@ steps: slurm_mem: 20GB slurm_gpus: 1 - - label: "GPU AMIP target: topography and diagnostic EDMF" + - label: "GPU AMIP target: topography and diagnostic EDMF (nsys)" key: "gpu_amip_target_topo_diagedmf_shortrun" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml" + command: > + nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts + julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" agents: slurm_mem: 20GB diff --git a/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml b/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml index 129e3f740..c07a7ec75 100644 --- a/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml +++ b/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml @@ -19,7 +19,7 @@ netcdf_output_at_levels: true run_name: "gpu_amip_target_topo_diagedmf_shortrun" start_date: "19790301" surface_setup: "PrescribedSurface" -t_end: "200secs" +t_end: "12hours" topo_smoothing: true topography: "Earth" turb_flux_partition: "CombinedStateFluxes" From a27e9f68dea99bffbaebf7f57da708d399c5f07f Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Thu, 21 Mar 2024 20:06:56 -0700 Subject: [PATCH 2/7] update output paths --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4e2fd1a37..91c231eba 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -383,7 +383,7 @@ steps: - label: "GPU Slabplanet: albedo from static map (nsys)" key: "gpu_slabplanet_albedo_static_map" command: > - nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_slabplanet_albedo_static_map_artifacts + nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/ julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" agents: @@ -410,7 +410,7 @@ steps: - label: "GPU AMIP target: topography and diagnostic EDMF (nsys)" key: "gpu_amip_target_topo_diagedmf_shortrun" command: > - nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts + nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/ julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" agents: From a0834bf8ad3c34346b15500efd57b2030e061b99 Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Thu, 21 Mar 2024 20:07:41 -0700 Subject: [PATCH 3/7] comment out other jobs --- .buildkite/pipeline.yml | 668 ++++++++++++++++++++-------------------- 1 file changed, 332 insertions(+), 336 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 91c231eba..4daa3829d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -75,310 +75,310 @@ steps: - wait - - group: "Unit Tests" - steps: - - label: "MPI Regridder unit tests" - key: "regridder_mpi_tests" - command: "srun julia --color=yes --project=test/ test/mpi_tests/regridder_mpi_tests.jl --config_file $MPI_CONFIG_PATH/regridder_mpi.yml" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - NPROCS: 2 - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "MPI BCReader unit tests" - key: "bcreader_mpi_tests" - command: "srun julia --color=yes --project=test/ test/mpi_tests/bcreader_mpi_tests.jl --run_name bcreader_mpi --job_id bcreader_mpi" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "MPI Checkpointer unit tests" - key: "checkpointer_mpi_tests" - command: "srun julia --color=yes --project=test/ test/mpi_tests/checkpointer_mpi_tests.jl --run_name checkpointer_mpi --job_id checkpointer_mpi" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "MPI Utilities unit tests" - key: "utilities_mpi_tests" - command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_mpi --job_id utilities_mpi" - timeout_in_minutes: 20 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "GPU Utilities unit tests" - key: "utilities_gpu_tests" - command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_gpu --job_id utilities_gpu" - agents: - slurm_mem: 5GB - slurm_gpus: 1 - - - label: "Perf flame graph diff tests" - command: "julia --color=yes --project=perf/ perf/flame_test.jl --run_name flame_test --job_id flame_perf_target" - timeout_in_minutes: 5 - agents: - slurm_mem: 16GB - - - group: "Integration Tests" - steps: - - # Drivers for release > - - # SLABPLANET - - # Slabplanet default: - # - this is the most lightweight example with conservation and visual checks, with CLI specification as follows - # - numerics: dt = dt_cpl = 200s, nelem = 4 - # - physics: bulk aerodynamic surface fluxes, gray radiation, idealized insolation, equil moisture model, 0-moment microphysics - # - input data: monotonous remapping (land mask, SST, SIC) - # - slurm: unthreaded, 1 ntask - # - diagnostics: check and plot energy conservation, output plots after 9 days - - label: "Slabplanet: default" - key: "slabplanet_default" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_default.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_default_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: default with Float32" - key: "slabplanet_ft32" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_ft32.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_ft32_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: partitioned turbulent fluxes" - key: "slabplanet_partitioned_fluxes" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_partitioned_fluxes.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_partitioned_fluxes_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: non-monotonous surface remap" - key: "slabplanet_non-monotonous" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_nonmono.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_nonmono_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: albedo from static map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_static_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_static_map_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: albedo from temporal map" - key: "slabplanet_albedo_temporal_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_temporal_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_temporal_map_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: albedo from function" - key: "slabplanet_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: eisenman sea ice" - key: "slabplanet_eisenman" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_eisenman.yml" - artifact_paths: "experiments/AMIP/output/slabplanet_eisenman/slabplanet_eisenman_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "Slabplanet: extra atmos diagnostics" - key: "slabplanet_atmos_diags" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_atmos_diags.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_atmos_diags_artifacts/*" - agents: - slurm_mem: 20GB - - # AMIP - - # ... - - - # PERFORMANCE - - # slabplanet default: track unthreaded performance (alloc tests, flame graph, flame graph diff, build history) - - label: ":rocket: Slabplanet: default (unthreaded)" - key: "slabplanet_unthreaded" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_unthreaded.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/default_unthreaded_artifacts/*" - env: - FLAME_PLOT: "" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: ":rocket: Slabplanet: default (unthreaded) - flame graph and allocation tests" - command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_default_unthreaded.yml" - artifact_paths: "perf/output/perf_default_unthreaded/*" - agents: - slurm_mem: 20GB - - - label: ":rocket: Slabplanet: default (unthreaded) - flame graph diff" - command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_default_unthreaded.yml" - artifact_paths: "perf/output/perf_diff_default_unthreaded/*" - agents: - slurm_mem: 20GB - - # < end Drivers for release - - # CLIMACORE EXPERIMENTS - - - label: "sea_breeze" - command: "julia --color=yes --project=experiments/ClimaCore/sea_breeze experiments/ClimaCore/sea_breeze/run.jl" - artifact_paths: "experiments/ClimaCore/sea_breeze/output/*" - agents: - slurm_mem: 20GB - - - label: "heat-diffusion" - command: "julia --color=yes --project=experiments/ClimaCore/ experiments/ClimaCore/heat-diffusion/run.jl" - artifact_paths: "experiments/ClimaCore/output/heat-diffusion_artifacts/*" - agents: - slurm_mem: 20GB - - # AMIP AND SLABPLANET EXPERIMENTS - - - label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_mono.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/default_mono_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - - label: "Moist earth with slab surface - notmono: monin gray no_sponge idealinsol freq_dt_cpl notmono" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_notmono.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/default_notmono_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - # - label: "Moist earth with slab surface - test: monin allsky sponge idealinsol infreq_dt_cpl" - # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab_test1 --job_id target_params_in_slab_test1" # Unconverged SF (reproduced locally); works with 200s dt_cpl - # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test1_artifacts/total_energy*.png" - - - label: "Moist earth with slab surface - test: bulk allsky sponge realinsol infreq_dt_cpl" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test2.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test2_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - - label: "Moist earth with slab surface - test: monin gray sponge realinsol infreq_dt_cpl" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test3.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test3_artifacts/total_energy*.png" - agents: - slurm_mem: 20GB - - # breaking: - # - label: "Moist earth with slab surface - monin allsky no_sponge idealinsol infreq_dt_cpl" - # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge false --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --idealized_insolation true --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab1" - # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab1_artifacts/total_energy*.png" - - - label: "AMIP target: albedo from function" - key: "target_amip_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/amip/target_amip_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - - - label: "AMIP - Float64 + hourly checkpoint" - key: "amip" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_artifacts/*" - env: - FLAME_PLOT: "" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "AMIP - Float64 + hourly checkpoint + co2" - key: "coarse_single_ft64_hourly_checkpoints_co2" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints_co2.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_co2_artifacts/*" - env: - FLAME_PLOT: "" - BUILD_HISTORY_HANDLE: "" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "AMIP - Float64 test" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_artifacts/*" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "AMIP - Float32 test" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft32.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft32_artifacts/*" - agents: - slurm_ntasks: 1 - slurm_mem: 20GB - - - label: "MPI AMIP" - command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_mpi_n2.yml" - artifact_paths: "experiments/AMIP/output/amip/coarse_mpi_n2_artifacts/*" - timeout_in_minutes: 240 - env: - CLIMACORE_DISTRIBUTED: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: "batch script" - command: "sbatch test/mpi_tests/local_checks.sh" - - # short high-res performance test - - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph - key: "unthreaded_amip_fine" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_n1_shortrun.yml" - artifact_paths: "experiments/AMIP/output/amip/target_amip_n1_shortrun_artifacts/*" - env: - BUILD_HISTORY_HANDLE: "" - agents: - slurm_mem: 20GB - - # PERFORMANCE RUNS: flame graphs + allocation tests - - - label: ":rocket: flame graph and allocation tests: perf_coarse_single_ft64" - command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_coarse_single_ft64.yml" - artifact_paths: "perf/output/perf_coarse_single_ft64/*" - agents: - slurm_mem: 20GB - - - label: ":rocket: performance: flame graph diff: perf_diff_coarse_single_ft64" - command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_coarse_single_ft64.yml" - artifact_paths: "perf/output/perf_diff_coarse_single_ft64/*" - agents: - slurm_mem: 20GB + # - group: "Unit Tests" + # steps: + # - label: "MPI Regridder unit tests" + # key: "regridder_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/mpi_tests/regridder_mpi_tests.jl --config_file $MPI_CONFIG_PATH/regridder_mpi.yml" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # NPROCS: 2 + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "MPI BCReader unit tests" + # key: "bcreader_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/mpi_tests/bcreader_mpi_tests.jl --run_name bcreader_mpi --job_id bcreader_mpi" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "MPI Checkpointer unit tests" + # key: "checkpointer_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/mpi_tests/checkpointer_mpi_tests.jl --run_name checkpointer_mpi --job_id checkpointer_mpi" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "MPI Utilities unit tests" + # key: "utilities_mpi_tests" + # command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_mpi --job_id utilities_mpi" + # timeout_in_minutes: 20 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "GPU Utilities unit tests" + # key: "utilities_gpu_tests" + # command: "srun julia --color=yes --project=test/ test/utilities_tests.jl --run_name utilities_gpu --job_id utilities_gpu" + # agents: + # slurm_mem: 5GB + # slurm_gpus: 1 + + # - label: "Perf flame graph diff tests" + # command: "julia --color=yes --project=perf/ perf/flame_test.jl --run_name flame_test --job_id flame_perf_target" + # timeout_in_minutes: 5 + # agents: + # slurm_mem: 16GB + + # - group: "Integration Tests" + # steps: + + # # Drivers for release > + + # # SLABPLANET + + # # Slabplanet default: + # # - this is the most lightweight example with conservation and visual checks, with CLI specification as follows + # # - numerics: dt = dt_cpl = 200s, nelem = 4 + # # - physics: bulk aerodynamic surface fluxes, gray radiation, idealized insolation, equil moisture model, 0-moment microphysics + # # - input data: monotonous remapping (land mask, SST, SIC) + # # - slurm: unthreaded, 1 ntask + # # - diagnostics: check and plot energy conservation, output plots after 9 days + # - label: "Slabplanet: default" + # key: "slabplanet_default" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_default.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_default_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: default with Float32" + # key: "slabplanet_ft32" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_ft32.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_ft32_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: partitioned turbulent fluxes" + # key: "slabplanet_partitioned_fluxes" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_partitioned_fluxes.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_partitioned_fluxes_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: non-monotonous surface remap" + # key: "slabplanet_non-monotonous" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_nonmono.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_nonmono_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: albedo from static map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_static_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_static_map_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: albedo from temporal map" + # key: "slabplanet_albedo_temporal_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_temporal_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_temporal_map_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: albedo from function" + # key: "slabplanet_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: eisenman sea ice" + # key: "slabplanet_eisenman" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_eisenman.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet_eisenman/slabplanet_eisenman_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "Slabplanet: extra atmos diagnostics" + # key: "slabplanet_atmos_diags" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_atmos_diags.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_atmos_diags_artifacts/*" + # agents: + # slurm_mem: 20GB + + # # AMIP + + # # ... + + + # # PERFORMANCE + + # # slabplanet default: track unthreaded performance (alloc tests, flame graph, flame graph diff, build history) + # - label: ":rocket: Slabplanet: default (unthreaded)" + # key: "slabplanet_unthreaded" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_unthreaded.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/default_unthreaded_artifacts/*" + # env: + # FLAME_PLOT: "" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: ":rocket: Slabplanet: default (unthreaded) - flame graph and allocation tests" + # command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_default_unthreaded.yml" + # artifact_paths: "perf/output/perf_default_unthreaded/*" + # agents: + # slurm_mem: 20GB + + # - label: ":rocket: Slabplanet: default (unthreaded) - flame graph diff" + # command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_default_unthreaded.yml" + # artifact_paths: "perf/output/perf_diff_default_unthreaded/*" + # agents: + # slurm_mem: 20GB + + # # < end Drivers for release + + # # CLIMACORE EXPERIMENTS + + # - label: "sea_breeze" + # command: "julia --color=yes --project=experiments/ClimaCore/sea_breeze experiments/ClimaCore/sea_breeze/run.jl" + # artifact_paths: "experiments/ClimaCore/sea_breeze/output/*" + # agents: + # slurm_mem: 20GB + + # - label: "heat-diffusion" + # command: "julia --color=yes --project=experiments/ClimaCore/ experiments/ClimaCore/heat-diffusion/run.jl" + # artifact_paths: "experiments/ClimaCore/output/heat-diffusion_artifacts/*" + # agents: + # slurm_mem: 20GB + + # # AMIP AND SLABPLANET EXPERIMENTS + + # - label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_mono.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/default_mono_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # - label: "Moist earth with slab surface - notmono: monin gray no_sponge idealinsol freq_dt_cpl notmono" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/default_notmono.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/default_notmono_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # # - label: "Moist earth with slab surface - test: monin allsky sponge idealinsol infreq_dt_cpl" + # # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab_test1 --job_id target_params_in_slab_test1" # Unconverged SF (reproduced locally); works with 200s dt_cpl + # # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test1_artifacts/total_energy*.png" + + # - label: "Moist earth with slab surface - test: bulk allsky sponge realinsol infreq_dt_cpl" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test2.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test2_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # - label: "Moist earth with slab surface - test: monin gray sponge realinsol infreq_dt_cpl" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_params_in_slab_test3.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab_test3_artifacts/total_energy*.png" + # agents: + # slurm_mem: 20GB + + # # breaking: + # # - label: "Moist earth with slab surface - monin allsky no_sponge idealinsol infreq_dt_cpl" + # # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge false --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --idealized_insolation true --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab1" + # # artifact_paths: "experiments/AMIP/output/slabplanet/target_params_in_slab1_artifacts/total_energy*.png" + + # - label: "AMIP target: albedo from function" + # key: "target_amip_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/amip/target_amip_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + + # - label: "AMIP - Float64 + hourly checkpoint" + # key: "amip" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_artifacts/*" + # env: + # FLAME_PLOT: "" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "AMIP - Float64 + hourly checkpoint + co2" + # key: "coarse_single_ft64_hourly_checkpoints_co2" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints_co2.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_hourly_checkpoints_co2_artifacts/*" + # env: + # FLAME_PLOT: "" + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "AMIP - Float64 test" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft64.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft64_artifacts/*" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "AMIP - Float32 test" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_single_ft32.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_single_ft32_artifacts/*" + # agents: + # slurm_ntasks: 1 + # slurm_mem: 20GB + + # - label: "MPI AMIP" + # command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/coarse_mpi_n2.yml" + # artifact_paths: "experiments/AMIP/output/amip/coarse_mpi_n2_artifacts/*" + # timeout_in_minutes: 240 + # env: + # CLIMACORE_DISTRIBUTED: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: "batch script" + # command: "sbatch test/mpi_tests/local_checks.sh" + + # # short high-res performance test + # - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph + # key: "unthreaded_amip_fine" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/target_amip_n1_shortrun.yml" + # artifact_paths: "experiments/AMIP/output/amip/target_amip_n1_shortrun_artifacts/*" + # env: + # BUILD_HISTORY_HANDLE: "" + # agents: + # slurm_mem: 20GB + + # # PERFORMANCE RUNS: flame graphs + allocation tests + + # - label: ":rocket: flame graph and allocation tests: perf_coarse_single_ft64" + # command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_coarse_single_ft64.yml" + # artifact_paths: "perf/output/perf_coarse_single_ft64/*" + # agents: + # slurm_mem: 20GB + + # - label: ":rocket: performance: flame graph diff: perf_diff_coarse_single_ft64" + # command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_coarse_single_ft64.yml" + # artifact_paths: "perf/output/perf_diff_coarse_single_ft64/*" + # agents: + # slurm_mem: 20GB - group: "GPU integration tests" steps: - # GPU RUNS: slabplanet - - label: "GPU Slabplanet: albedo from function" - key: "gpu_slabplanet_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 + # # GPU RUNS: slabplanet + # - label: "GPU Slabplanet: albedo from function" + # key: "gpu_slabplanet_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 - label: "GPU Slabplanet: albedo from static map (nsys)" key: "gpu_slabplanet_albedo_static_map" @@ -390,22 +390,22 @@ steps: slurm_mem: 20GB slurm_gpus: 1 - - label: "GPU Slabplanet: albedo from temporal map" - key: "gpu_slabplanet_albedo_temporal_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_temporal_map.yml" - artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_temporal_map_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - # GPU RUNS: AMIP - - label: "GPU AMIP test: albedo from function" - key: "gpu_amip_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_function_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 + # - label: "GPU Slabplanet: albedo from temporal map" + # key: "gpu_slabplanet_albedo_temporal_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_temporal_map.yml" + # artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_temporal_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # # GPU RUNS: AMIP + # - label: "GPU AMIP test: albedo from function" + # key: "gpu_amip_albedo_function" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_function_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 - label: "GPU AMIP target: topography and diagnostic EDMF (nsys)" key: "gpu_amip_target_topo_diagedmf_shortrun" @@ -417,25 +417,21 @@ steps: slurm_mem: 20GB slurm_gpus: 1 - - label: "GPU AMIP: albedo from static map" - key: "gpu_amip_albedo_static_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_static_map.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_static_map_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - label: "GPU AMIP: albedo from temporal map" - key: "gpu_amip_albedo_temporal_map" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_temporal_map.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_temporal_map_artifacts/*" - agents: - slurm_mem: 20GB - slurm_gpus: 1 - - - - - wait + # - label: "GPU AMIP: albedo from static map" + # key: "gpu_amip_albedo_static_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_static_map.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_static_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 + + # - label: "GPU AMIP: albedo from temporal map" + # key: "gpu_amip_albedo_temporal_map" + # command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_temporal_map.yml" + # artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_temporal_map_artifacts/*" + # agents: + # slurm_mem: 20GB + # slurm_gpus: 1 - wait From 6183f9e6e5b69961e32e4c35a84a7bcf1a28bc2c Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Thu, 21 Mar 2024 21:55:20 -0700 Subject: [PATCH 4/7] fix nsys paths --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4daa3829d..7bd5ddda0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -383,7 +383,7 @@ steps: - label: "GPU Slabplanet: albedo from static map (nsys)" key: "gpu_slabplanet_albedo_static_map" command: > - nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/ + nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/gpu_slabplanet_albedo_static_map_nsys julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" agents: @@ -410,7 +410,7 @@ steps: - label: "GPU AMIP target: topography and diagnostic EDMF (nsys)" key: "gpu_amip_target_topo_diagedmf_shortrun" command: > - nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/ + nsys profile --trace=nvtx,cuda --output=experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/gpu_amip_target_topo_diagedmf_shortrun_nsys julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" agents: From c60150692d7dad39e6190c89ba43cea6ba5aee9d Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Thu, 21 Mar 2024 22:10:35 -0700 Subject: [PATCH 5/7] add NVTX.@annotate --- experiments/AMIP/components/atmosphere/climaatmos.jl | 2 +- experiments/AMIP/components/land/climaland_bucket.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/AMIP/components/atmosphere/climaatmos.jl b/experiments/AMIP/components/atmosphere/climaatmos.jl index 08ba2e717..0508895c3 100644 --- a/experiments/AMIP/components/atmosphere/climaatmos.jl +++ b/experiments/AMIP/components/atmosphere/climaatmos.jl @@ -206,7 +206,7 @@ function update_field!(sim::ClimaAtmosSimulation, ::Val{:turbulent_fluxes}, fiel end # extensions required by FieldExchanger -step!(sim::ClimaAtmosSimulation, t) = step!(sim.integrator, t - sim.integrator.t, true) +NVTX.@annotate step!(sim::ClimaAtmosSimulation, t) = step!(sim.integrator, t - sim.integrator.t, true) reinit!(sim::ClimaAtmosSimulation) = reinit!(sim.integrator) function update_sim!(atmos_sim::ClimaAtmosSimulation, csf, turbulent_fluxes) diff --git a/experiments/AMIP/components/land/climaland_bucket.jl b/experiments/AMIP/components/land/climaland_bucket.jl index 4b4d9cd64..48397eefd 100644 --- a/experiments/AMIP/components/land/climaland_bucket.jl +++ b/experiments/AMIP/components/land/climaland_bucket.jl @@ -207,7 +207,7 @@ function update_field!(sim::BucketSimulation, ::Val{:turbulent_moisture_flux}, f end # extensions required by FieldExchanger -step!(sim::BucketSimulation, t) = step!(sim.integrator, t - sim.integrator.t, true) +NVTX.@annotate step!(sim::BucketSimulation, t) = step!(sim.integrator, t - sim.integrator.t, true) reinit!(sim::BucketSimulation) = reinit!(sim.integrator) # extensions required by FluxCalculator (partitioned fluxes) From 64a8c0c3837385d120ae056bdf575b443d6a7978 Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Fri, 22 Mar 2024 13:36:43 -0700 Subject: [PATCH 6/7] import NVTX --- experiments/AMIP/components/atmosphere/climaatmos.jl | 1 + experiments/AMIP/components/land/climaland_bucket.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/experiments/AMIP/components/atmosphere/climaatmos.jl b/experiments/AMIP/components/atmosphere/climaatmos.jl index 0508895c3..dc2cc4ce1 100644 --- a/experiments/AMIP/components/atmosphere/climaatmos.jl +++ b/experiments/AMIP/components/atmosphere/climaatmos.jl @@ -1,6 +1,7 @@ # atmos_init: for ClimaAtmos pre-AMIP interface using StaticArrays using Statistics: mean +import NVTX import ClimaAtmos as CA import ClimaAtmos: CT1, CT2, CT12, CT3, C3, C12, unit_basis_vector_data, ⊗ diff --git a/experiments/AMIP/components/land/climaland_bucket.jl b/experiments/AMIP/components/land/climaland_bucket.jl index 48397eefd..e821f9514 100644 --- a/experiments/AMIP/components/land/climaland_bucket.jl +++ b/experiments/AMIP/components/land/climaland_bucket.jl @@ -5,6 +5,7 @@ import Thermodynamics as TD using Dates: DateTime using ClimaComms: AbstractCommsContext import ClimaParams +import NVTX import ClimaLand using ClimaLand.Bucket: BucketModel, BucketModelParameters, AbstractAtmosphericDrivers, AbstractRadiativeDrivers From 98652a3c38776fa1c12fab4e125fbf1476c940d8 Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Fri, 22 Mar 2024 15:01:18 -0700 Subject: [PATCH 7/7] add non-nsys runs --- .buildkite/pipeline.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7bd5ddda0..9fad19888 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -380,6 +380,14 @@ steps: # slurm_mem: 20GB # slurm_gpus: 1 + - label: "GPU Slabplanet: albedo from static map (without nsys)" + key: "gpu_slabplanet_albedo_static_map_nonsys" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_slabplanet_albedo_static_map.yml" + artifact_paths: "experiments/AMIP/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" + agents: + slurm_mem: 20GB + slurm_gpus: 1 + - label: "GPU Slabplanet: albedo from static map (nsys)" key: "gpu_slabplanet_albedo_static_map" command: > @@ -407,6 +415,14 @@ steps: # slurm_mem: 20GB # slurm_gpus: 1 + - label: "GPU AMIP target: topography and diagnostic EDMF (without nsys)" + key: "gpu_amip_target_topo_diagedmf_shortrun_nonsys" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" + agents: + slurm_mem: 20GB + slurm_gpus: 1 + - label: "GPU AMIP target: topography and diagnostic EDMF (nsys)" key: "gpu_amip_target_topo_diagedmf_shortrun" command: >