diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 61e64b8b7f1..2d3218b9865 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -56,1466 +56,1466 @@ steps: - wait - - group: "Regression tests" - steps: - - - label: ":computer: Ensure mse tables are reset when necessary" - command: "julia --color=yes --project=examples regression_tests/test_reset.jl" - - - group: "Radiation" - steps: - - - label: ":computer: single column radiative equilibrium gray" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml - --job_id single_column_radiative_equilibrium_gray - artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" - - - label: ":computer: single column radiative equilibrium clearsky" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml - --job_id single_column_radiative_equilibrium_clearsky - artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" - - - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml - --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp - artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" - - - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml - --job_id single_column_radiative_equilibrium_allsky_idealized_clouds - artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" - - - group: "Precipitation" - steps: - - label: ":umbrella: 1-moment precipitation sanity test single column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_precipitation_test.yml - --job_id single_column_precipitation_test - artifact_paths: "single_column_precipitation_test/output_active/*" - - - group: "Gravity wave" - steps: - - - label: ":computer: non-orographic gravity wave parameterization unit test 3d" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" - artifact_paths: "nonorographic_gravity_wave_test_3d/*" - agents: - slurm_mem: 20GB - - - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" - artifact_paths: "nonorographic_gravity_wave_test_mima/*" - agents: - slurm_mem: 20GB - - - label: ":computer: non-orographic gravity wave parameterization unit test single column" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" - artifact_paths: "nonorographic_gravity_wave_test_single_column/*" - - - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" - artifact_paths: "orographic_gravity_wave_test_baseflux/*" - - - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" - command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" - artifact_paths: "orographic_gravity_wave_test_3d/*" - - - label: ":computer: single column non-orographic gravity wave parameterization" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml - --job_id single_column_nonorographic_gravity_wave - artifact_paths: "single_column_nonorographic_gravity_wave/*" - - - group: "Column Examples" - steps: - - - label: ":computer: single column hydrostatic balance float64" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml - --job_id single_column_hydrostatic_balance_ft64 - artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" - - - group: "Box Examples" - steps: - - - label: ":computer: Box hydrostatic balance (ρe_tot)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_hydrostatic_balance_rhoe.yml - --job_id box_hydrostatic_balance_rhoe - artifact_paths: "box_hydrostatic_balance_rhoe/output_active/*" - - - label: ":computer: 3D density current" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_density_current_test.yml - --job_id box_density_current_test - artifact_paths: "box_density_current_test/output_active/*" - - - label: ":computer: Box (ρe_tot) rcemipii with diagnostic edmf" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/rcemipii_box_diagnostic_edmfx.yml - --job_id rcemipii_box_diagnostic_edmfx - artifact_paths: "rcemipii_box_diagnostic_edmfx/output_active/*" - soft_fail: true - - - group: "Plane Examples" - steps: - - label: ":computer: Agnesi linear hydrostatic mountain experiment (uniform)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_agnesi_mountain_test_uniform.yml - --job_id plane_agnesi_mountain_test_uniform - artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*" - - - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_agnesi_mountain_test_stretched.yml - --job_id plane_agnesi_mountain_test_stretched - artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*" - - - label: ":computer: Density current experiment" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_density_current_test.yml - --job_id plane_density_current_test - artifact_paths: "plane_density_current_test/output_active/*" - - - label: ":computer: Analytic No Topography Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_test.yml - --job_id plane_analytic_no_topography_test - artifact_paths: "plane_analytic_no_topography_test/output_active/*" - - - label: ":computer: Analytic No Topography Test (2D, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_float32_test.yml - --job_id plane_analytic_no_topography_float32_test - artifact_paths: "plane_analytic_no_topography_float32_test/output_active/*" - - - label: ":computer: Analytic Cosine Hills Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_test.yml - --job_id plane_analytic_cosine_hills_test - artifact_paths: "plane_analytic_cosine_hills_test/output_active/*" - - - label: ":computer: Analytic Cosine Hills Test (2D, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_float32_test.yml - --job_id plane_analytic_cosine_hills_float32_test - artifact_paths: "plane_analytic_cosine_hills_float32_test/output_active/*" - - - group: "Conservation check" - steps: - - - label: ":computer: baroclinic wave check conservation" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation.yml - --job_id sphere_baroclinic_wave_conservation - artifact_paths: "sphere_baroclinic_wave_conservation/output_active/*" - - - label: ":computer: baroclinic wave check conservation float64" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation_ft64.yml - --job_id sphere_baroclinic_wave_conservation_ft64 - artifact_paths: "sphere_baroclinic_wave_conservation_ft64/output_active/*" - - - label: ":computer: baroclinic wave check conservation with sources" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation_source.yml - --job_id sphere_baroclinic_wave_conservation_source - artifact_paths: "sphere_baroclinic_wave_conservation_source/output_active/*" - - - group: "Sphere Examples (Dycore)" - steps: - - - label: ":computer: hydrostatic balance (ρe) float64" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_hydrostatic_balance_rhoe_ft64.yml - --job_id sphere_hydrostatic_balance_rhoe_ft64 - artifact_paths: "sphere_hydrostatic_balance_rhoe_ft64/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: baroclinic wave (ρe)" - key: sphere_baroclinic_wave_rhoe - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe.yml - --job_id sphere_baroclinic_wave_rhoe - artifact_paths: "sphere_baroclinic_wave_rhoe/output_active/*" - - - label: ":computer: no lim baroclinic wave (ρe) equilmoist" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist.yml - --job_id sphere_baroclinic_wave_rhoe_equilmoist - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id sphere_baroclinic_wave_rhoe_equilmoist - --out_dir sphere_baroclinic_wave_rhoe_equilmoist/output_active - artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: no lim baroclinic wave (ρe) equilmoist (deep sphere)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml - --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist - --out_dir deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active - artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - # Add this back when we figure out what to do with SSP and zalesak - # - label: ":computer: SSP zalesak tracer & energy upwind baroclinic wave (ρe_tot) equilmoist" - # command: > - # julia --color=yes --project=examples examples/hybrid/driver.jl - # --config_file $CONFIG_PATH/$$JOB_NAME.yml - # --job_id JOB_NAME - # artifact_paths: "$$JOB_NAME/output_active/*" - # agents: - # slurm_mem: 64GB - # env: - # JOB_NAME: "sphere_zalesak_upwind_tracer_energy_ssp_baroclinic_wave_rhoe_equilmoist" - - - label: ":computer: held suarez (ρe) hightop" - key: sphere_held_suarez_rhoe_hightop - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_hightop.yml - --job_id sphere_held_suarez_rhoe_hightop - artifact_paths: "sphere_held_suarez_rhoe_hightop/output_active/*" - - - label: ":computer: no lim held suarez (ρe) equilmoist hightop sponge" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml - --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge - --out_dir sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active - artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*" - agents: - slurm_constraint: icelake|cascadelake|skylake|epyc - - - group: "Sphere Examples (Aquaplanet)" - steps: - - - label: ":umbrella: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res.yml - --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res - artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (raw_topo) high top zonally asymmetric" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml - --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric - --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active - artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - label: ":computer: aquaplanet (ρe_tot) nonequilmoist allsky radiation monin_obukhov varying insolation high top 1-moment" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_nonequilmoist_allsky.yml - --job_id sphere_aquaplanet_rhoe_nonequilmoist_allsky - artifact_paths: "sphere_aquaplanet_rhoe_nonequilmoist_allsky/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet (ρe_tot) slabocean equilmoist allsky radiation monin_obukhov varying insolation" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean.yml - --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean - artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: aquaplanet (ρe_tot) rcemipii with diagnostic edmf" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/rcemipii_sphere_diagnostic_edmfx.yml - --job_id rcemipii_sphere_diagnostic_edmfx - artifact_paths: "rcemipii_sphere_diagnostic_edmfx/output_active/*" - agents: - slurm_mem: 20GB - - - group: "Sphere Examples (Topography)" - steps: - - - label: ":computer: baroclinic wave (ρe) topography (dcmip)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml - --job_id sphere_baroclinic_wave_rhoe_topography_dcmip_rs - artifact_paths: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active/*" - - - label: ":computer: held suarez (ρe) topography (dcmip)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_topography_dcmip.yml - --job_id sphere_held_suarez_rhoe_topography_dcmip - artifact_paths: "sphere_held_suarez_rhoe_topography_dcmip/output_active/*" - - - label: ":computer: held suarez (ρe) equilmoist topography (dcmip)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_topography_dcmip.yml - --job_id sphere_held_suarez_rhoe_equilmoist_topography_dcmip - artifact_paths: "sphere_held_suarez_rhoe_equilmoist_topography_dcmip/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":computer: Diagnostic DCMIP200 surface elevation spectra" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200.yml - --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200 - artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200/output_active/*" - - - label: ":computer: Diagnostic Earth surface elevation spectra" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth.yml - --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth - artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*" - - - group: "MPI Examples" - steps: - - - label: ":computer: Prep restart for MPI" - key: "mpi_baro_wave_make_restart" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml - --job_id mpi_make_restart - artifact_paths: "mpi_make_restart/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - - label: ":computer: Test restart for MPI baroclinic wave" - key: "restart_mpi_baro_wave" - depends_on: "mpi_baro_wave_make_restart" - command: > - tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active - - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave_rhoe.yml - --job_id restart_mpi_baroclinic_wave_rhoe - artifact_paths: "restart_mpi_baroclinic_wave_rhoe/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - timeout_in_minutes: 20 - soft_fail: true - #retry: - # automatic: true - - - label: ":computer: MPI no lim aquaplanet (ρe) equilmoist clearsky radiation" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky.yml - --job_id mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky - artifact_paths: "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16GB - - - label: ":computer: Prep for calling remap pipeline" - key: "prep_remap" - command: > - srun julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $MPI_CONFIG_PATH/prep_remap.yml - --job_id prep_remap - artifact_paths: "prep_remap/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 16G - - - label: ":computer: Exercise remap pipeline" - key: "remap_pipeline" - depends_on: "prep_remap" - command: > - tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active - - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl - --data_dir prep_remap/output_active --out_dir remap_pipeline_output - artifact_paths: "remap_pipeline_output/*" - - - group: "Configs" - steps: - - # TODO: we should somehow decouple this unit test from the perf env / scripts - - label: ":computer: checkbounds" - command: > - julia --color=yes --check-bounds=yes --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/checkbounds.yml - --job_id checkbounds - artifact_paths: "checkbounds/output_active/*" - agents: - slurm_mem: 20GB - - - group: "Calibration" - steps: - - label: "Calibration interface unit tests" - command: julia --project=calibration/test calibration/test/interface.jl - - label: "end to end test" - command: julia --project=calibration/test calibration/test/e2e_test.jl - artifact_paths: "calibration_end_to_end_test/*" - - - group: "Diagnostic EDMFX" - steps: - - - label: ":genie: Diagnostic EDMFX test in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml - --job_id diagnostic_edmfx_test_box - artifact_paths: "diagnostic_edmfx_test_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX GABLS in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml - --job_id diagnostic_edmfx_gabls_box - artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX Bomex in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml - --job_id diagnostic_edmfx_bomex_box - artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml - --job_id diagnostic_edmfx_bomex_stretched_box - artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml - --job_id diagnostic_edmfx_dycoms_rf01_explicit_box - artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml - --job_id diagnostic_edmfx_dycoms_rf01_box - artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX DYCOMS_RF02 in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf02_box.yml - --job_id diagnostic_edmfx_dycoms_rf02_box - artifact_paths: "diagnostic_edmfx_dycoms_rf02_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX Rico in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml - --job_id diagnostic_edmfx_rico_box - artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX TRMM in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml - --job_id diagnostic_edmfx_trmm_box - artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml - --job_id diagnostic_edmfx_trmm_stretched_box - artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml - --job_id diagnostic_edmfx_trmm_box_0M - artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Diagnostic EDMFX aquaplanet with TKE" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml - --job_id diagnostic_edmfx_aquaplanet - - julia --color=yes --project=examples regression_tests/test_mse.jl - --job_id diagnostic_edmfx_aquaplanet - --out_dir diagnostic_edmfx_aquaplanet/output_active - artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" - agents: - slurm_mem: 20GB - slurm_constraint: icelake|cascadelake|skylake|epyc - - - group: "Prognostic EDMFX" - steps: - - - label: ":genie: Prognostic EDMFX advection test in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_adv_test_column.yml - --job_id prognostic_edmfx_adv_test_column - artifact_paths: "prognostic_edmfx_adv_test_column/output_active/*" - - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX simple plume test in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_simpleplume_column.yml - --job_id prognostic_edmfx_simpleplume_column - artifact_paths: "prognostic_edmfx_simpleplume_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX GABLS in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_gabls_column.yml - --job_id prognostic_edmfx_gabls_column - artifact_paths: "prognostic_edmfx_gabls_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX Bomex in a column (pi group)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_bomex_pigroup_column.yml - --job_id prognostic_edmfx_bomex_pigroup_column - artifact_paths: "prognostic_edmfx_bomex_pigroup_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX Bomex with precribed TKE in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_bomex_fixtke_column.yml - --job_id prognostic_edmfx_bomex_fixtke_column - artifact_paths: "prognostic_edmfx_bomex_fixtke_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX Bomex stretched grid in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_bomex_stretched_column.yml - --job_id prognostic_edmfx_bomex_stretched_column - artifact_paths: "prognostic_edmfx_bomex_stretched_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX Bomex in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_bomex_column.yml - --job_id prognostic_edmfx_bomex_column - artifact_paths: "prognostic_edmfx_bomex_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX Bomex in a column (implicit)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_bomex_implicit_column.yml - --job_id prognostic_edmfx_bomex_column_implicit - artifact_paths: "prognostic_edmfx_bomex_column_implicit/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX Dycoms RF01 in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_dycoms_rf01_column.yml - --job_id prognostic_edmfx_dycoms_rf01_column - artifact_paths: "prognostic_edmfx_dycoms_rf01_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Prognostic EDMFX Rico in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_rico_column.yml - --job_id prognostic_edmfx_rico_column - artifact_paths: "prognostic_edmfx_rico_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":umbrella: Prognostic EDMFX TRMM in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_trmm_column.yml - --job_id prognostic_edmfx_trmm_column - artifact_paths: "prognostic_edmfx_trmm_column/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX TRMM with 0-moment in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_trmm_column_0M.yml - --job_id prognostic_edmfx_trmm_column_0M - artifact_paths: "prognostic_edmfx_trmm_column_0M/output_active/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX GCM driven in a column" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_gcmdriven_column.yml - --job_id prognostic_edmfx_gcmdriven_column - artifact_paths: "prognostic_edmfx_gcmdriven_column/output_active/*" - agents: - slurm_mem: 20GB - soft_fail: true - - - label: ":genie: Prognostic EDMFX Bomex in a box" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_bomex_box.yml - --job_id prognostic_edmfx_bomex_box - artifact_paths: "prognostic_edmfx_bomex_box/*" - agents: - slurm_mem: 20GB - - - label: ":genie: Prognostic EDMFX aquaplanet" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet.yml - --job_id prognostic_edmfx_aquaplanet - artifact_paths: "prognostic_edmfx_aquaplanet/output_active/*" - agents: - slurm_mem: 20GB - - - group: "GPU" - steps: - - - label: "GPU: baroclinic wave" - key: "gpu_baroclinic_wave_rhoe" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/gpu_baroclinic_wave_rhoe.yml - --job_id gpu_baroclinic_wave_rhoe - artifact_paths: "gpu_implicit_barowave_ref/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 16G - slurm_gpus: 1 - - - label: "GPU: compare BW with CPU" - command: > - tar xvf sphere_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C sphere_baroclinic_wave_rhoe - - tar xvf gpu_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C gpu_baroclinic_wave_rhoe - - julia --color=yes --project=examples post_processing/compare_outputs.jl - --output_folder_1 sphere_baroclinic_wave_rhoe - --output_folder_2 gpu_baroclinic_wave_rhoe --t_end 10days - --compare_state false - depends_on: - - "sphere_baroclinic_wave_rhoe" - - "gpu_baroclinic_wave_rhoe" - - - label: "GPU: GPU dry baroclinic wave" - key: "target_gpu_implicit_baroclinic_wave" - command: - - mkdir -p target_gpu_implicit_baroclinic_wave - - > - nsys profile --trace=nvtx,cuda --output=target_gpu_implicit_baroclinic_wave/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml - --job_id target_gpu_implicit_baroclinic_wave - artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 32G - - - label: "GPU: GPU dry baroclinic wave - 4 gpus" - key: "target_gpu_implicit_baroclinic_wave_4process" - command: - # nsys profile --trace=nvtx,cuda,mpi --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} - - mkdir -p target_gpu_implicit_baroclinic_wave_4process - - > - srun --cpu-bind=threads --cpus-per-task=4 - julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl - --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave_4process.yml - --job_id target_gpu_implicit_baroclinic_wave_4process - artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus_per_task: 1 - slurm_cpus_per_task: 4 - slurm_ntasks: 4 - slurm_mem: 32G - - - label: "GPU: GPU moist Held-Suarez" - command: - - > - nsys profile --trace=nvtx,cuda --output=central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/central_gpu_hs_rhoe_equil_55km_nz63_0M.yml - --job_id central_gpu_hs_rhoe_equil_55km_nz63_0M - artifact_paths: "central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16G - - - label: "GPU: GPU moist Held-Suarez cloud diagnostics per stage" - command: - - > - nsys profile --trace=nvtx,cuda --output=central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M.yml - --job_id central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M - artifact_paths: "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 16G - - - label: ":umbrella: GPU: gpu_aquaplanet_dyamond" - command: - - mkdir -p gpu_aquaplanet_dyamond - - > - nsys profile --trace=nvtx,cuda --output=gpu_aquaplanet_dyamond/output_active/report - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file ${CONFIG_PATH}/gpu_aquaplanet_dyamond.yml - --job_id gpu_aquaplanet_dyamond - artifact_paths: "gpu_aquaplanet_dyamond/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Diagnostic EDMFX aquaplanet" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/gpu_diagnostic_edmfx_aquaplanet.yml - --job_id gpu_diagnostic_edmfx_aquaplanet - artifact_paths: "gpu_diagnostic_edmfx_aquaplanet/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20G - - - label: "GPU: Prognostic EDMFX aquaplanet" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/gpu_prognostic_edmfx_aquaplanet.yml - --job_id gpu_prognostic_edmfx_aquaplanet - artifact_paths: "gpu_prognostic_edmfx_aquaplanet/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 20G - - - label: "GPU: Analytic No Topography Test (2D, Long Duration)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_long_test.yml - --job_id gpu_plane_analytic_no_topography_long_test - artifact_paths: "gpu_plane_analytic_no_topography_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (2D, Long Duration, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_long_float32_test.yml - --job_id gpu_plane_analytic_no_topography_long_float32_test - artifact_paths: "gpu_plane_analytic_no_topography_long_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (2D, Long Duration, No Hyperdiff)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_no_hyperdiff_long_test.yml - --job_id gpu_plane_analytic_no_topography_no_hyperdiff_long_test - artifact_paths: "gpu_plane_analytic_no_topography_no_hyperdiff_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (2D, Long Duration, No Hyperdiff, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_no_hyperdiff_long_float32_test.yml - --job_id gpu_plane_analytic_no_topography_no_hyperdiff_long_float32_test - artifact_paths: "gpu_plane_analytic_no_topography_no_hyperdiff_long_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (2D, Long Duration, Discrete Balance)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_discrete_balance_long_test.yml - --job_id gpu_plane_analytic_no_topography_discrete_balance_long_test - artifact_paths: "gpu_plane_analytic_no_topography_discrete_balance_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (2D, Long Duration, Discrete Balance, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_no_topography_discrete_balance_long_float32_test.yml - --job_id gpu_plane_analytic_no_topography_discrete_balance_long_float32_test - artifact_paths: "gpu_plane_analytic_no_topography_discrete_balance_long_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (Extruded 2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/extruded_plane_analytic_no_topography_test.yml - --job_id gpu_extruded_plane_analytic_no_topography_test - artifact_paths: "gpu_extruded_plane_analytic_no_topography_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (3D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_analytic_no_topography_test.yml - --job_id gpu_box_analytic_no_topography_test - artifact_paths: "gpu_box_analytic_no_topography_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic No Topography Test (3D, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_analytic_no_topography_float32_test.yml - --job_id gpu_box_analytic_no_topography_float32_test - artifact_paths: "gpu_box_analytic_no_topography_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_long_test.yml - --job_id gpu_plane_analytic_cosine_hills_long_test - artifact_paths: "gpu_plane_analytic_cosine_hills_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_long_float32_test.yml - --job_id gpu_plane_analytic_cosine_hills_long_float32_test - artifact_paths: "gpu_plane_analytic_cosine_hills_long_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Stronger Sponge)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_strong_sponge_long_test.yml - --job_id gpu_plane_analytic_cosine_hills_strong_sponge_long_test - artifact_paths: "gpu_plane_analytic_cosine_hills_strong_sponge_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Weaker Sponge)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_weak_sponge_long_test.yml - --job_id gpu_plane_analytic_cosine_hills_weak_sponge_long_test - artifact_paths: "gpu_plane_analytic_cosine_hills_weak_sponge_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Higher Domain Top)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_high_top_long_test.yml - --job_id gpu_plane_analytic_cosine_hills_high_top_long_test - artifact_paths: "gpu_plane_analytic_cosine_hills_high_top_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - group: "Regression tests" + # steps: + + # - label: ":computer: Ensure mse tables are reset when necessary" + # command: "julia --color=yes --project=examples regression_tests/test_reset.jl" + + # - group: "Radiation" + # steps: + + # - label: ":computer: single column radiative equilibrium gray" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_gray.yml + # --job_id single_column_radiative_equilibrium_gray + # artifact_paths: "single_column_radiative_equilibrium_gray/output_active/*" + + # - label: ":computer: single column radiative equilibrium clearsky" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky.yml + # --job_id single_column_radiative_equilibrium_clearsky + # artifact_paths: "single_column_radiative_equilibrium_clearsky/output_active/*" + + # - label: ":computer: single column radiative equilibrium clearsky prognostic surface temperature" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_clearsky_prognostic_surface_temp.yml + # --job_id single_column_radiative_equilibrium_clearsky_prognostic_surface_temp + # artifact_paths: "single_column_radiative_equilibrium_clearsky_prognostic_surface_temp/output_active/*" + + # - label: ":computer: single column radiative equilibrium allsky idealized clouds varying insolation" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_radiative_equilibrium_allsky_idealized_clouds.yml + # --job_id single_column_radiative_equilibrium_allsky_idealized_clouds + # artifact_paths: "single_column_radiative_equilibrium_allsky_idealized_clouds/output_active/*" + + # - group: "Precipitation" + # steps: + # - label: ":umbrella: 1-moment precipitation sanity test single column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_precipitation_test.yml + # --job_id single_column_precipitation_test + # artifact_paths: "single_column_precipitation_test/output_active/*" + + # - group: "Gravity wave" + # steps: + + # - label: ":computer: non-orographic gravity wave parameterization unit test 3d" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl" + # artifact_paths: "nonorographic_gravity_wave_test_3d/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: non-orographic gravity wave parameterization test with MiMA output" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_mima.jl" + # artifact_paths: "nonorographic_gravity_wave_test_mima/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: non-orographic gravity wave parameterization unit test single column" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_single_column.jl" + # artifact_paths: "nonorographic_gravity_wave_test_single_column/*" + + # - label: ":computer: orographic gravity wave parameterization unit test for base flux calculation" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_baseflux.jl" + # artifact_paths: "orographic_gravity_wave_test_baseflux/*" + + # - label: ":computer: orographic gravity wave parameterization unit test for 3d calculation" + # command: "julia --color=yes --project=examples test/parameterized_tendencies/gravity_wave/orographic_gravity_wave/ogwd_3d.jl" + # artifact_paths: "orographic_gravity_wave_test_3d/*" + + # - label: ":computer: single column non-orographic gravity wave parameterization" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_nonorographic_gravity_wave.yml + # --job_id single_column_nonorographic_gravity_wave + # artifact_paths: "single_column_nonorographic_gravity_wave/*" + + # - group: "Column Examples" + # steps: + + # - label: ":computer: single column hydrostatic balance float64" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/single_column_hydrostatic_balance_ft64.yml + # --job_id single_column_hydrostatic_balance_ft64 + # artifact_paths: "single_column_hydrostatic_balance_ft64/output_active/*" + + # - group: "Box Examples" + # steps: + + # - label: ":computer: Box hydrostatic balance (ρe_tot)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/box_hydrostatic_balance_rhoe.yml + # --job_id box_hydrostatic_balance_rhoe + # artifact_paths: "box_hydrostatic_balance_rhoe/output_active/*" + + # - label: ":computer: 3D density current" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/box_density_current_test.yml + # --job_id box_density_current_test + # artifact_paths: "box_density_current_test/output_active/*" + + # - label: ":computer: Box (ρe_tot) rcemipii with diagnostic edmf" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/rcemipii_box_diagnostic_edmfx.yml + # --job_id rcemipii_box_diagnostic_edmfx + # artifact_paths: "rcemipii_box_diagnostic_edmfx/output_active/*" + # soft_fail: true - - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Higher Domain Top, Higher Sponge)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_cosine_hills_high_top_high_sponge_long_test.yml - --job_id gpu_plane_analytic_cosine_hills_high_top_high_sponge_long_test - artifact_paths: "gpu_plane_analytic_cosine_hills_high_top_high_sponge_long_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + - group: "Plane Examples" + steps: + # - label: ":computer: Agnesi linear hydrostatic mountain experiment (uniform)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_agnesi_mountain_test_uniform.yml + # --job_id plane_agnesi_mountain_test_uniform + # artifact_paths: "plane_agnesi_mountain_test_uniform/output_active/*" - - label: "GPU: Analytic Cosine Hills Test (Extruded 2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/extruded_plane_analytic_cosine_hills_test.yml - --job_id gpu_extruded_plane_analytic_cosine_hills_test - artifact_paths: "gpu_extruded_plane_analytic_cosine_hills_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: ":computer: Agnesi linear hydrostatic mountain experiment (stretched)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_agnesi_mountain_test_stretched.yml + # --job_id plane_agnesi_mountain_test_stretched + # artifact_paths: "plane_agnesi_mountain_test_stretched/output_active/*" - - label: "GPU: Analytic Cosine Hills Test (3D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_analytic_cosine_hills_test.yml - --job_id gpu_box_analytic_cosine_hills_test - artifact_paths: "gpu_box_analytic_cosine_hills_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: ":computer: Density current experiment" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_density_current_test.yml + # --job_id plane_density_current_test + # artifact_paths: "plane_density_current_test/output_active/*" - - label: "GPU: Analytic Cosine Hills Test (3D, Float32)" + - label: ":computer: Analytic No Topography Test (2D)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/box_analytic_cosine_hills_float32_test.yml - --job_id gpu_box_analytic_cosine_hills_float32_test - artifact_paths: "gpu_box_analytic_cosine_hills_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + --config_file $CONFIG_PATH/plane_analytic_no_topography_test.yml + --job_id plane_analytic_no_topography_test + artifact_paths: "plane_analytic_no_topography_test/output_active/*" - - label: "GPU: Analytic Agnesi Mountain Test (2D)" + - label: ":computer: Analytic No Topography Test (2D, Float32)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_agnesi_mountain_test.yml - --job_id gpu_plane_analytic_agnesi_mountain_test - artifact_paths: "gpu_plane_analytic_agnesi_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + --config_file $CONFIG_PATH/plane_analytic_no_topography_float32_test.yml + --job_id plane_analytic_no_topography_float32_test + artifact_paths: "plane_analytic_no_topography_float32_test/output_active/*" - - label: "GPU: Analytic Huge Schar Mountain Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_huge_schar_mountain_test.yml - --job_id gpu_plane_analytic_huge_schar_mountain_test - artifact_paths: "gpu_plane_analytic_huge_schar_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: ":computer: Analytic Cosine Hills Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_test.yml + # --job_id plane_analytic_cosine_hills_test + # artifact_paths: "plane_analytic_cosine_hills_test/output_active/*" - - label: "GPU: Analytic Big Schar Mountain Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_big_schar_mountain_test.yml - --job_id gpu_plane_analytic_big_schar_mountain_test - artifact_paths: "gpu_plane_analytic_big_schar_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: ":computer: Analytic Cosine Hills Test (2D, Float32)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_float32_test.yml + # --job_id plane_analytic_cosine_hills_float32_test + # artifact_paths: "plane_analytic_cosine_hills_float32_test/output_active/*" + + # - group: "Conservation check" + # steps: + + # - label: ":computer: baroclinic wave check conservation" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation.yml + # --job_id sphere_baroclinic_wave_conservation + # artifact_paths: "sphere_baroclinic_wave_conservation/output_active/*" + + # - label: ":computer: baroclinic wave check conservation float64" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation_ft64.yml + # --job_id sphere_baroclinic_wave_conservation_ft64 + # artifact_paths: "sphere_baroclinic_wave_conservation_ft64/output_active/*" + + # - label: ":computer: baroclinic wave check conservation with sources" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/sphere_baroclinic_wave_conservation_source.yml + # --job_id sphere_baroclinic_wave_conservation_source + # artifact_paths: "sphere_baroclinic_wave_conservation_source/output_active/*" + + # - group: "Sphere Examples (Dycore)" + # steps: + + # - label: ":computer: hydrostatic balance (ρe) float64" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_hydrostatic_balance_rhoe_ft64.yml + # --job_id sphere_hydrostatic_balance_rhoe_ft64 + # artifact_paths: "sphere_hydrostatic_balance_rhoe_ft64/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: baroclinic wave (ρe)" + # key: sphere_baroclinic_wave_rhoe + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe.yml + # --job_id sphere_baroclinic_wave_rhoe + # artifact_paths: "sphere_baroclinic_wave_rhoe/output_active/*" + + # - label: ":computer: no lim baroclinic wave (ρe) equilmoist" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_equilmoist.yml + # --job_id sphere_baroclinic_wave_rhoe_equilmoist + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id sphere_baroclinic_wave_rhoe_equilmoist + # --out_dir sphere_baroclinic_wave_rhoe_equilmoist/output_active + # artifact_paths: "sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: no lim baroclinic wave (ρe) equilmoist (deep sphere)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/deep_sphere_baroclinic_wave_rhoe_equilmoist.yml + # --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id deep_sphere_baroclinic_wave_rhoe_equilmoist + # --out_dir deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active + # artifact_paths: "deep_sphere_baroclinic_wave_rhoe_equilmoist/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # # Add this back when we figure out what to do with SSP and zalesak + # # - label: ":computer: SSP zalesak tracer & energy upwind baroclinic wave (ρe_tot) equilmoist" + # # command: > + # # julia --color=yes --project=examples examples/hybrid/driver.jl + # # --config_file $CONFIG_PATH/$$JOB_NAME.yml + # # --job_id JOB_NAME + # # artifact_paths: "$$JOB_NAME/output_active/*" + # # agents: + # # slurm_mem: 64GB + # # env: + # # JOB_NAME: "sphere_zalesak_upwind_tracer_energy_ssp_baroclinic_wave_rhoe_equilmoist" + + # - label: ":computer: held suarez (ρe) hightop" + # key: sphere_held_suarez_rhoe_hightop + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_hightop.yml + # --job_id sphere_held_suarez_rhoe_hightop + # artifact_paths: "sphere_held_suarez_rhoe_hightop/output_active/*" + + # - label: ":computer: no lim held suarez (ρe) equilmoist hightop sponge" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_hightop_sponge.yml + # --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id sphere_held_suarez_rhoe_equilmoist_hightop_sponge + # --out_dir sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active + # artifact_paths: "sphere_held_suarez_rhoe_equilmoist_hightop_sponge/output_active/*" + # agents: + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - group: "Sphere Examples (Aquaplanet)" + # steps: + + # - label: ":umbrella: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res.yml + # --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res + # artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_res/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet (ρe_tot) equil allsky monin_obukhov varying insol gravity wave (raw_topo) high top zonally asymmetric" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric.yml + # --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric + # --out_dir sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active + # artifact_paths: "sphere_aquaplanet_rhoe_equilmoist_allsky_gw_raw_zonallyasymmetric/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - label: ":computer: aquaplanet (ρe_tot) nonequilmoist allsky radiation monin_obukhov varying insolation high top 1-moment" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_aquaplanet_rhoe_nonequilmoist_allsky.yml + # --job_id sphere_aquaplanet_rhoe_nonequilmoist_allsky + # artifact_paths: "sphere_aquaplanet_rhoe_nonequilmoist_allsky/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet (ρe_tot) slabocean equilmoist allsky radiation monin_obukhov varying insolation" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean.yml + # --job_id aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean + # artifact_paths: "aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: aquaplanet (ρe_tot) rcemipii with diagnostic edmf" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/rcemipii_sphere_diagnostic_edmfx.yml + # --job_id rcemipii_sphere_diagnostic_edmfx + # artifact_paths: "rcemipii_sphere_diagnostic_edmfx/output_active/*" + # agents: + # slurm_mem: 20GB + + # - group: "Sphere Examples (Topography)" + # steps: + + # - label: ":computer: baroclinic wave (ρe) topography (dcmip)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml + # --job_id sphere_baroclinic_wave_rhoe_topography_dcmip_rs + # artifact_paths: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs/output_active/*" + + # - label: ":computer: held suarez (ρe) topography (dcmip)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_topography_dcmip.yml + # --job_id sphere_held_suarez_rhoe_topography_dcmip + # artifact_paths: "sphere_held_suarez_rhoe_topography_dcmip/output_active/*" + + # - label: ":computer: held suarez (ρe) equilmoist topography (dcmip)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_held_suarez_rhoe_equilmoist_topography_dcmip.yml + # --job_id sphere_held_suarez_rhoe_equilmoist_topography_dcmip + # artifact_paths: "sphere_held_suarez_rhoe_equilmoist_topography_dcmip/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":computer: Diagnostic DCMIP200 surface elevation spectra" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200.yml + # --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200 + # artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_dcmip200/output_active/*" + + # - label: ":computer: Diagnostic Earth surface elevation spectra" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth.yml + # --job_id sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth + # artifact_paths: "sphere_ssp_baroclinic_wave_rhoe_equilmoist_earth/output_active/*" + + # - group: "MPI Examples" + # steps: + + # - label: ":computer: Prep restart for MPI" + # key: "mpi_baro_wave_make_restart" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/mpi_make_restart.yml + # --job_id mpi_make_restart + # artifact_paths: "mpi_make_restart/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # - label: ":computer: Test restart for MPI baroclinic wave" + # key: "restart_mpi_baro_wave" + # depends_on: "mpi_baro_wave_make_restart" + # command: > + # tar xvf mpi_make_restart/output_active/hdf5_files.tar -C mpi_make_restart/output_active + + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/restart_mpi_baroclinic_wave_rhoe.yml + # --job_id restart_mpi_baroclinic_wave_rhoe + # artifact_paths: "restart_mpi_baroclinic_wave_rhoe/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + # timeout_in_minutes: 20 + # soft_fail: true + # #retry: + # # automatic: true + + # - label: ":computer: MPI no lim aquaplanet (ρe) equilmoist clearsky radiation" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky.yml + # --job_id mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky + # artifact_paths: "mpi_sphere_aquaplanet_rhoe_equilmoist_clearsky/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16GB + + # - label: ":computer: Prep for calling remap pipeline" + # key: "prep_remap" + # command: > + # srun julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $MPI_CONFIG_PATH/prep_remap.yml + # --job_id prep_remap + # artifact_paths: "prep_remap/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 16G + + # - label: ":computer: Exercise remap pipeline" + # key: "remap_pipeline" + # depends_on: "prep_remap" + # command: > + # tar xvf prep_remap/output_active/hdf5_files.tar -C prep_remap/output_active + + # julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl + # --data_dir prep_remap/output_active --out_dir remap_pipeline_output + # artifact_paths: "remap_pipeline_output/*" + + # - group: "Configs" + # steps: + + # # TODO: we should somehow decouple this unit test from the perf env / scripts + # - label: ":computer: checkbounds" + # command: > + # julia --color=yes --check-bounds=yes --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/checkbounds.yml + # --job_id checkbounds + # artifact_paths: "checkbounds/output_active/*" + # agents: + # slurm_mem: 20GB + + # - group: "Calibration" + # steps: + # - label: "Calibration interface unit tests" + # command: julia --project=calibration/test calibration/test/interface.jl + # - label: "end to end test" + # command: julia --project=calibration/test calibration/test/e2e_test.jl + # artifact_paths: "calibration_end_to_end_test/*" + + # - group: "Diagnostic EDMFX" + # steps: + + # - label: ":genie: Diagnostic EDMFX test in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_test_box.yml + # --job_id diagnostic_edmfx_test_box + # artifact_paths: "diagnostic_edmfx_test_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX GABLS in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_gabls_box.yml + # --job_id diagnostic_edmfx_gabls_box + # artifact_paths: "diagnostic_edmfx_gabls_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX Bomex in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_box.yml + # --job_id diagnostic_edmfx_bomex_box + # artifact_paths: "diagnostic_edmfx_bomex_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX Bomex stretched grid in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_bomex_stretched_box.yml + # --job_id diagnostic_edmfx_bomex_stretched_box + # artifact_paths: "diagnostic_edmfx_bomex_stretched_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box (explicit)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_explicit_box.yml + # --job_id diagnostic_edmfx_dycoms_rf01_explicit_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf01_explicit_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX DYCOMS_RF01 in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf01_box.yml + # --job_id diagnostic_edmfx_dycoms_rf01_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf01_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX DYCOMS_RF02 in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_dycoms_rf02_box.yml + # --job_id diagnostic_edmfx_dycoms_rf02_box + # artifact_paths: "diagnostic_edmfx_dycoms_rf02_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX Rico in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_rico_box.yml + # --job_id diagnostic_edmfx_rico_box + # artifact_paths: "diagnostic_edmfx_rico_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX TRMM in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box.yml + # --job_id diagnostic_edmfx_trmm_box + # artifact_paths: "diagnostic_edmfx_trmm_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Diagnostic EDMFX TRMM stretched grid in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_stretched_box.yml + # --job_id diagnostic_edmfx_trmm_stretched_box + # artifact_paths: "diagnostic_edmfx_trmm_stretched_box/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX TRMM 0M in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_trmm_box_0M.yml + # --job_id diagnostic_edmfx_trmm_box_0M + # artifact_paths: "diagnostic_edmfx_trmm_box_0M/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Diagnostic EDMFX aquaplanet with TKE" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet.yml + # --job_id diagnostic_edmfx_aquaplanet + + # julia --color=yes --project=examples regression_tests/test_mse.jl + # --job_id diagnostic_edmfx_aquaplanet + # --out_dir diagnostic_edmfx_aquaplanet/output_active + # artifact_paths: "diagnostic_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_mem: 20GB + # slurm_constraint: icelake|cascadelake|skylake|epyc + + # - group: "Prognostic EDMFX" + # steps: + + # - label: ":genie: Prognostic EDMFX advection test in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_adv_test_column.yml + # --job_id prognostic_edmfx_adv_test_column + # artifact_paths: "prognostic_edmfx_adv_test_column/output_active/*" + + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX simple plume test in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_simpleplume_column.yml + # --job_id prognostic_edmfx_simpleplume_column + # artifact_paths: "prognostic_edmfx_simpleplume_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX GABLS in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_gabls_column.yml + # --job_id prognostic_edmfx_gabls_column + # artifact_paths: "prognostic_edmfx_gabls_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX Bomex in a column (pi group)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_bomex_pigroup_column.yml + # --job_id prognostic_edmfx_bomex_pigroup_column + # artifact_paths: "prognostic_edmfx_bomex_pigroup_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX Bomex with precribed TKE in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_bomex_fixtke_column.yml + # --job_id prognostic_edmfx_bomex_fixtke_column + # artifact_paths: "prognostic_edmfx_bomex_fixtke_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX Bomex stretched grid in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_bomex_stretched_column.yml + # --job_id prognostic_edmfx_bomex_stretched_column + # artifact_paths: "prognostic_edmfx_bomex_stretched_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX Bomex in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_bomex_column.yml + # --job_id prognostic_edmfx_bomex_column + # artifact_paths: "prognostic_edmfx_bomex_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX Bomex in a column (implicit)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_bomex_implicit_column.yml + # --job_id prognostic_edmfx_bomex_column_implicit + # artifact_paths: "prognostic_edmfx_bomex_column_implicit/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX Dycoms RF01 in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_dycoms_rf01_column.yml + # --job_id prognostic_edmfx_dycoms_rf01_column + # artifact_paths: "prognostic_edmfx_dycoms_rf01_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Prognostic EDMFX Rico in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_rico_column.yml + # --job_id prognostic_edmfx_rico_column + # artifact_paths: "prognostic_edmfx_rico_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":umbrella: Prognostic EDMFX TRMM in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_trmm_column.yml + # --job_id prognostic_edmfx_trmm_column + # artifact_paths: "prognostic_edmfx_trmm_column/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX TRMM with 0-moment in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_trmm_column_0M.yml + # --job_id prognostic_edmfx_trmm_column_0M + # artifact_paths: "prognostic_edmfx_trmm_column_0M/output_active/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX GCM driven in a column" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_gcmdriven_column.yml + # --job_id prognostic_edmfx_gcmdriven_column + # artifact_paths: "prognostic_edmfx_gcmdriven_column/output_active/*" + # agents: + # slurm_mem: 20GB + # soft_fail: true + + # - label: ":genie: Prognostic EDMFX Bomex in a box" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_bomex_box.yml + # --job_id prognostic_edmfx_bomex_box + # artifact_paths: "prognostic_edmfx_bomex_box/*" + # agents: + # slurm_mem: 20GB + + # - label: ":genie: Prognostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/prognostic_edmfx_aquaplanet.yml + # --job_id prognostic_edmfx_aquaplanet + # artifact_paths: "prognostic_edmfx_aquaplanet/output_active/*" + # agents: + # slurm_mem: 20GB - - label: "GPU: Analytic Schar Mountain Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_test.yml - --job_id gpu_plane_analytic_schar_mountain_test - artifact_paths: "gpu_plane_analytic_schar_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + - group: "GPU" + steps: - - label: "GPU: Analytic Small Schar Mountain Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_small_schar_mountain_test.yml - --job_id gpu_plane_analytic_small_schar_mountain_test - artifact_paths: "gpu_plane_analytic_small_schar_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: "GPU: baroclinic wave" + # key: "gpu_baroclinic_wave_rhoe" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/gpu_baroclinic_wave_rhoe.yml + # --job_id gpu_baroclinic_wave_rhoe + # artifact_paths: "gpu_implicit_barowave_ref/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 16G + # slurm_gpus: 1 - - label: "GPU: Analytic Tiny Schar Mountain Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_tiny_schar_mountain_test.yml - --job_id gpu_plane_analytic_tiny_schar_mountain_test - artifact_paths: "gpu_plane_analytic_tiny_schar_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: "GPU: compare BW with CPU" + # command: > + # tar xvf sphere_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C sphere_baroclinic_wave_rhoe + + # tar xvf gpu_baroclinic_wave_rhoe/output_active/hdf5_files.tar -C gpu_baroclinic_wave_rhoe + + # julia --color=yes --project=examples post_processing/compare_outputs.jl + # --output_folder_1 sphere_baroclinic_wave_rhoe + # --output_folder_2 gpu_baroclinic_wave_rhoe --t_end 10days + # --compare_state false + # depends_on: + # - "sphere_baroclinic_wave_rhoe" + # - "gpu_baroclinic_wave_rhoe" + + # - label: "GPU: GPU dry baroclinic wave" + # key: "target_gpu_implicit_baroclinic_wave" + # command: + # - mkdir -p target_gpu_implicit_baroclinic_wave + # - > + # nsys profile --trace=nvtx,cuda --output=target_gpu_implicit_baroclinic_wave/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave.yml + # --job_id target_gpu_implicit_baroclinic_wave + # artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 32G + + # - label: "GPU: GPU dry baroclinic wave - 4 gpus" + # key: "target_gpu_implicit_baroclinic_wave_4process" + # command: + # # nsys profile --trace=nvtx,cuda,mpi --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK} + # - mkdir -p target_gpu_implicit_baroclinic_wave_4process + # - > + # srun --cpu-bind=threads --cpus-per-task=4 + # julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl + # --config_file ${GPU_CONFIG_PATH}/target_gpu_implicit_baroclinic_wave_4process.yml + # --job_id target_gpu_implicit_baroclinic_wave_4process + # artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus_per_task: 1 + # slurm_cpus_per_task: 4 + # slurm_ntasks: 4 + # slurm_mem: 32G + + # - label: "GPU: GPU moist Held-Suarez" + # command: + # - > + # nsys profile --trace=nvtx,cuda --output=central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/central_gpu_hs_rhoe_equil_55km_nz63_0M.yml + # --job_id central_gpu_hs_rhoe_equil_55km_nz63_0M + # artifact_paths: "central_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16G + + # - label: "GPU: GPU moist Held-Suarez cloud diagnostics per stage" + # command: + # - > + # nsys profile --trace=nvtx,cuda --output=central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M.yml + # --job_id central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M + # artifact_paths: "central_cloud_diag_gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 16G + + # - label: ":umbrella: GPU: gpu_aquaplanet_dyamond" + # command: + # - mkdir -p gpu_aquaplanet_dyamond + # - > + # nsys profile --trace=nvtx,cuda --output=gpu_aquaplanet_dyamond/output_active/report + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file ${CONFIG_PATH}/gpu_aquaplanet_dyamond.yml + # --job_id gpu_aquaplanet_dyamond + # artifact_paths: "gpu_aquaplanet_dyamond/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 - - label: "GPU: Analytic Teeny Tiny Schar Mountain Test (2D)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_teeny_tiny_schar_mountain_test.yml - --job_id gpu_plane_analytic_teeny_tiny_schar_mountain_test - artifact_paths: "gpu_plane_analytic_teeny_tiny_schar_mountain_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: "GPU: Diagnostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/gpu_diagnostic_edmfx_aquaplanet.yml + # --job_id gpu_diagnostic_edmfx_aquaplanet + # artifact_paths: "gpu_diagnostic_edmfx_aquaplanet/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20G - - label: "GPU: Analytic Schar Mountain Test (2D, Float32)" - command: > - julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_float32_test.yml - --job_id gpu_plane_analytic_schar_mountain_float32_test - artifact_paths: "gpu_plane_analytic_schar_mountain_float32_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: "GPU: Prognostic EDMFX aquaplanet" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/gpu_prognostic_edmfx_aquaplanet.yml + # --job_id gpu_prognostic_edmfx_aquaplanet + # artifact_paths: "gpu_prognostic_edmfx_aquaplanet/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 20G - - label: "GPU: Analytic Tiny Schar Mountain Test (2D, Float32)" + - label: "GPU: Analytic No Topography Test (2D, Long Duration)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_tiny_schar_mountain_float32_test.yml - --job_id gpu_plane_analytic_tiny_schar_mountain_float32_test - artifact_paths: "gpu_plane_analytic_tiny_schar_mountain_float32_test/output_active/*" + --config_file $CONFIG_PATH/plane_analytic_no_topography_long_test.yml + --job_id gpu_plane_analytic_no_topography_long_test + artifact_paths: "gpu_plane_analytic_no_topography_long_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, No Hyperdiff)" + - label: "GPU: Analytic No Topography Test (2D, Long Duration, Float32)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_no_hyperdiff_test.yml - --job_id gpu_plane_analytic_schar_mountain_no_hyperdiff_test - artifact_paths: "gpu_plane_analytic_schar_mountain_no_hyperdiff_test/output_active/*" + --config_file $CONFIG_PATH/plane_analytic_no_topography_long_float32_test.yml + --job_id gpu_plane_analytic_no_topography_long_float32_test + artifact_paths: "gpu_plane_analytic_no_topography_long_float32_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, No Hyperdiff, Float32)" + - label: "GPU: Analytic No Topography Test (2D, Long Duration, No Hyperdiff)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_no_hyperdiff_float32_test.yml - --job_id gpu_plane_analytic_schar_mountain_no_hyperdiff_float32_test - artifact_paths: "gpu_plane_analytic_schar_mountain_no_hyperdiff_float32_test/output_active/*" + --config_file $CONFIG_PATH/plane_analytic_no_topography_no_hyperdiff_long_test.yml + --job_id gpu_plane_analytic_no_topography_no_hyperdiff_long_test + artifact_paths: "gpu_plane_analytic_no_topography_no_hyperdiff_long_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, Stretched Grid)" + - label: "GPU: Analytic No Topography Test (2D, Long Duration, No Hyperdiff, Float32)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_stretched_grid_test.yml - --job_id gpu_plane_analytic_schar_mountain_stretched_grid_test - artifact_paths: "gpu_plane_analytic_schar_mountain_stretched_grid_test/output_active/*" + --config_file $CONFIG_PATH/plane_analytic_no_topography_no_hyperdiff_long_float32_test.yml + --job_id gpu_plane_analytic_no_topography_no_hyperdiff_long_float32_test + artifact_paths: "gpu_plane_analytic_no_topography_no_hyperdiff_long_float32_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, Higher Horizontal Resolution)" + - label: "GPU: Analytic No Topography Test (2D, Long Duration, Discrete Balance)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_horz_res_test.yml - --job_id gpu_plane_analytic_schar_mountain_high_horz_res_test - artifact_paths: "gpu_plane_analytic_schar_mountain_high_horz_res_test/output_active/*" + --config_file $CONFIG_PATH/plane_analytic_no_topography_discrete_balance_long_test.yml + --job_id gpu_plane_analytic_no_topography_discrete_balance_long_test + artifact_paths: "gpu_plane_analytic_no_topography_discrete_balance_long_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, Higher Vertical Resolution)" + - label: "GPU: Analytic No Topography Test (2D, Long Duration, Discrete Balance, Float32)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_vert_res_test.yml - --job_id gpu_plane_analytic_schar_mountain_high_vert_res_test - artifact_paths: "gpu_plane_analytic_schar_mountain_high_vert_res_test/output_active/*" + --config_file $CONFIG_PATH/plane_analytic_no_topography_discrete_balance_long_float32_test.yml + --job_id gpu_plane_analytic_no_topography_discrete_balance_long_float32_test + artifact_paths: "gpu_plane_analytic_no_topography_discrete_balance_long_float32_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, Higher Domain Top)" + - label: "GPU: Analytic No Topography Test (Extruded 2D)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_top_test.yml - --job_id gpu_plane_analytic_schar_mountain_high_top_test - artifact_paths: "gpu_plane_analytic_schar_mountain_high_top_test/output_active/*" + --config_file $CONFIG_PATH/extruded_plane_analytic_no_topography_test.yml + --job_id gpu_extruded_plane_analytic_no_topography_test + artifact_paths: "gpu_extruded_plane_analytic_no_topography_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, Higher Domain Top, Higher Sponge)" + - label: "GPU: Analytic No Topography Test (3D)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_top_high_sponge_test.yml - --job_id gpu_plane_analytic_schar_mountain_high_top_high_sponge_test - artifact_paths: "gpu_plane_analytic_schar_mountain_high_top_high_sponge_test/output_active/*" + --config_file $CONFIG_PATH/box_analytic_no_topography_test.yml + --job_id gpu_box_analytic_no_topography_test + artifact_paths: "gpu_box_analytic_no_topography_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "GPU: Analytic Schar Mountain Test (2D, Higher Domain Top, Weaker and Higher Sponge)" + - label: "GPU: Analytic No Topography Test (3D, Float32)" command: > julia --color=yes --project=examples examples/hybrid/driver.jl - --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_top_weak_high_sponge_test.yml - --job_id gpu_plane_analytic_schar_mountain_high_top_weak_high_sponge_test - artifact_paths: "gpu_plane_analytic_schar_mountain_high_top_weak_high_sponge_test/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Benchmarks" - steps: - - - label: ":computer: Benchmark: GPU implicit baro wave wrt h_elem" - key: "gpu_implicit_barowave_wrt_h_elem" - command: > - - julia --color=yes --project=examples perf/benchmark_dump.jl - --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_wrt_h_elem.yml - --job_id gpu_implicit_barowave_wrt_h_elem - artifact_paths: "gpu_implicit_barowave_wrt_h_elem/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Benchmark: GPU implicit baro wave" - command: > - - julia --color=yes --project=examples perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave.yml - --job_id gpu_implicit_barowave - artifact_paths: "gpu_implicit_barowave/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Benchmark: GPU implicit baro wave moist" - command: > - - julia --color=yes --project=examples perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_moist.yml - --job_id gpu_implicit_barowave_moist - artifact_paths: "gpu_implicit_barowave_moist/output_active/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 16G - slurm_gpus: 1 - - - label: ":computer: Benchmark: CPU implicit baro wave" - command: > - julia --color=yes --project=examples perf/benchmark_step.jl - --config_file $PERF_CONFIG_PATH/cpu_implicit_barowave.yml - --job_id cpu_implicit_barowave - artifact_paths: "cpu_implicit_barowave/output_active/*" - - # Benchmarks - - label: ":computer: Benchmark: CPU perf target (default)" - command: > - julia --color=yes --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_perf_target.yml - --job_id bm_perf_target - agents: - slurm_mem: 24GB - - - label: ":computer: Benchmark: GPU perf target" - command: > - julia --color=yes --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_perf_target.yml - --job_id bm_perf_target_gpu - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 24GB - slurm_gpus: 1 - - - label: ":computer: Benchmark: CPU perf target (Threaded)" - command: > - julia --color=yes --threads 8 --project=perf perf/benchmark.jl - --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml - --job_id bm_perf_target_threaded - agents: - slurm_mem: 24GB - slurm_cpus_per_task: 8 - - - label: ":computer: Benchmark: GPU diag edmf" - command: > - julia --color=yes --project=perf perf/benchmark.jl - --config_file ${CONFIG_PATH}/aquaplanet_progedmf.yml - --job_id bm_diag_edmf_gpu - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 24GB - slurm_gpus: 1 - - - label: ":computer: Benchmark: GPU prog edmf" - command: > - julia --color=yes --project=perf perf/benchmark.jl - --config_file ${CONFIG_PATH}/aquaplanet_progedmf.yml - --job_id bm_prog_edmf_gpu - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_mem: 24GB - slurm_gpus: 1 - - - group: "Flame graphs" - steps: - - - label: ":fire: Flame graph: perf target (default)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target.yml - --job_id flame_perf_target - artifact_paths: "flame_perf_target/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: perf target (with tracers)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_tracers.yml - --job_id flame_perf_target_tracers - artifact_paths: "flame_perf_target_tracers/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: perf target (diagnostic edmfx)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_diagnostic_edmfx.yml - --job_id flame_perf_target_diagnostic_edmfx - artifact_paths: "flame_perf_target_diagnostic_edmfx/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: perf target (prognostic edmfx aquaplanet)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_prognostic_edmfx_aquaplanet.yml - --job_id flame_perf_target_prognostic_edmfx_aquaplanet - artifact_paths: "flame_perf_target_prognostic_edmfx_aquaplanet/*" - agents: - slurm_mem: 48GB - - - label: ":fire: Flame graph: perf target (frierson diffusion)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_frierson.yml - --job_id flame_perf_target_frierson - artifact_paths: "flame_perf_target_frierson/*" - agents: - slurm_mem: 48GB - - - label: ":fire: Flame graph: perf target (Threaded)" - command: > - julia --threads 8 --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml - --job_id flame_perf_target_threaded - artifact_paths: "flame_perf_target_threaded/*" - agents: - slurm_cpus_per_task: 8 - slurm_mem: 24GB - - - label: ":fire: Flame graph: perf target (Callbacks)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_target_callbacks.yml - --job_id flame_perf_target_callbacks - artifact_paths: "flame_perf_target_callbacks/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: gravity wave" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_gw.yml - --job_id flame_perf_gw - artifact_paths: "flame_perf_gw/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: perf target (diagnostics)" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_perf_diagnostics.yml - --job_id flame_perf_diagnostics - artifact_paths: "flame_perf_diagnostics/*" - agents: - slurm_mem: 24GB - - - label: ":fire: Flame graph: gpu job" - command: > - julia --color=yes --project=perf perf/flame.jl - --config_file $PERF_CONFIG_PATH/flame_gpu_implicit_barowave_moist.yml - --job_id flame_gpu_implicit_barowave_moist - artifact_paths: "flame_gpu_implicit_barowave_moist/*" + --config_file $CONFIG_PATH/box_analytic_no_topography_float32_test.yml + --job_id gpu_box_analytic_no_topography_float32_test + artifact_paths: "gpu_box_analytic_no_topography_float32_test/output_active/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: - slurm_mem: 48GB slurm_gpus: 1 - gres: "gpu:p100:1" - - - group: "Inference/Invalidations" - steps: - - # Inference - - label: ":rocket: JET n-failures (inference)" - command: > - julia --color=yes --project=perf perf/jet_test_nfailures.jl - --config_file $PERF_CONFIG_PATH/jet_n_failures.yml - --job_id jet_n_failures - agents: - slurm_mem: 24GB - - # Latency - - label: ":mag::rocket: Invalidations" - command: > - julia --color=yes --project=perf perf/invalidations.jl - artifact_paths: "invalidations/*" - agents: - slurm_mem: 24GB - - - wait: ~ - continue_on_failure: true - - - label: ":robot_face: Print new mse tables" - command: "julia --color=yes --project=examples regression_tests/print_new_mse.jl" - - - label: ":robot_face: Print new reference counter" - command: "julia --color=yes --project=examples regression_tests/print_new_ref_counter.jl" - - - label: ":bar_chart: Tabulate performance summary" - command: "julia --color=yes --project=perf perf/tabulate_perf_summary.jl" - - - label: ":chart_with_downwards_trend: build history" - command: - - build_history main - artifact_paths: - - "build_history.html" - - - wait - - label: ":robot_face: Move main results" - command: "julia --color=yes --project=examples regression_tests/move_output.jl" + # - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_long_test.yml + # --job_id gpu_plane_analytic_cosine_hills_long_test + # artifact_paths: "gpu_plane_analytic_cosine_hills_long_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Float32)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_long_float32_test.yml + # --job_id gpu_plane_analytic_cosine_hills_long_float32_test + # artifact_paths: "gpu_plane_analytic_cosine_hills_long_float32_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Stronger Sponge)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_strong_sponge_long_test.yml + # --job_id gpu_plane_analytic_cosine_hills_strong_sponge_long_test + # artifact_paths: "gpu_plane_analytic_cosine_hills_strong_sponge_long_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Weaker Sponge)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_weak_sponge_long_test.yml + # --job_id gpu_plane_analytic_cosine_hills_weak_sponge_long_test + # artifact_paths: "gpu_plane_analytic_cosine_hills_weak_sponge_long_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Higher Domain Top)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_high_top_long_test.yml + # --job_id gpu_plane_analytic_cosine_hills_high_top_long_test + # artifact_paths: "gpu_plane_analytic_cosine_hills_high_top_long_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (2D, Long Duration, Higher Domain Top, Higher Sponge)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_cosine_hills_high_top_high_sponge_long_test.yml + # --job_id gpu_plane_analytic_cosine_hills_high_top_high_sponge_long_test + # artifact_paths: "gpu_plane_analytic_cosine_hills_high_top_high_sponge_long_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (Extruded 2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/extruded_plane_analytic_cosine_hills_test.yml + # --job_id gpu_extruded_plane_analytic_cosine_hills_test + # artifact_paths: "gpu_extruded_plane_analytic_cosine_hills_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (3D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/box_analytic_cosine_hills_test.yml + # --job_id gpu_box_analytic_cosine_hills_test + # artifact_paths: "gpu_box_analytic_cosine_hills_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Cosine Hills Test (3D, Float32)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/box_analytic_cosine_hills_float32_test.yml + # --job_id gpu_box_analytic_cosine_hills_float32_test + # artifact_paths: "gpu_box_analytic_cosine_hills_float32_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Agnesi Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_agnesi_mountain_test.yml + # --job_id gpu_plane_analytic_agnesi_mountain_test + # artifact_paths: "gpu_plane_analytic_agnesi_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Huge Schar Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_huge_schar_mountain_test.yml + # --job_id gpu_plane_analytic_huge_schar_mountain_test + # artifact_paths: "gpu_plane_analytic_huge_schar_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Big Schar Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_big_schar_mountain_test.yml + # --job_id gpu_plane_analytic_big_schar_mountain_test + # artifact_paths: "gpu_plane_analytic_big_schar_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_test.yml + # --job_id gpu_plane_analytic_schar_mountain_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Small Schar Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_small_schar_mountain_test.yml + # --job_id gpu_plane_analytic_small_schar_mountain_test + # artifact_paths: "gpu_plane_analytic_small_schar_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Tiny Schar Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_tiny_schar_mountain_test.yml + # --job_id gpu_plane_analytic_tiny_schar_mountain_test + # artifact_paths: "gpu_plane_analytic_tiny_schar_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Teeny Tiny Schar Mountain Test (2D)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_teeny_tiny_schar_mountain_test.yml + # --job_id gpu_plane_analytic_teeny_tiny_schar_mountain_test + # artifact_paths: "gpu_plane_analytic_teeny_tiny_schar_mountain_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Float32)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_float32_test.yml + # --job_id gpu_plane_analytic_schar_mountain_float32_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_float32_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Tiny Schar Mountain Test (2D, Float32)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_tiny_schar_mountain_float32_test.yml + # --job_id gpu_plane_analytic_tiny_schar_mountain_float32_test + # artifact_paths: "gpu_plane_analytic_tiny_schar_mountain_float32_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, No Hyperdiff)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_no_hyperdiff_test.yml + # --job_id gpu_plane_analytic_schar_mountain_no_hyperdiff_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_no_hyperdiff_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, No Hyperdiff, Float32)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_no_hyperdiff_float32_test.yml + # --job_id gpu_plane_analytic_schar_mountain_no_hyperdiff_float32_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_no_hyperdiff_float32_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Stretched Grid)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_stretched_grid_test.yml + # --job_id gpu_plane_analytic_schar_mountain_stretched_grid_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_stretched_grid_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Higher Horizontal Resolution)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_horz_res_test.yml + # --job_id gpu_plane_analytic_schar_mountain_high_horz_res_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_high_horz_res_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Higher Vertical Resolution)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_vert_res_test.yml + # --job_id gpu_plane_analytic_schar_mountain_high_vert_res_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_high_vert_res_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Higher Domain Top)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_top_test.yml + # --job_id gpu_plane_analytic_schar_mountain_high_top_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_high_top_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Higher Domain Top, Higher Sponge)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_top_high_sponge_test.yml + # --job_id gpu_plane_analytic_schar_mountain_high_top_high_sponge_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_high_top_high_sponge_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "GPU: Analytic Schar Mountain Test (2D, Higher Domain Top, Weaker and Higher Sponge)" + # command: > + # julia --color=yes --project=examples examples/hybrid/driver.jl + # --config_file $CONFIG_PATH/plane_analytic_schar_mountain_high_top_weak_high_sponge_test.yml + # --job_id gpu_plane_analytic_schar_mountain_high_top_weak_high_sponge_test + # artifact_paths: "gpu_plane_analytic_schar_mountain_high_top_weak_high_sponge_test/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Benchmarks" + # steps: + + # - label: ":computer: Benchmark: GPU implicit baro wave wrt h_elem" + # key: "gpu_implicit_barowave_wrt_h_elem" + # command: > + + # julia --color=yes --project=examples perf/benchmark_dump.jl + # --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_wrt_h_elem.yml + # --job_id gpu_implicit_barowave_wrt_h_elem + # artifact_paths: "gpu_implicit_barowave_wrt_h_elem/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: GPU implicit baro wave" + # command: > + + # julia --color=yes --project=examples perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave.yml + # --job_id gpu_implicit_barowave + # artifact_paths: "gpu_implicit_barowave/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: GPU implicit baro wave moist" + # command: > + + # julia --color=yes --project=examples perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/gpu_implicit_barowave_moist.yml + # --job_id gpu_implicit_barowave_moist + # artifact_paths: "gpu_implicit_barowave_moist/output_active/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 16G + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: CPU implicit baro wave" + # command: > + # julia --color=yes --project=examples perf/benchmark_step.jl + # --config_file $PERF_CONFIG_PATH/cpu_implicit_barowave.yml + # --job_id cpu_implicit_barowave + # artifact_paths: "cpu_implicit_barowave/output_active/*" + + # # Benchmarks + # - label: ":computer: Benchmark: CPU perf target (default)" + # command: > + # julia --color=yes --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_perf_target.yml + # --job_id bm_perf_target + # agents: + # slurm_mem: 24GB + + # - label: ":computer: Benchmark: GPU perf target" + # command: > + # julia --color=yes --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_perf_target.yml + # --job_id bm_perf_target_gpu + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: CPU perf target (Threaded)" + # command: > + # julia --color=yes --threads 8 --project=perf perf/benchmark.jl + # --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml + # --job_id bm_perf_target_threaded + # agents: + # slurm_mem: 24GB + # slurm_cpus_per_task: 8 + + # - label: ":computer: Benchmark: GPU diag edmf" + # command: > + # julia --color=yes --project=perf perf/benchmark.jl + # --config_file ${CONFIG_PATH}/aquaplanet_progedmf.yml + # --job_id bm_diag_edmf_gpu + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 + + # - label: ":computer: Benchmark: GPU prog edmf" + # command: > + # julia --color=yes --project=perf perf/benchmark.jl + # --config_file ${CONFIG_PATH}/aquaplanet_progedmf.yml + # --job_id bm_prog_edmf_gpu + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 24GB + # slurm_gpus: 1 + + # - group: "Flame graphs" + # steps: + + # - label: ":fire: Flame graph: perf target (default)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target.yml + # --job_id flame_perf_target + # artifact_paths: "flame_perf_target/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: perf target (with tracers)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_tracers.yml + # --job_id flame_perf_target_tracers + # artifact_paths: "flame_perf_target_tracers/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: perf target (diagnostic edmfx)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_diagnostic_edmfx.yml + # --job_id flame_perf_target_diagnostic_edmfx + # artifact_paths: "flame_perf_target_diagnostic_edmfx/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: perf target (prognostic edmfx aquaplanet)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_prognostic_edmfx_aquaplanet.yml + # --job_id flame_perf_target_prognostic_edmfx_aquaplanet + # artifact_paths: "flame_perf_target_prognostic_edmfx_aquaplanet/*" + # agents: + # slurm_mem: 48GB + + # - label: ":fire: Flame graph: perf target (frierson diffusion)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_frierson.yml + # --job_id flame_perf_target_frierson + # artifact_paths: "flame_perf_target_frierson/*" + # agents: + # slurm_mem: 48GB + + # - label: ":fire: Flame graph: perf target (Threaded)" + # command: > + # julia --threads 8 --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml + # --job_id flame_perf_target_threaded + # artifact_paths: "flame_perf_target_threaded/*" + # agents: + # slurm_cpus_per_task: 8 + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: perf target (Callbacks)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_target_callbacks.yml + # --job_id flame_perf_target_callbacks + # artifact_paths: "flame_perf_target_callbacks/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: gravity wave" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_gw.yml + # --job_id flame_perf_gw + # artifact_paths: "flame_perf_gw/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: perf target (diagnostics)" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_perf_diagnostics.yml + # --job_id flame_perf_diagnostics + # artifact_paths: "flame_perf_diagnostics/*" + # agents: + # slurm_mem: 24GB + + # - label: ":fire: Flame graph: gpu job" + # command: > + # julia --color=yes --project=perf perf/flame.jl + # --config_file $PERF_CONFIG_PATH/flame_gpu_implicit_barowave_moist.yml + # --job_id flame_gpu_implicit_barowave_moist + # artifact_paths: "flame_gpu_implicit_barowave_moist/*" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_mem: 48GB + # slurm_gpus: 1 + # gres: "gpu:p100:1" + + # - group: "Inference/Invalidations" + # steps: + + # # Inference + # - label: ":rocket: JET n-failures (inference)" + # command: > + # julia --color=yes --project=perf perf/jet_test_nfailures.jl + # --config_file $PERF_CONFIG_PATH/jet_n_failures.yml + # --job_id jet_n_failures + # agents: + # slurm_mem: 24GB + + # # Latency + # - label: ":mag::rocket: Invalidations" + # command: > + # julia --color=yes --project=perf perf/invalidations.jl + # artifact_paths: "invalidations/*" + # agents: + # slurm_mem: 24GB + + # - wait: ~ + # continue_on_failure: true + + # - label: ":robot_face: Print new mse tables" + # command: "julia --color=yes --project=examples regression_tests/print_new_mse.jl" + + # - label: ":robot_face: Print new reference counter" + # command: "julia --color=yes --project=examples regression_tests/print_new_ref_counter.jl" + + # - label: ":bar_chart: Tabulate performance summary" + # command: "julia --color=yes --project=perf perf/tabulate_perf_summary.jl" + + # - label: ":chart_with_downwards_trend: build history" + # command: + # - build_history main + # artifact_paths: + # - "build_history.html" + + # - wait + + # - label: ":robot_face: Move main results" + # command: "julia --color=yes --project=examples regression_tests/move_output.jl" diff --git a/src/cache/precomputed_quantities.jl b/src/cache/precomputed_quantities.jl index 5aac9935072..83c530d8a4d 100644 --- a/src/cache/precomputed_quantities.jl +++ b/src/cache/precomputed_quantities.jl @@ -46,8 +46,13 @@ function precomputed_quantities(Y, atmos) n = n_mass_flux_subdomains(atmos.turbconv_model) gs_quantities = (; ᶜspecific = specific_gs.(Y.c), - ᶜu = similar(Y.c, C123{FT}), + ᶜuʰ = similar(Y.c, CT12{FT}), + ᶜu³ᴸ = similar(Y.c, CT3{FT}), + ᶜu³ᴿ = similar(Y.c, CT3{FT}), + ᶠuʰ = similar(Y.f, CT12{FT}), ᶠu³ = similar(Y.f, CT3{FT}), + ᶠu = similar(Y.f, CT123{FT}), + ᶜu_fake = similar(Y.c, C123{FT}), # not part of reconstruction ᶜK = similar(Y.c, FT), ᶜts = similar(Y.c, TST), ᶜp = similar(Y.c, FT), @@ -165,13 +170,6 @@ function precomputed_quantities(Y, atmos) ) end -# Interpolates the third contravariant component of Y.c.uₕ to cell faces. -function set_ᶠuₕ³!(ᶠuₕ³, Y) - ᶜJ = Fields.local_geometry_field(Y.c).J - @. ᶠuₕ³ = ᶠwinterp(Y.c.ρ * ᶜJ, CT3(Y.c.uₕ)) - return nothing -end - """ set_velocity_at_surface!(Y, ᶠuₕ³, turbconv_model) @@ -180,14 +178,16 @@ Modifies `Y.f.u₃` so that `ᶠu³` is 0 at the surface. Specifically, since the `turbconv_model` is EDMFX, the `Y.f.sgsʲs` are also modified so that each `u₃ʲ` is equal to `u₃` at the surface. """ -function set_velocity_at_surface!(Y, ᶠuₕ³, turbconv_model) - sfc_u₃ = Fields.level(Y.f.u₃.components.data.:1, half) - sfc_uₕ³ = Fields.level(ᶠuₕ³.components.data.:1, half) - sfc_g³³ = g³³_field(sfc_u₃) - @. sfc_u₃ = -sfc_uₕ³ / sfc_g³³ # u³ = uₕ³ + w³ = uₕ³ + w₃ * g³³ +function set_velocity_at_surface!(Y, turbconv_model) + int_uₕ = Fields.level(Y.c.uₕ, 1) + int_g³³ = g³³_field(int_uₕ) + sfc_u₃ = Fields.level(Y.f.u₃, half) + sfc_u₃_shifted = Fields.Field(Fields.field_values(sfc_u₃), axes(int_uₕ)) + @. sfc_u₃_shifted.components.data.:1 = + -CT3(int_uₕ).components.data.:1 / int_g³³ if turbconv_model isa PrognosticEDMFX for j in 1:n_mass_flux_subdomains(turbconv_model) - sfc_u₃ʲ = Fields.level(Y.f.sgsʲs.:($j).u₃.components.data.:1, half) + sfc_u₃ʲ = Fields.level(Y.f.sgsʲs.:($j).u₃, half) @. sfc_u₃ʲ = sfc_u₃ end end @@ -219,13 +219,78 @@ end # This is used to set the grid-scale velocity quantities ᶜu, ᶠu³, ᶜK based on # ᶠu₃, and it is also used to set the SGS quantities based on ᶠu₃⁰ and ᶠu₃ʲ. -function set_velocity_quantities!(ᶜu, ᶠu³, ᶜK, ᶠu₃, ᶜuₕ, ᶠuₕ³) - @. ᶜu = C123(ᶜuₕ) + ᶜinterp(C123(ᶠu₃)) - @. ᶠu³ = ᶠuₕ³ + CT3(ᶠu₃) - compute_kinetic!(ᶜK, ᶜuₕ, ᶠu₃) +function set_velocity_quantities!(ᶜuʰ, ᶜu³ᴸ, ᶜu³ᴿ, ᶠuʰ, ᶠu³, ᶠu, ᶜu_fake, ᶜK, Y) + ᶠleft_bias0 = Operators.LeftBiasedC2F(; bottom = Operators.SetValue(CT3(0))) + ᶠright_bias0 = Operators.RightBiasedC2F(; top = Operators.SetValue(CT3(0))) + ᶜJ = Fields.local_geometry_field(Y.c).J + ᶜρ = Y.c.ρ + ᶜuₕ = Y.c.uₕ + ᶠu₃ = Y.f.u₃ + @. ᶜuʰ = CT12(ᶜuₕ) + (CT12(ᶜleft_bias(ᶠu₃)) + CT12(ᶜright_bias(ᶠu₃))) / 2 + @. ᶠuʰ = ᶠinterp(ᶜJ * ᶜρ * ᶜuʰ) / ᶠinterp(ᶜJ * ᶜρ) + @. ᶜu³ᴸ = CT3(ᶜuₕ) + CT3(ᶜleft_bias(ᶠu₃)) + @. ᶜu³ᴿ = CT3(ᶜuₕ) + CT3(ᶜright_bias(ᶠu₃)) + @. ᶠu³ = + (ᶠleft_bias0(ᶜJ * ᶜρ * ᶜu³ᴿ) + ᶠright_bias0(ᶜJ * ᶜρ * ᶜu³ᴸ)) / + (2 * ᶠinterp(ᶜJ * ᶜρ)) + @. ᶠu = CT123(ᶠuʰ) + CT123(ᶠu³) + @. ᶜu_fake = C123(ᶜuₕ) + C123(ᶜinterp(ᶠu₃)) + @. ᶜK = + ( + dot(ᶜuₕ, ᶜuʰ) + + (dot(ᶜleft_bias(ᶠu₃), ᶜu³ᴸ) + dot(ᶜright_bias(ᶠu₃), ᶜu³ᴿ)) / 2 + ) / 2 return nothing end +#= +ᶜuʰ = ᶜgʰʰ * ᶜuₕ + ᶜgʰ³ * (ᶜL(ᶠu₃) + ᶜR(ᶠu₃)) / 2 +ᶜu³ᴸ = ᶜg³ʰ * ᶜuₕ + ᶜg³³ * ᶜL(ᶠu₃) +ᶜu³ᴿ = ᶜg³ʰ * ᶜuₕ + ᶜg³³ * ᶜR(ᶠu₃) +ᶠu³ = (ᶠL(ᶜJ * ᶜρ * ᶜu³ᴿ) + ᶠR(ᶜJ * ᶜρ * ᶜu³ᴸ)) / (2 * ᶠinterp(ᶜJ * ᶜρ)) + = ( + ᶠL(ᶜJ * ᶜρ * (ᶜg³ʰ * ᶜuₕ + ᶜg³³ * ᶜR(ᶠu₃))) + + ᶠR(ᶜJ * ᶜρ * (ᶜg³ʰ * ᶜuₕ + ᶜg³³ * ᶜL(ᶠu₃))) + ) / (2 * ᶠinterp(ᶜJ * ᶜρ)) + = ( + ᶠinterp(ᶜJ * ᶜρ * ᶜg³ʰ * ᶜuₕ) + + (ᶠL(ᶜJ * ᶜρ * ᶜg³³ * ᶜR(ᶠu₃)) + ᶠR(ᶜJ * ᶜρ * ᶜg³³ * ᶜL(ᶠu₃))) / 2 + ) / ᶠinterp(ᶜJ * ᶜρ) +ᶜK = (dot(ᶜuₕ, ᶜuʰ) + (dot(ᶜL(ᶠu₃), ᶜu³ᴸ) + dot(ᶜR(ᶠu₃), ᶜu³ᴿ)) / 2) / 2 + = ( + ᶜuₕ' * ᶜgʰʰ * ᶜuₕ + + ᶜuₕ' * ᶜgʰ³ * (ᶜL(ᶠu₃) + ᶜR(ᶠu₃)) / 2 + + (ᶜL(ᶠu₃) + ᶜR(ᶠu₃))' * ᶜg³ʰ * ᶜuₕ / 2 + + (ᶜL(ᶠu₃)' * ᶜg³³ * ᶜL(ᶠu₃) + ᶜR(ᶠu₃)' * ᶜg³³ * ᶜR(ᶠu₃)) / 2 + ) / 2 + = ( + ᶜuₕ' * ᶜgʰʰ * ᶜuₕ + + ᶜuₕ' * ᶜgʰ³ * (ᶜL(ᶠu₃) + ᶜR(ᶠu₃)) + + (ᶜL(ᶠu₃)' * ᶜg³³ * ᶜL(ᶠu₃) + ᶜR(ᶠu₃)' * ᶜg³³ * ᶜR(ᶠu₃)) / 2 + ) / 2 + +∂ᶜK∂ᶜuₕ = (ᶜgʰʰ * ᶜuₕ + ᶜgʰ³ * (ᶜL(ᶠu₃) + ᶜR(ᶠu₃)) / 2)' = ᶜuʰ' +∂ᶜK∂ᶠu₃ = + = ( + ᶜuₕ' * ᶜgʰ³ * (ᶜL_matrix + ᶜR_matrix) + + ᶜL(ᶠu₃)' * ᶜg³³ * ᶜL_matrix + ᶜR(ᶠu₃)' * ᶜg³³ * ᶜR_matrix + ) / 2 + = ( + (ᶜg³ʰ * ᶜuₕ + ᶜg³³ * ᶜL(ᶠu₃))' * ᶜL_matrix + + (ᶜg³ʰ * ᶜuₕ + ᶜg³³ * ᶜR(ᶠu₃))' * ᶜR_matrix + ) / 2 + = (ᶜu³ᴸ' * ᶜL_matrix + ᶜu³ᴿ' * ᶜR_matrix) / 2 + +∂ᶠu³∂ᶜuₕ = + = ᶠinterp_matrix ⋅ Diag(ᶜJ * ᶜρ * ᶜg³ʰ) / ᶠinterp(ᶜJ * ᶜρ) + = ᶠwinterp_matrix(ᶜJ * ᶜρ) ⋅ Diag(ᶜg³ʰ) +∂ᶠu³∂ᶠu₃ = + = ( + ᶠL_matrix ⋅ Diag(ᶜJ * ᶜρ * ᶜg³³) ⋅ ᶜR_matrix + + ᶠR_matrix ⋅ Diag(ᶜJ * ᶜρ * ᶜg³³) ⋅ ᶜL_matrix + ) / (2 * ᶠinterp(ᶜJ * ᶜρ)) +=# + function set_sgs_ᶠu₃!(w_function, ᶠu₃, Y, turbconv_model) ρaʲs(sgsʲs) = map(sgsʲ -> sgsʲ.ρa, sgsʲs) u₃ʲs(sgsʲs) = map(sgsʲ -> sgsʲ.u₃, sgsʲs) @@ -451,18 +516,17 @@ NVTX.@annotate function set_precomputed_quantities!(Y, p, t) n = n_mass_flux_subdomains(turbconv_model) thermo_args = (thermo_params, moisture_model) (; ᶜΦ) = p.core - (; ᶜspecific, ᶜu, ᶠu³, ᶜK, ᶜts, ᶜp) = p.precomputed - ᶠuₕ³ = p.scratch.ᶠtemp_CT3 + (; ᶜspecific, ᶜuʰ, ᶜu³ᴸ, ᶜu³ᴿ, ᶠuʰ, ᶠu³, ᶠu, ᶜu_fake, ᶜK, ᶜts, ᶜp) = + p.precomputed @. ᶜspecific = specific_gs(Y.c) - set_ᶠuₕ³!(ᶠuₕ³, Y) # TODO: We might want to move this to dss! (and rename dss! to something # like enforce_constraints!). - set_velocity_at_surface!(Y, ᶠuₕ³, turbconv_model) + set_velocity_at_surface!(Y, turbconv_model) set_velocity_at_top!(Y, turbconv_model) - set_velocity_quantities!(ᶜu, ᶠu³, ᶜK, Y.f.u₃, Y.c.uₕ, ᶠuₕ³) + set_velocity_quantities!(ᶜuʰ, ᶜu³ᴸ, ᶜu³ᴿ, ᶠuʰ, ᶠu³, ᶠu, ᶜu_fake, ᶜK, Y) if n > 0 # TODO: In the following increments to ᶜK, we actually need to add # quantities of the form ᶜρaχ⁰ / ᶜρ⁰ and ᶜρaχʲ / ᶜρʲ to ᶜK, rather than diff --git a/src/diagnostics/core_diagnostics.jl b/src/diagnostics/core_diagnostics.jl index e0c560c0293..7b8d2f231cc 100644 --- a/src/diagnostics/core_diagnostics.jl +++ b/src/diagnostics/core_diagnostics.jl @@ -74,9 +74,9 @@ add_diagnostic_variable!( comments = "Eastward (zonal) wind component", compute! = (out, state, cache, time) -> begin if isnothing(out) - return copy(u_component.(Geometry.UVector.(cache.precomputed.ᶜu))) + return copy(u_component.(Geometry.UVector.(cache.precomputed.ᶠu))) else - out .= u_component.(Geometry.UVector.(cache.precomputed.ᶜu)) + out .= u_component.(Geometry.UVector.(cache.precomputed.ᶠu)) end end, ) @@ -92,9 +92,9 @@ add_diagnostic_variable!( comments = "Northward (meridional) wind component", compute! = (out, state, cache, time) -> begin if isnothing(out) - return copy(v_component.(Geometry.VVector.(cache.precomputed.ᶜu))) + return copy(v_component.(Geometry.VVector.(cache.precomputed.ᶠu))) else - out .= v_component.(Geometry.VVector.(cache.precomputed.ᶜu)) + out .= v_component.(Geometry.VVector.(cache.precomputed.ᶠu)) end end, ) @@ -113,9 +113,9 @@ add_diagnostic_variable!( comments = "Vertical wind component", compute! = (out, state, cache, time) -> begin if isnothing(out) - return copy(w_component.(Geometry.WVector.(cache.precomputed.ᶜu))) + return copy(w_component.(Geometry.WVector.(cache.precomputed.ᶠu))) else - out .= w_component.(Geometry.WVector.(cache.precomputed.ᶜu)) + out .= w_component.(Geometry.WVector.(cache.precomputed.ᶠu)) end end, ) @@ -746,9 +746,9 @@ add_diagnostic_variable!( comments = "Predicted value of eastward (zonal) wind component", compute! = (out, state, cache, time) -> begin if isnothing(out) - u_component.(cache.predicted_steady_state.ᶜu) + u_component.(cache.predicted_steady_state.ᶠu) else - out .= u_component.(cache.predicted_steady_state.ᶜu) + out .= u_component.(cache.predicted_steady_state.ᶠu) end end, ) @@ -761,9 +761,9 @@ add_diagnostic_variable!( comments = "Predicted value of northward (meridional) wind component", compute! = (out, state, cache, time) -> begin if isnothing(out) - v_component.(cache.predicted_steady_state.ᶜu) + v_component.(cache.predicted_steady_state.ᶠu) else - out .= v_component.(cache.predicted_steady_state.ᶜu) + out .= v_component.(cache.predicted_steady_state.ᶠu) end end, ) @@ -776,9 +776,9 @@ add_diagnostic_variable!( comments = "Predicted value of vertical wind component", compute! = (out, state, cache, time) -> begin if isnothing(out) - w_component.(cache.predicted_steady_state.ᶜu) + w_component.(cache.predicted_steady_state.ᶠu) else - out .= w_component.(cache.predicted_steady_state.ᶜu) + out .= w_component.(cache.predicted_steady_state.ᶠu) end end, ) @@ -791,12 +791,12 @@ add_diagnostic_variable!( comments = "Error of eastward (zonal) wind component against predicted value", compute! = (out, state, cache, time) -> begin if isnothing(out) - u_component.(Geometry.UVWVector.(cache.precomputed.ᶜu)) .- - u_component.(cache.predicted_steady_state.ᶜu) + u_component.(Geometry.UVWVector.(cache.precomputed.ᶠu)) .- + u_component.(cache.predicted_steady_state.ᶠu) else out .= - u_component.(Geometry.UVWVector.(cache.precomputed.ᶜu)) .- - u_component.(cache.predicted_steady_state.ᶜu) + u_component.(Geometry.UVWVector.(cache.precomputed.ᶠu)) .- + u_component.(cache.predicted_steady_state.ᶠu) end end, ) @@ -809,12 +809,12 @@ add_diagnostic_variable!( comments = "Error of northward (meridional) wind component against predicted value", compute! = (out, state, cache, time) -> begin if isnothing(out) - v_component.(Geometry.UVWVector.(cache.precomputed.ᶜu)) .- - v_component.(cache.predicted_steady_state.ᶜu) + v_component.(Geometry.UVWVector.(cache.precomputed.ᶠu)) .- + v_component.(cache.predicted_steady_state.ᶠu) else out .= - v_component.(Geometry.UVWVector.(cache.precomputed.ᶜu)) .- - v_component.(cache.predicted_steady_state.ᶜu) + v_component.(Geometry.UVWVector.(cache.precomputed.ᶠu)) .- + v_component.(cache.predicted_steady_state.ᶠu) end end, ) @@ -827,12 +827,12 @@ add_diagnostic_variable!( comments = "Error of vertical wind component against predicted value", compute! = (out, state, cache, time) -> begin if isnothing(out) - w_component.(Geometry.UVWVector.(cache.precomputed.ᶜu)) .- - w_component.(cache.predicted_steady_state.ᶜu) + w_component.(Geometry.UVWVector.(cache.precomputed.ᶠu)) .- + w_component.(cache.predicted_steady_state.ᶠu) else out .= - w_component.(Geometry.UVWVector.(cache.precomputed.ᶜu)) .- - w_component.(cache.predicted_steady_state.ᶜu) + w_component.(Geometry.UVWVector.(cache.precomputed.ᶠu)) .- + w_component.(cache.predicted_steady_state.ᶠu) end end, ) diff --git a/src/prognostic_equations/advection.jl b/src/prognostic_equations/advection.jl index d76d4360579..04957bbc281 100644 --- a/src/prognostic_equations/advection.jl +++ b/src/prognostic_equations/advection.jl @@ -9,7 +9,7 @@ import ClimaCore.Geometry as Geometry NVTX.@annotate function horizontal_advection_tendency!(Yₜ, Y, p, t) n = n_mass_flux_subdomains(p.atmos.turbconv_model) (; ᶜΦ) = p.core - (; ᶜu, ᶜK, ᶜp) = p.precomputed + (; ᶜuʰ, ᶜK, ᶜp) = p.precomputed if p.atmos.turbconv_model isa AbstractEDMF (; ᶜu⁰) = p.precomputed end @@ -17,7 +17,7 @@ NVTX.@annotate function horizontal_advection_tendency!(Yₜ, Y, p, t) (; ᶜuʲs) = p.precomputed end - @. Yₜ.c.ρ -= wdivₕ(Y.c.ρ * ᶜu) + @. Yₜ.c.ρ -= wdivₕ(Y.c.ρ * ᶜuʰ) if p.atmos.turbconv_model isa PrognosticEDMFX for j in 1:n @. Yₜ.c.sgsʲs.:($$j).ρa -= wdivₕ(Y.c.sgsʲs.:($$j).ρa * ᶜuʲs.:($$j)) @@ -25,7 +25,7 @@ NVTX.@annotate function horizontal_advection_tendency!(Yₜ, Y, p, t) end (; ᶜh_tot) = p.precomputed - @. Yₜ.c.ρe_tot -= wdivₕ(Y.c.ρ * ᶜh_tot * ᶜu) + @. Yₜ.c.ρe_tot -= wdivₕ(Y.c.ρ * ᶜh_tot * ᶜuʰ) if p.atmos.turbconv_model isa PrognosticEDMFX for j in 1:n @@ -47,14 +47,14 @@ end NVTX.@annotate function horizontal_tracer_advection_tendency!(Yₜ, Y, p, t) n = n_mass_flux_subdomains(p.atmos.turbconv_model) - (; ᶜu) = p.precomputed + (; ᶜuʰ) = p.precomputed if p.atmos.turbconv_model isa PrognosticEDMFX (; ᶜuʲs) = p.precomputed end for ρχ_name in filter(is_tracer_var, propertynames(Y.c)) - @. Yₜ.c.:($$ρχ_name) -= wdivₕ(Y.c.:($$ρχ_name) * ᶜu) + @. Yₜ.c.:($$ρχ_name) -= wdivₕ(Y.c.:($$ρχ_name) * ᶜuʰ) end if p.atmos.turbconv_model isa PrognosticEDMFX @@ -74,8 +74,9 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t) point_type = eltype(Fields.coordinate_field(Y.c)) (; dt) = p ᶜJ = Fields.local_geometry_field(Y.c).J + ᶠJ = Fields.local_geometry_field(Y.f).J (; ᶜf³, ᶠf¹², ᶜΦ) = p.core - (; ᶜu, ᶠu³, ᶜK) = p.precomputed + (; ᶜuʰ, ᶠuʰ, ᶠu³, ᶜK) = p.precomputed (; edmfx_upwinding) = n > 0 || advect_tke ? p.atmos.numerics : all_nothing (; ᶜuʲs, ᶜKʲs, ᶠKᵥʲs) = n > 0 ? p.precomputed : all_nothing (; ᶠu³⁰) = advect_tke ? p.precomputed : all_nothing @@ -87,8 +88,8 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t) ᶜtke⁰ = advect_tke ? p.precomputed.ᶜtke⁰ : nothing ᶜa_scalar = p.scratch.ᶜtemp_scalar ᶜω³ = p.scratch.ᶜtemp_CT3 - ᶠω¹² = p.scratch.ᶠtemp_CT12 - ᶠω¹²ʲs = p.scratch.ᶠtemp_CT12ʲs + ᶠωʰ = p.scratch.ᶠtemp_CT12 + ᶠωʰʲs = p.scratch.ᶠtemp_CT12ʲs if point_type <: Geometry.Abstract3DPoint @. ᶜω³ = curlₕ(Y.c.uₕ) @@ -96,13 +97,13 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t) @. ᶜω³ = zero(ᶜω³) end - @. ᶠω¹² = ᶠcurlᵥ(Y.c.uₕ) + @. ᶠωʰ = ᶠcurlᵥ(Y.c.uₕ) for j in 1:n - @. ᶠω¹²ʲs.:($$j) = ᶠω¹² + @. ᶠωʰʲs.:($$j) = ᶠωʰ end - @. ᶠω¹² += CT12(curlₕ(Y.f.u₃)) + @. ᶠωʰ += CT12(curlₕ(Y.f.u₃)) for j in 1:n - @. ᶠω¹²ʲs.:($$j) += CT12(curlₕ(Y.f.sgsʲs.:($$j).u₃)) + @. ᶠωʰʲs.:($$j) += CT12(curlₕ(Y.f.sgsʲs.:($$j).u₃)) end # Without the CT12(), the right-hand side would be a CT1 or CT2 in 2D space. @@ -114,6 +115,7 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t) coeff, Yₜ.c.ρe_tot, ᶜJ, + ᶠJ, Y.c.ρ, ᶠu³, ᶜh_tot, @@ -126,30 +128,36 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t) χ_name == :e_tot && continue for (coeff, upwinding) in ((1, tracer_upwinding), (-1, Val(:none))) tracer_upwinding isa Val{:none} && continue - vertical_transport!(coeff, ᶜρχₜ, ᶜJ, Y.c.ρ, ᶠu³, ᶜχ, dt, upwinding) + vertical_transport!( + coeff, + ᶜρχₜ, + ᶜJ, + ᶠJ, + Y.c.ρ, + ᶠu³, + ᶜχ, + dt, + upwinding, + ) end end if isnothing(ᶠf¹²) # shallow atmosphere - @. Yₜ.c.uₕ -= - ᶜinterp(ᶠω¹² × (ᶠinterp(Y.c.ρ * ᶜJ) * ᶠu³)) / (Y.c.ρ * ᶜJ) + - (ᶜf³ + ᶜω³) × CT12(ᶜu) - @. Yₜ.f.u₃ -= ᶠω¹² × ᶠinterp(CT12(ᶜu)) + ᶠgradᵥ(ᶜK) + @. Yₜ.c.uₕ -= ᶜinterp(ᶠωʰ × ᶠu³) + (ᶜf³ + ᶜω³) × ᶜuʰ + @. Yₜ.f.u₃ -= ᶠωʰ × ᶠuʰ + ᶠgradᵥ(ᶜK) for j in 1:n @. Yₜ.f.sgsʲs.:($$j).u₃ -= - ᶠω¹²ʲs.:($$j) × ᶠinterp(CT12(ᶜuʲs.:($$j))) + + ᶠωʰʲs.:($$j) × ᶠinterp(CT12(ᶜuʲs.:($$j))) + ᶠgradᵥ(ᶜKʲs.:($$j) - ᶜinterp(ᶠKᵥʲs.:($$j))) end else # deep atmosphere - @. Yₜ.c.uₕ -= - ᶜinterp((ᶠf¹² + ᶠω¹²) × (ᶠinterp(Y.c.ρ * ᶜJ) * ᶠu³)) / - (Y.c.ρ * ᶜJ) + (ᶜf³ + ᶜω³) × CT12(ᶜu) - @. Yₜ.f.u₃ -= (ᶠf¹² + ᶠω¹²) × ᶠinterp(CT12(ᶜu)) + ᶠgradᵥ(ᶜK) + @. Yₜ.c.uₕ -= ᶜinterp((ᶠf¹² + ᶠωʰ) × ᶠu³) + (ᶜf³ + ᶜω³) × ᶜuʰ + @. Yₜ.f.u₃ -= (ᶠf¹² + ᶠωʰ) × ᶠuʰ + ᶠgradᵥ(ᶜK) for j in 1:n @. Yₜ.f.sgsʲs.:($$j).u₃ -= - (ᶠf¹² + ᶠω¹²ʲs.:($$j)) × ᶠinterp(CT12(ᶜuʲs.:($$j))) + + (ᶠf¹² + ᶠωʰʲs.:($$j)) × ᶠinterp(CT12(ᶜuʲs.:($$j))) + ᶠgradᵥ(ᶜKʲs.:($$j) - ᶜinterp(ᶠKᵥʲs.:($$j))) end end @@ -159,6 +167,7 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t) vertical_transport!( Yₜ.c.sgs⁰.ρatke, ᶜJ, + ᶠJ, ᶜρ⁰, ᶠu³⁰, ᶜa_scalar, diff --git a/src/prognostic_equations/hyperdiffusion.jl b/src/prognostic_equations/hyperdiffusion.jl index cf046c05d9c..e7b0de38486 100644 --- a/src/prognostic_equations/hyperdiffusion.jl +++ b/src/prognostic_equations/hyperdiffusion.jl @@ -76,8 +76,8 @@ NVTX.@annotate function prep_hyperdiffusion_tendency!(Yₜ, Y, p, t) # Grid scale hyperdiffusion @. ᶜ∇²u = - C123(wgradₕ(divₕ(p.precomputed.ᶜu))) - - C123(wcurlₕ(C123(curlₕ(p.precomputed.ᶜu)))) + C123(wgradₕ(divₕ(p.precomputed.ᶜuʰ))) - + C123(wcurlₕ(C123(curlₕ(p.precomputed.ᶜu_fake)))) # Do we need full 3D u? @. ᶜ∇²specific_energy = wdivₕ(gradₕ(ᶜspecific.e_tot + ᶜp / Y.c.ρ)) diff --git a/src/prognostic_equations/implicit/implicit_solver.jl b/src/prognostic_equations/implicit/implicit_solver.jl index d8f721e9679..5d76f1a542c 100644 --- a/src/prognostic_equations/implicit/implicit_solver.jl +++ b/src/prognostic_equations/implicit/implicit_solver.jl @@ -399,6 +399,9 @@ NVTX.@annotate function Wfact!(A, Y, p, dtγ, t) # Remove unnecessary values from p to avoid allocations in bycolumn. p′ = (; p.precomputed.ᶜspecific, + p.precomputed.ᶜuʰ, + p.precomputed.ᶜu³ᴸ, + p.precomputed.ᶜu³ᴿ, p.precomputed.ᶜK, p.precomputed.ᶜts, p.precomputed.ᶜp, @@ -467,7 +470,7 @@ end function update_implicit_equation_jacobian!(A, Y, p, dtγ) (; matrix, diffusion_flag, sgs_advection_flag, topography_flag) = A - (; ᶜspecific, ᶜK, ᶜts, ᶜp, ᶜΦ, ᶠgradᵥ_ᶜΦ) = p + (; ᶜspecific, ᶜuʰ, ᶜu³ᴸ, ᶜu³ᴿ, ᶜK, ᶜts, ᶜp, ᶜΦ, ᶠgradᵥ_ᶜΦ) = p (; ᶜtemp_C3, ∂ᶜK_∂ᶜuₕ, @@ -492,31 +495,40 @@ function update_implicit_equation_jacobian!(A, Y, p, dtγ) thermo_params = CAP.thermodynamics_params(params) ᶜρ = Y.c.ρ - ᶜuₕ = Y.c.uₕ - ᶠu₃ = Y.f.u₃ ᶜJ = Fields.local_geometry_field(Y.c).J + ᶠJ = Fields.local_geometry_field(Y.f).J ᶜgⁱʲ = Fields.local_geometry_field(Y.c).gⁱʲ - ᶠgⁱʲ = Fields.local_geometry_field(Y.f).gⁱʲ + + ᶠleft_bias0 = Operators.LeftBiasedC2F(; bottom = Operators.SetValue(CT3(0))) + ᶠright_bias0 = Operators.RightBiasedC2F(; top = Operators.SetValue(CT3(0))) + ᶠleft_bias0_matrix = MatrixFields.operator_matrix(ᶠleft_bias0) + ᶠright_bias0_matrix = MatrixFields.operator_matrix(ᶠright_bias0) ᶜkappa_m = p.ᶜtemp_scalar @. ᶜkappa_m = TD.gas_constant_air(thermo_params, ᶜts) / TD.cv_m(thermo_params, ᶜts) - if use_derivative(topography_flag) - @. ∂ᶜK_∂ᶜuₕ = DiagonalMatrixRow( - adjoint(CTh(ᶜuₕ)) + adjoint(ᶜinterp(ᶠu₃)) * g³ʰ(ᶜgⁱʲ), - ) - else - @. ∂ᶜK_∂ᶜuₕ = DiagonalMatrixRow(adjoint(CTh(ᶜuₕ))) - end + # if use_derivative(topography_flag) + # @. ∂ᶜK_∂ᶜuₕ = DiagonalMatrixRow( + # adjoint(CTh(ᶜuₕ)) + adjoint(ᶜinterp(ᶠu₃)) * g³ʰ(ᶜgⁱʲ), + # ) + # else + # @. ∂ᶜK_∂ᶜuₕ = DiagonalMatrixRow(adjoint(CTh(ᶜuₕ))) + # end + # @. ∂ᶜK_∂ᶠu₃ = + # ᶜinterp_matrix() ⋅ DiagonalMatrixRow(adjoint(CT3(ᶠu₃))) + + # DiagonalMatrixRow(adjoint(CT3(ᶜuₕ))) ⋅ ᶜinterp_matrix() + @. ∂ᶜK_∂ᶜuₕ = DiagonalMatrixRow(adjoint(CTh(ᶜuʰ))) @. ∂ᶜK_∂ᶠu₃ = - ᶜinterp_matrix() ⋅ DiagonalMatrixRow(adjoint(CT3(ᶠu₃))) + - DiagonalMatrixRow(adjoint(CT3(ᶜuₕ))) ⋅ ᶜinterp_matrix() + ( + DiagonalMatrixRow(adjoint(ᶜu³ᴸ)) ⋅ ᶜleft_bias_matrix() + + DiagonalMatrixRow(adjoint(ᶜu³ᴿ)) ⋅ ᶜright_bias_matrix() + ) / 2 @. ᶠp_grad_matrix = DiagonalMatrixRow(-1 / ᶠinterp(ᶜρ)) ⋅ ᶠgradᵥ_matrix() @. ᶜadvection_matrix = - -(ᶜadvdivᵥ_matrix()) ⋅ DiagonalMatrixRow(ᶠwinterp(ᶜJ, ᶜρ)) + -(ᶜadvdivᵥ_matrix()) ⋅ DiagonalMatrixRow(ᶠinterp(ᶜJ * ᶜρ) / ᶠJ) if use_derivative(topography_flag) ∂ᶜρ_err_∂ᶜuₕ = matrix[@name(c.ρ), @name(c.uₕ)] @@ -525,7 +537,13 @@ function update_implicit_equation_jacobian!(A, Y, p, dtγ) DiagonalMatrixRow(g³ʰ(ᶜgⁱʲ)) end ∂ᶜρ_err_∂ᶠu₃ = matrix[@name(c.ρ), @name(f.u₃)] - @. ∂ᶜρ_err_∂ᶠu₃ = dtγ * ᶜadvection_matrix ⋅ DiagonalMatrixRow(g³³(ᶠgⁱʲ)) + @. ∂ᶜρ_err_∂ᶠu₃ = + dtγ * ᶜadvection_matrix ⋅ DiagonalMatrixRow(1 / ᶠinterp(ᶜJ * ᶜρ)) ⋅ ( + ᶠleft_bias0_matrix() ⋅ DiagonalMatrixRow(ᶜJ * ᶜρ * g³³(ᶜgⁱʲ)) ⋅ + ᶜright_bias_matrix() + + ᶠright_bias0_matrix() ⋅ DiagonalMatrixRow(ᶜJ * ᶜρ * g³³(ᶜgⁱʲ)) ⋅ + ᶜleft_bias_matrix() + ) / 2 tracer_info = ( (@name(c.ρe_tot), @name(ᶜh_tot)), @@ -542,7 +560,13 @@ function update_implicit_equation_jacobian!(A, Y, p, dtγ) dtγ * ᶜadvection_matrix ⋅ DiagonalMatrixRow(ᶠinterp(ᶜχ)) ⋅ ᶠwinterp_matrix(ᶜJ * ᶜρ) ⋅ DiagonalMatrixRow(g³ʰ(ᶜgⁱʲ)) @. ∂ᶜρχ_err_∂ᶠu₃ = - dtγ * ᶜadvection_matrix ⋅ DiagonalMatrixRow(ᶠinterp(ᶜχ) * g³³(ᶠgⁱʲ)) + dtγ * ᶜadvection_matrix ⋅ + DiagonalMatrixRow(ᶠinterp(ᶜχ) / ᶠinterp(ᶜJ * ᶜρ)) ⋅ ( + ᶠleft_bias0_matrix() ⋅ DiagonalMatrixRow(ᶜJ * ᶜρ * g³³(ᶜgⁱʲ)) ⋅ + ᶜright_bias_matrix() + + ᶠright_bias0_matrix() ⋅ DiagonalMatrixRow(ᶜJ * ᶜρ * g³³(ᶜgⁱʲ)) ⋅ + ᶜleft_bias_matrix() + ) / 2 end ∂ᶠu₃_err_∂ᶜρ = matrix[@name(f.u₃), @name(c.ρ)] @@ -682,7 +706,8 @@ function update_implicit_equation_jacobian!(A, Y, p, dtγ) ∂ᶜρqₚ_err_∂ᶜρqₚ = matrix[ρqₚ_name, ρqₚ_name] ᶜwₚ = MatrixFields.get_field(p, wₚ_name) ᶠtmp = p.ᶠtemp_CT3 - @. ᶠtmp = CT3(unit_basis_vector_data(CT3, ᶠlg)) * ᶠwinterp(ᶜJ, ᶜρ) + @. ᶠtmp = + CT3(unit_basis_vector_data(CT3, ᶠlg)) * ᶠinterp(ᶜJ * ᶜρ) / ᶠJ @. ∂ᶜρqₚ_err_∂ᶜρqₚ += dtγ * -(ᶜprecipdivᵥ_matrix()) ⋅ DiagonalMatrixRow(ᶠtmp) ⋅ ᶠright_bias_matrix() ⋅ DiagonalMatrixRow(-(ᶜwₚ) / ᶜρ) diff --git a/src/prognostic_equations/implicit/implicit_tendency.jl b/src/prognostic_equations/implicit/implicit_tendency.jl index 6548508bcdb..8a80c0c1d3a 100644 --- a/src/prognostic_equations/implicit/implicit_tendency.jl +++ b/src/prognostic_equations/implicit/implicit_tendency.jl @@ -45,49 +45,64 @@ end # the implicit tendency function. Since dt >= dtγ, we can safely use dt for now. # TODO: Can we rewrite ᶠfct_boris_book and ᶠfct_zalesak so that their broadcast # expressions are less convoluted? -vertical_transport!(ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding::Val, ᶜdivᵥ) = - vertical_transport!(1, ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding, ᶜdivᵥ) -vertical_transport!(ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding::Val) = - vertical_transport!(1, ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding, ᶜadvdivᵥ) +vertical_transport!(ᶜρχₜ, ᶜJ, ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding::Val, ᶜdivᵥ) = + vertical_transport!(1, ᶜρχₜ, ᶜJ, ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding, ᶜdivᵥ) +vertical_transport!(ᶜρχₜ, ᶜJ, ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding::Val) = + vertical_transport!(1, ᶜρχₜ, ᶜJ, ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding, ᶜadvdivᵥ) vertical_transport!( coeff::Int, ᶜρχₜ, ᶜJ, + ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt::Real, upwinding::Val, -) = vertical_transport!(coeff, ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, upwinding, ᶜadvdivᵥ) +) = vertical_transport!( + coeff, + ᶜρχₜ, + ᶜJ, + ᶠJ, + ᶜρ, + ᶠu³, + ᶜχ, + dt, + upwinding, + ᶜadvdivᵥ, +) -vertical_transport!(coeff, ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, ::Val{:none}, ᶜdivᵥ) = - @. ᶜρχₜ += -coeff * (ᶜdivᵥ(ᶠwinterp(ᶜJ, ᶜρ) * ᶠu³ * ᶠinterp(ᶜχ))) +vertical_transport!(coeff, ᶜρχₜ, ᶜJ, ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, ::Val{:none}, ᶜdivᵥ) = + @. ᶜρχₜ += -coeff * (ᶜdivᵥ(ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ᶠu³ * ᶠinterp(ᶜχ))) vertical_transport!( coeff, ᶜρχₜ, ᶜJ, + ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, ::Val{:first_order}, ᶜdivᵥ, -) = @. ᶜρχₜ += -coeff * (ᶜdivᵥ(ᶠwinterp(ᶜJ, ᶜρ) * ᶠupwind1(ᶠu³, ᶜχ))) +) = @. ᶜρχₜ += -coeff * (ᶜdivᵥ(ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ᶠupwind1(ᶠu³, ᶜχ))) vertical_transport!( coeff, ᶜρχₜ, ᶜJ, + ᶠJ, ᶜρ, ᶠu³, ᶜχ, dt, ::Val{:third_order}, ᶜdivᵥ, -) = @. ᶜρχₜ += -coeff * (ᶜdivᵥ(ᶠwinterp(ᶜJ, ᶜρ) * ᶠupwind3(ᶠu³, ᶜχ))) +) = @. ᶜρχₜ += -coeff * (ᶜdivᵥ(ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ᶠupwind3(ᶠu³, ᶜχ))) vertical_transport!( coeff, ᶜρχₜ, ᶜJ, + ᶠJ, ᶜρ, ᶠu³, ᶜχ, @@ -96,24 +111,34 @@ vertical_transport!( ᶜdivᵥ, ) = @. ᶜρχₜ += -coeff * (ᶜdivᵥ( - ᶠwinterp(ᶜJ, ᶜρ) * ( + ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ( ᶠupwind1(ᶠu³, ᶜχ) + ᶠfct_boris_book( ᶠupwind3(ᶠu³, ᶜχ) - ᶠupwind1(ᶠu³, ᶜχ), - ᶜχ / dt - ᶜdivᵥ(ᶠwinterp(ᶜJ, ᶜρ) * ᶠupwind1(ᶠu³, ᶜχ)) / ᶜρ, + ᶜχ / dt - ᶜdivᵥ(ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ᶠupwind1(ᶠu³, ᶜχ)) / ᶜρ, + ) + ), + )) +vertical_transport!( + coeff, + ᶜρχₜ, + ᶜJ, + ᶠJ, + ᶜρ, + ᶠu³, + ᶜχ, + dt, + ::Val{:zalesak}, + ᶜdivᵥ, +) = @. ᶜρχₜ += + -coeff * (ᶜdivᵥ( + ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ( + ᶠupwind1(ᶠu³, ᶜχ) + ᶠfct_zalesak( + ᶠupwind3(ᶠu³, ᶜχ) - ᶠupwind1(ᶠu³, ᶜχ), + ᶜχ / dt, + ᶜχ / dt - ᶜdivᵥ(ᶠinterp(ᶜJ * ᶜρ) / ᶠJ * ᶠupwind1(ᶠu³, ᶜχ)) / ᶜρ, ) ), )) -vertical_transport!(coeff, ᶜρχₜ, ᶜJ, ᶜρ, ᶠu³, ᶜχ, dt, ::Val{:zalesak}, ᶜdivᵥ) = - @. ᶜρχₜ += - -coeff * (ᶜdivᵥ( - ᶠwinterp(ᶜJ, ᶜρ) * ( - ᶠupwind1(ᶠu³, ᶜχ) + ᶠfct_zalesak( - ᶠupwind3(ᶠu³, ᶜχ) - ᶠupwind1(ᶠu³, ᶜχ), - ᶜχ / dt, - ᶜχ / dt - ᶜdivᵥ(ᶠwinterp(ᶜJ, ᶜρ) * ᶠupwind1(ᶠu³, ᶜχ)) / ᶜρ, - ) - ), - )) vertical_advection!(ᶜρχₜ, ᶠu³, ᶜχ, ::Val{:none}) = @. ᶜρχₜ -= ᶜadvdivᵥ(ᶠu³ * ᶠinterp(ᶜχ)) - ᶜχ * ᶜadvdivᵥ(ᶠu³) @@ -127,17 +152,19 @@ function implicit_vertical_advection_tendency!(Yₜ, Y, p, t) (; dt) = p n = n_mass_flux_subdomains(turbconv_model) ᶜJ = Fields.local_geometry_field(Y.c).J + ᶠJ = Fields.local_geometry_field(Y.f).J (; ᶠgradᵥ_ᶜΦ) = p.core (; ᶜh_tot, ᶜspecific, ᶠu³, ᶜp) = p.precomputed - @. Yₜ.c.ρ -= ᶜdivᵥ(ᶠwinterp(ᶜJ, Y.c.ρ) * ᶠu³) + @. Yₜ.c.ρ -= ᶜdivᵥ(ᶠinterp(ᶜJ * Y.c.ρ) / ᶠJ * ᶠu³) # Central advection of active tracers (e_tot and q_tot) - vertical_transport!(Yₜ.c.ρe_tot, ᶜJ, Y.c.ρ, ᶠu³, ᶜh_tot, dt, Val(:none)) + vertical_transport!(Yₜ.c.ρe_tot, ᶜJ, ᶠJ, Y.c.ρ, ᶠu³, ᶜh_tot, dt, Val(:none)) if !(moisture_model isa DryModel) vertical_transport!( Yₜ.c.ρq_tot, ᶜJ, + ᶠJ, Y.c.ρ, ᶠu³, ᶜspecific.q_tot, @@ -153,17 +180,17 @@ function implicit_vertical_advection_tendency!(Yₜ, Y, p, t) # using downward biasing and free outflow bottom boundary condition (; ᶜwᵣ, ᶜwₛ) = p.precomputed @. Yₜ.c.ρq_rai -= ᶜprecipdivᵥ( - ᶠwinterp(ᶜJ, Y.c.ρ) * + ᶠinterp(ᶜJ * Y.c.ρ) / ᶠJ * ᶠright_bias(Geometry.WVector(-(ᶜwᵣ)) * ᶜspecific.q_rai), ) @. Yₜ.c.ρq_sno -= ᶜprecipdivᵥ( - ᶠwinterp(ᶜJ, Y.c.ρ) * + ᶠinterp(ᶜJ * Y.c.ρ) / ᶠJ * ᶠright_bias(Geometry.WVector(-(ᶜwₛ)) * ᶜspecific.q_sno), ) end - # @. Yₜ.f.u₃ -= ᶠgradᵥ(ᶜp) / ᶠinterp(Y.c.ρ) + ᶠgradᵥ_ᶜΦ - @. Yₜ.f.u₃ -= ᶠinterp(ᶜp / Y.c.ρ) * ᶠgradᵥ(log(ᶜp)) + ᶠgradᵥ_ᶜΦ + @. Yₜ.f.u₃ -= ᶠgradᵥ(ᶜp) / ᶠinterp(Y.c.ρ) + ᶠgradᵥ_ᶜΦ + # @. Yₜ.f.u₃ -= ᶠinterp(ᶜp / Y.c.ρ) * ᶠgradᵥ(log(ᶜp)) + ᶠgradᵥ_ᶜΦ if rayleigh_sponge isa RayleighSponge (; ᶠβ_rayleigh_w) = p.rayleigh_sponge diff --git a/src/surface_conditions/surface_conditions.jl b/src/surface_conditions/surface_conditions.jl index ce065af2638..11e9a2c3fbf 100644 --- a/src/surface_conditions/surface_conditions.jl +++ b/src/surface_conditions/surface_conditions.jl @@ -15,15 +15,20 @@ function update_surface_conditions!(Y, p, t) ) int_local_geometry_values = Fields.field_values(Fields.level(Fields.local_geometry_field(Y.c), 1)) - (; ᶜts, ᶜu, sfc_conditions) = p.precomputed + (; ᶜts, sfc_conditions) = p.precomputed (; params, sfc_setup, atmos) = p thermo_params = CAP.thermodynamics_params(params) surface_fluxes_params = CAP.surface_fluxes_params(params) surface_temp_params = CAP.surface_temp_params(params) int_ts_values = Fields.field_values(Fields.level(ᶜts, 1)) - int_u_values = Fields.field_values(Fields.level(ᶜu, 1)) int_z_values = Fields.field_values(Fields.level(Fields.coordinate_field(Y.c).z, 1)) + FT = eltype(int_z_values) + int_u_values = Base.broadcasted( + *, + FT(NaN), + Base.broadcasted(CT12, int_z_values, int_z_values), + ) sfc_conditions_values = Fields.field_values(sfc_conditions) wrapped_sfc_setup = sfc_setup_wrapper(sfc_setup) sfc_temp_var = diff --git a/src/utils/utilities.jl b/src/utils/utilities.jl index b5c04f5f268..5153e9656d8 100644 --- a/src/utils/utilities.jl +++ b/src/utils/utilities.jl @@ -65,14 +65,7 @@ individual velocity components: - `uᵥ` should be a `Covariant3Vector`-valued field at cell faces. """ function compute_kinetic!(κ::Fields.Field, uₕ::Fields.Field, uᵥ::Fields.Field) - @assert eltype(uₕ) <: Union{C1, C2, C12} - @assert eltype(uᵥ) <: C3 - @. κ = - 1 / 2 * ( - dot(C123(uₕ), CT123(uₕ)) + - ᶜinterp(dot(C123(uᵥ), CT123(uᵥ))) + - 2 * dot(CT123(uₕ), ᶜinterp(C123(uᵥ))) - ) + error("TODO") end """