Merge branch 'main' into sk/update_gpu_aquaplanet_chap_config

.buildkite/longruns/pipeline.yml

@@ -1,22 +1,17 @@
 agents:
-  queue: central
+  queue: new-central
   slurm_mem_per_cpu: 8G
-  modules: julia/1.10.0 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
+  modules: climacommon/2024_02_27
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
   OPENBLAS_NUM_THREADS: 1
   JULIA_NVTX_CALLBACKS: gc
-  OMPI_MCA_opal_warn_on_missing_libcuda: 0
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   JULIA_CPU_TARGET: 'broadwell;skylake;icelake;cascadelake;epyc'
   CONFIG_PATH: "config/longrun_configs"
   SLURM_KILL_BAD_EXIT: 1
   JULIA_NVTX_CALLBACKS: gc
-  JULIA_CUDA_MEMORY_POOL: none
-  JULIA_MPI_HAS_CUDA: "true"
-  MPITRAMPOLINE_LIB: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so"
-  MPITRAMPOLINE_MPIEXEC: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec"
 
 timeout_in_minutes: 1440
 
@@ -48,17 +43,6 @@ steps:
 
     steps:
 
-      - label: ":computer: hydrostatic balance (ρe_tot)"
-        command:
-          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
-        artifact_paths: "$$JOB_NAME/*"
-        agents:
-          slurm_gpus: 1
-          slurm_ntasks: 32
-          slurm_time: 24:00:00
-        env:
-          JOB_NAME: "longrun_sphere_hydrostatic_balance_rhoe"
-
       # TODO: uncomment when zalesak works
       # - label: ":computer: lim ARS zalesak baroclinic wave (ρe_tot) equilmoist high resolution"
       #   command:

.buildkite/longruns_gpu/pipeline.yml

@@ -42,6 +42,17 @@ steps:
   - group: "Targeted resolution AMIP long runs"
     steps:
 
+      - label: ":computer: hydrostatic balance (ρe_tot)"
+        command:
+          - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
+        artifact_paths: "$$JOB_NAME/*"
+        agents:
+          slurm_gpus: 1
+          slurm_cpus_per_task: 4
+          slurm_time: 24:00:00
+        env:
+          JOB_NAME: "longrun_sphere_hydrostatic_balance_rhoe"
+
       - label: ":computer: dry baroclinic wave"
         command:
           - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml

.buildkite/pipeline.yml

@@ -638,29 +638,6 @@ steps:
           - "sphere_baroclinic_wave_rhoe"
           - "gpu_baroclinic_wave_rhoe"
 
-      - label: "GPU: HS (ρe) hightop"
-        key: "gpu_held_suarez_rhoe_hightop"
-        command: >
-          julia --color=yes --project=examples examples/hybrid/driver.jl
-          --config_file $PERF_CONFIG_PATH/gpu_held_suarez_rhoe_hightop.yml
-        artifact_paths: "gpu_held_suarez_rhoe_hightop/*"
-        agents:
-          slurm_gpus: 1
-
-      - label: "GPU: compare HS (ρe) hightop with CPU"
-        command: >
-          tar xvf sphere_held_suarez_rhoe_hightop/hdf5_files.tar -C sphere_held_suarez_rhoe_hightop
-
-          tar xvf gpu_held_suarez_rhoe_hightop/hdf5_files.tar -C gpu_held_suarez_rhoe_hightop
-
-          julia --color=yes --project=examples post_processing/compare_outputs.jl
-          --output_folder_1 sphere_held_suarez_rhoe_hightop/
-          --output_folder_2 gpu_held_suarez_rhoe_hightop/
-          --t_end 8days --compare_state false
-        depends_on:
-          - "sphere_held_suarez_rhoe_hightop"
-          - "gpu_held_suarez_rhoe_hightop"
-
       - label: "GPU: GPU dry baroclinic wave"
         key: "target_gpu_implicit_baroclinic_wave"
         command:

Project.toml

@@ -50,9 +50,9 @@ Artifacts = "1"
 AtmosphericProfilesLibrary = "0.1"
 CLIMAParameters = "0.9"
 CPUSummary = "0.2"
-ClimaComms = "0.5.6"
+ClimaComms = "0.5"
 ClimaCore = "0.12"
-ClimaTimeSteppers = "0.7.14"
+ClimaTimeSteppers = "0.7"
 CloudMicrophysics = "0.16"
 Colors = "0.12"
 Dates = "1"

docs/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.0"
 manifest_format = "2.0"
-project_hash = "9f84ec58e509508e57495c836422d311eb03b970"
+project_hash = "8b4b2869b2bfd32e14c865e13a99a29882804775"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "41c37aa88889c171f1300ceac1313c06e891d245"
@@ -329,9 +329,9 @@ weakdeps = ["Krylov"]
 
 [[deps.ClimaTimeSteppers]]
 deps = ["ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"]
-git-tree-sha1 = "216385dda9966d820b484f399b90447f073893a6"
+git-tree-sha1 = "96bbba6d14467a2b9512ba0a536395350bb361ff"
 uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-version = "0.7.16"
+version = "0.7.17"
 
 [[deps.CloseOpenIntervals]]
 deps = ["Static", "StaticArrayInterface"]

examples/Manifest.toml

@@ -194,9 +194,9 @@ version = "0.5.0"
 
 [[deps.CFTime]]
 deps = ["Dates", "Printf"]
-git-tree-sha1 = "ed2e76c1c3c43fd9d0cb9248674620b29d71f2d1"
+git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1"
 uuid = "179af706-886a-5703-950a-314cd64e0468"
-version = "0.1.2"
+version = "0.1.3"
 
 [[deps.CLIMAParameters]]
 deps = ["DocStringExtensions", "TOML", "Test"]
@@ -352,9 +352,9 @@ version = "0.7.4"
 
 [[deps.ClimaTimeSteppers]]
 deps = ["ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"]
-git-tree-sha1 = "216385dda9966d820b484f399b90447f073893a6"
+git-tree-sha1 = "96bbba6d14467a2b9512ba0a536395350bb361ff"
 uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-version = "0.7.16"
+version = "0.7.17"
 
 [[deps.CloseOpenIntervals]]
 deps = ["Static", "StaticArrayInterface"]
@@ -548,9 +548,9 @@ version = "0.1.0+0"
 
 [[deps.DiffEqBase]]
 deps = ["ArrayInterface", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"]
-git-tree-sha1 = "2ad3a2dcd5f28f535aa884d199cc2f0a9d335729"
+git-tree-sha1 = "4b525a4fb824c7c3a98d5d1b18fbaee239dfed7c"
 uuid = "2b5f629d-d688-5b77-993f-72d75c75574e"
-version = "6.147.1"
+version = "6.147.2"
 
     [deps.DiffEqBase.extensions]
     DiffEqBaseChainRulesCoreExt = "ChainRulesCore"
@@ -1997,9 +1997,9 @@ version = "1.2.0"
 
 [[deps.Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e"
+git-tree-sha1 = "9e8fed0505b0c15b4c1295fd59ea47b411c019cf"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.4.1"
+version = "1.4.2"
 
 [[deps.PrettyTables]]
 deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"]

perf/Manifest.toml

@@ -199,9 +199,9 @@ version = "0.5.0"
 
 [[deps.CFTime]]
 deps = ["Dates", "Printf"]
-git-tree-sha1 = "ed2e76c1c3c43fd9d0cb9248674620b29d71f2d1"
+git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1"
 uuid = "179af706-886a-5703-950a-314cd64e0468"
-version = "0.1.2"
+version = "0.1.3"
 
 [[deps.CLIMAParameters]]
 deps = ["DocStringExtensions", "TOML", "Test"]
@@ -357,9 +357,9 @@ version = "0.7.4"
 
 [[deps.ClimaTimeSteppers]]
 deps = ["ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"]
-git-tree-sha1 = "216385dda9966d820b484f399b90447f073893a6"
+git-tree-sha1 = "96bbba6d14467a2b9512ba0a536395350bb361ff"
 uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-version = "0.7.16"
+version = "0.7.17"
 
 [[deps.CloseOpenIntervals]]
 deps = ["Static", "StaticArrayInterface"]
@@ -559,9 +559,9 @@ version = "0.1.0+0"
 
 [[deps.DiffEqBase]]
 deps = ["ArrayInterface", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"]
-git-tree-sha1 = "2ad3a2dcd5f28f535aa884d199cc2f0a9d335729"
+git-tree-sha1 = "4b525a4fb824c7c3a98d5d1b18fbaee239dfed7c"
 uuid = "2b5f629d-d688-5b77-993f-72d75c75574e"
-version = "6.147.1"
+version = "6.147.2"
 
     [deps.DiffEqBase.extensions]
     DiffEqBaseChainRulesCoreExt = "ChainRulesCore"
@@ -2049,9 +2049,9 @@ version = "1.2.0"
 
 [[deps.Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e"
+git-tree-sha1 = "9e8fed0505b0c15b4c1295fd59ea47b411c019cf"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.4.1"
+version = "1.4.2"
 
 [[deps.PrettyTables]]
 deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"]

regression_tests/ref_counter.jl

@@ -1 +1 @@
-152
+153

src/prognostic_equations/advection.jl

@@ -154,6 +154,11 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t)
                 (ᶜf³[colidx] + ᶜω³[colidx]) × CT12(ᶜu[colidx])
             @. Yₜ.f.u₃[colidx] -=
                 ᶠω¹²[colidx] × ᶠinterp(CT12(ᶜu[colidx])) + ᶠgradᵥ(ᶜK[colidx])
+            for j in 1:n
+                @. Yₜ.f.sgsʲs.:($$j).u₃[colidx] -=
+                    ᶠω¹²ʲs.:($$j)[colidx] × ᶠinterp(CT12(ᶜuʲs.:($$j)[colidx])) +
+                    ᶠgradᵥ(ᶜKʲs.:($$j)[colidx] - ᶜKᵥʲs.:($$j)[colidx])
+            end
         else
             # deep atmosphere
             @. Yₜ.c.uₕ[colidx] -=
@@ -165,11 +170,12 @@ NVTX.@annotate function explicit_vertical_advection_tendency!(Yₜ, Y, p, t)
             @. Yₜ.f.u₃[colidx] -=
                 (ᶠf¹²[colidx] + ᶠω¹²[colidx]) × ᶠinterp(CT12(ᶜu[colidx])) +
                 ᶠgradᵥ(ᶜK[colidx])
-        end
-        for j in 1:n
-            @. Yₜ.f.sgsʲs.:($$j).u₃[colidx] -=
-                ᶠω¹²ʲs.:($$j)[colidx] × ᶠinterp(CT12(ᶜuʲs.:($$j)[colidx])) +
-                ᶠgradᵥ(ᶜKʲs.:($$j)[colidx] - ᶜKᵥʲs.:($$j)[colidx])
+            for j in 1:n
+                @. Yₜ.f.sgsʲs.:($$j).u₃[colidx] -=
+                    (ᶠf¹²[colidx] + ᶠω¹²ʲs.:($$j)[colidx]) ×
+                    ᶠinterp(CT12(ᶜuʲs.:($$j)[colidx])) +
+                    ᶠgradᵥ(ᶜKʲs.:($$j)[colidx] - ᶜKᵥʲs.:($$j)[colidx])
+            end
         end
 
         if use_prognostic_tke(turbconv_model) # advect_tke triggers allocations

toml/bomex_box_rhoe.toml

@@ -1,4 +1,3 @@
 [C_E]
 value = 0.044
-type = "float"
 

toml/diagnostic_edmfx_box.toml

@@ -1,27 +1,20 @@
 [entr_inv_tau]
 value = 0.002
-type = "float"
 
 [entr_coeff]
 value = 0
-type = "float"
 
 [detr_inv_tau]
 value = 0
-type = "float"
 
 [detr_buoy_coeff]
 value = 0.12
-type = "float"
 
 [detr_vertdiv_coeff]
 value = 0.6
-type = "float"
 
 [min_area_limiter_scale]
 value = 0
-type = "float"
 
 [max_area_limiter_scale]
 value = 0
-type = "float"

toml/diagnostic_edmfx_trmm_box.toml

@@ -1,31 +1,23 @@
 [precipitation_timescale]
 value = 600
-type = "float"
 
 [entr_inv_tau]
 value = 0.002
-type = "float"
 
 [entr_coeff]
 value = 0
-type = "float"
 
 [detr_inv_tau]
 value = 0
-type = "float"
 
 [detr_vertdiv_coeff]
 value = 0.6
-type = "float"
 
 [detr_buoy_coeff]
 value = 0.12
-type = "float"
 
 [min_area_limiter_scale]
 value = 0
-type = "float"
 
 [max_area_limiter_scale]
 value = 0
-type = "float"

toml/flame_perf_gw.toml

@@ -1,3 +1,2 @@
 [zd_rayleigh]
 value = 30000.0
-type = "float"

toml/longrun_1M.toml

@@ -1,3 +1,2 @@
 [zd_rayleigh]
 value = 35000.0
-type = "float"

toml/longrun_aquaplanet_amip.toml

@@ -1,36 +1,27 @@
 [zd_rayleigh]
 value = 35000.0
-type = "float"
 
 [alpha_rayleigh_uh]
 alias = "alpha_rayleigh_uh"
 value = 0.0
-type = "float"
 
 [precipitation_timescale]
 value = 600
-type = "float"
 
 [entr_inv_tau]
 value = 0.001
-type = "float"
 
 [entr_coeff]
 value = 0
-type = "float"
 
 [detr_inv_tau]
 value = 0
-type = "float"
 
 [detr_buoy_coeff]
 value = 0.12
-type = "float"
 
 [min_area_limiter_scale]
 value = 0
-type = "float"
 
 [max_area_limiter_scale]
 value = 0
-type = "float"

toml/longrun_aquaplanet_dyamond.toml

@@ -1,8 +1,6 @@
 [zd_rayleigh]
 value = 35000.0
-type = "float"
 
 [alpha_rayleigh_uh]
 alias = "alpha_rayleigh_uh"
 value = 0.0
-type = "float"