Merge #2137 #2141

2137: Fix closures resulting in boxed variables r=charleskawczynski a=charleskawczynski This PR fixes a closure in the diagnostics, which results in boxing variables. Issue found in https://buildkite.com/clima/climaatmos-ci/builds/13072#018ab62d-8432-4918-8532-8c2bed825ff0 2141: Update dependencies r=charleskawczynski a=charleskawczynski This PR upgrades the dependencies, including ClimaTimeSteppers. The new version fuses some operations, resulting in precision error differences. Co-authored-by: Charles Kawczynski <kawczynski.charles@gmail.com>
CliMA · Sep 22, 2023 · 8a17601 · 8a17601
3 parents 0b4a150 + 6663019 + 7361335
commit 8a17601
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 67 deletions.
diff --git a/docs/Manifest.toml b/docs/Manifest.toml
@@ -265,10 +265,10 @@ uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
 version = "0.10.51"
 
 [[deps.ClimaTimeSteppers]]
-deps = ["CUDA", "ClimaComms", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "SciMLBase", "StaticArrays"]
-git-tree-sha1 = "54b602435b0107b6c2dfe7664e0f7ff5fc78fb91"
+deps = ["CUDA", "ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"]
+git-tree-sha1 = "70b0118d12a137550f1626bf4c20ebb66a45f2e0"
 uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-version = "0.7.7"
+version = "0.7.8"
 
 [[deps.CloseOpenIntervals]]
 deps = ["Static", "StaticArrayInterface"]
@@ -341,6 +341,11 @@ git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7"
 uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3"
 version = "0.3.2"
 
+[[deps.ConcreteStructs]]
+git-tree-sha1 = "f749037478283d372048690eb3b5f92a79432b34"
+uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+version = "0.2.3"
+
 [[deps.ConstructionBase]]
 deps = ["LinearAlgebra"]
 git-tree-sha1 = "c53fc348ca4d40d7b371e71fd52251839080cbc9"
@@ -960,24 +965,28 @@ uuid = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125"
 version = "2.5.2"
 
 [[deps.LinearSolve]]
-deps = ["ArrayInterface", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"]
-git-tree-sha1 = "69cbd612e6e67ba2f8121bc8725bc9d04d803599"
+deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"]
+git-tree-sha1 = "dd70543d3c4fc712c14d67e2d35b90817e8bc37d"
 uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
-version = "2.5.1"
+version = "2.6.0"
 
     [deps.LinearSolve.extensions]
+    LinearSolveBlockDiagonalsExt = "BlockDiagonals"
     LinearSolveCUDAExt = "CUDA"
     LinearSolveHYPREExt = "HYPRE"
     LinearSolveIterativeSolversExt = "IterativeSolvers"
+    LinearSolveKernelAbstractionsExt = "KernelAbstractions"
     LinearSolveKrylovKitExt = "KrylovKit"
     LinearSolveMKLExt = "MKL_jll"
     LinearSolveMetalExt = "Metal"
     LinearSolvePardisoExt = "Pardiso"
 
     [deps.LinearSolve.weakdeps]
+    BlockDiagonals = "0a1fb500-61f7-11e9-3c65-f5ef3456f9f0"
     CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
     HYPRE = "b5ffcf37-a2bd-41ab-a3da-4bd9bc8ad771"
     IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153"
+    KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
     KrylovKit = "0b1a1467-8014-51b9-945f-bf0ae24f4b77"
     MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
     Metal = "dde4c033-4e86-420c-a63e-0dd931031962"
@@ -1182,9 +1191,9 @@ version = "4.1.5+0"
 
 [[deps.OpenSSL_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "e78db7bd5c26fc5a6911b50a47ee302219157ea8"
+git-tree-sha1 = "ceeda72c9fd6bbebc4f4f598560789145a8b6c4c"
 uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
-version = "3.0.10+0"
+version = "3.0.11+0"
 
 [[deps.OpenSpecFun_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]

diff --git a/examples/Manifest.toml b/examples/Manifest.toml
@@ -334,10 +334,10 @@ uuid = "c8b6d40d-e815-466f-95ae-c48aefa668fa"
 version = "0.7.1"
 
 [[deps.ClimaTimeSteppers]]
-deps = ["CUDA", "ClimaComms", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "SciMLBase", "StaticArrays"]
-git-tree-sha1 = "54b602435b0107b6c2dfe7664e0f7ff5fc78fb91"
+deps = ["CUDA", "ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"]
+git-tree-sha1 = "70b0118d12a137550f1626bf4c20ebb66a45f2e0"
 uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-version = "0.7.7"
+version = "0.7.8"
 
 [[deps.CloseOpenIntervals]]
 deps = ["Static", "StaticArrayInterface"]
@@ -429,6 +429,11 @@ git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7"
 uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3"
 version = "0.3.2"
 
+[[deps.ConcreteStructs]]
+git-tree-sha1 = "f749037478283d372048690eb3b5f92a79432b34"
+uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+version = "0.2.3"
+
 [[deps.ConcurrentUtilities]]
 deps = ["Serialization", "Sockets"]
 git-tree-sha1 = "5372dbbf8f0bdb8c700db5367132925c0771ef7e"
@@ -1389,24 +1394,28 @@ uuid = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125"
 version = "2.5.2"
 
 [[deps.LinearSolve]]
-deps = ["ArrayInterface", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"]
-git-tree-sha1 = "69cbd612e6e67ba2f8121bc8725bc9d04d803599"
+deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"]
+git-tree-sha1 = "dd70543d3c4fc712c14d67e2d35b90817e8bc37d"
 uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
-version = "2.5.1"
+version = "2.6.0"
 
     [deps.LinearSolve.extensions]
+    LinearSolveBlockDiagonalsExt = "BlockDiagonals"
     LinearSolveCUDAExt = "CUDA"
     LinearSolveHYPREExt = "HYPRE"
     LinearSolveIterativeSolversExt = "IterativeSolvers"
+    LinearSolveKernelAbstractionsExt = "KernelAbstractions"
     LinearSolveKrylovKitExt = "KrylovKit"
     LinearSolveMKLExt = "MKL_jll"
     LinearSolveMetalExt = "Metal"
     LinearSolvePardisoExt = "Pardiso"
 
     [deps.LinearSolve.weakdeps]
+    BlockDiagonals = "0a1fb500-61f7-11e9-3c65-f5ef3456f9f0"
     CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
     HYPRE = "b5ffcf37-a2bd-41ab-a3da-4bd9bc8ad771"
     IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153"
+    KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
     KrylovKit = "0b1a1467-8014-51b9-945f-bf0ae24f4b77"
     MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
     Metal = "dde4c033-4e86-420c-a63e-0dd931031962"

diff --git a/perf/Manifest.toml b/perf/Manifest.toml
@@ -345,10 +345,10 @@ uuid = "c8b6d40d-e815-466f-95ae-c48aefa668fa"
 version = "0.7.1"
 
 [[deps.ClimaTimeSteppers]]
-deps = ["CUDA", "ClimaComms", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "SciMLBase", "StaticArrays"]
-git-tree-sha1 = "54b602435b0107b6c2dfe7664e0f7ff5fc78fb91"
+deps = ["CUDA", "ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"]
+git-tree-sha1 = "70b0118d12a137550f1626bf4c20ebb66a45f2e0"
 uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79"
-version = "0.7.7"
+version = "0.7.8"
 
 [[deps.CloseOpenIntervals]]
 deps = ["Static", "StaticArrayInterface"]
@@ -440,6 +440,11 @@ git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7"
 uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3"
 version = "0.3.2"
 
+[[deps.ConcreteStructs]]
+git-tree-sha1 = "f749037478283d372048690eb3b5f92a79432b34"
+uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+version = "0.2.3"
+
 [[deps.ConcurrentUtilities]]
 deps = ["Serialization", "Sockets"]
 git-tree-sha1 = "5372dbbf8f0bdb8c700db5367132925c0771ef7e"
@@ -1441,24 +1446,28 @@ uuid = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125"
 version = "2.5.2"
 
 [[deps.LinearSolve]]
-deps = ["ArrayInterface", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"]
-git-tree-sha1 = "69cbd612e6e67ba2f8121bc8725bc9d04d803599"
+deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"]
+git-tree-sha1 = "dd70543d3c4fc712c14d67e2d35b90817e8bc37d"
 uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
-version = "2.5.1"
+version = "2.6.0"
 
     [deps.LinearSolve.extensions]
+    LinearSolveBlockDiagonalsExt = "BlockDiagonals"
     LinearSolveCUDAExt = "CUDA"
     LinearSolveHYPREExt = "HYPRE"
     LinearSolveIterativeSolversExt = "IterativeSolvers"
+    LinearSolveKernelAbstractionsExt = "KernelAbstractions"
     LinearSolveKrylovKitExt = "KrylovKit"
     LinearSolveMKLExt = "MKL_jll"
     LinearSolveMetalExt = "Metal"
     LinearSolvePardisoExt = "Pardiso"
 
     [deps.LinearSolve.weakdeps]
+    BlockDiagonals = "0a1fb500-61f7-11e9-3c65-f5ef3456f9f0"
     CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
     HYPRE = "b5ffcf37-a2bd-41ab-a3da-4bd9bc8ad771"
     IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153"
+    KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
     KrylovKit = "0b1a1467-8014-51b9-945f-bf0ae24f4b77"
     MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
     Metal = "dde4c033-4e86-420c-a63e-0dd931031962"

diff --git a/regression_tests/ref_counter.jl b/regression_tests/ref_counter.jl
@@ -1 +1 @@
-127
+128
diff --git a/src/callbacks/callbacks.jl b/src/callbacks/callbacks.jl
@@ -207,48 +207,58 @@ NVTX.@annotate function rrtmgp_model_callback!(integrator)
     return nothing
 end
 
+function common_diagnostics(p, ᶜu, ᶜts)
+    (; env_thermo_quad, params) = p
+    thermo_params = CAP.thermodynamics_params(params)
+    ᶜρ = TD.air_density.(thermo_params, ᶜts)
+    diagnostics = (;
+        u_velocity = Geometry.UVector.(ᶜu),
+        v_velocity = Geometry.VVector.(ᶜu),
+        w_velocity = Geometry.WVector.(ᶜu),
+        temperature = TD.air_temperature.(thermo_params, ᶜts),
+        potential_temperature = TD.dry_pottemp.(thermo_params, ᶜts),
+        specific_enthalpy = TD.specific_enthalpy.(thermo_params, ᶜts),
+        buoyancy = CAP.grav(p.params) .* (p.ᶜρ_ref .- ᶜρ) ./ ᶜρ,
+    )
+    if !(p.atmos.moisture_model isa DryModel)
+        diagnostics = (;
+            diagnostics...,
+            q_vap = TD.vapor_specific_humidity.(thermo_params, ᶜts),
+            q_liq = TD.liquid_specific_humidity.(thermo_params, ᶜts),
+            q_ice = TD.ice_specific_humidity.(thermo_params, ᶜts),
+            q_tot = TD.total_specific_humidity.(thermo_params, ᶜts),
+            relative_humidity = TD.relative_humidity.(thermo_params, ᶜts),
+            cloud_fraction_gm = get_cloud_fraction.(
+                thermo_params,
+                env_thermo_quad,
+                p.ᶜp,
+                ᶜts,
+            ),
+        )
+    end
+    return diagnostics
+end
+
+# Adds a prefix to the front of each name in the named tuple. This function
+# is not type stable, but that probably doesn't matter for diagnostics.
+add_prefix(diagnostics::NamedTuple{names}, prefix) where {names} =
+    NamedTuple{Symbol.(prefix, names)}(values(diagnostics))
+
+cloud_fraction(thermo_params, ts, area::FT) where {FT} =
+    TD.has_condensate(thermo_params, ts) && area > 1e-3 ? FT(1) : FT(0)
+
 NVTX.@annotate function compute_diagnostics(integrator)
     (; t, u, p) = integrator
     Y = u
     (; params, env_thermo_quad) = p
     FT = eltype(params)
     thermo_params = CAP.thermodynamics_params(params)
 
-    function common_diagnostics(ᶜu, ᶜts)
-        ᶜρ = TD.air_density.(thermo_params, ᶜts)
-        diagnostics = (;
-            u_velocity = Geometry.UVector.(ᶜu),
-            v_velocity = Geometry.VVector.(ᶜu),
-            w_velocity = Geometry.WVector.(ᶜu),
-            temperature = TD.air_temperature.(thermo_params, ᶜts),
-            potential_temperature = TD.dry_pottemp.(thermo_params, ᶜts),
-            specific_enthalpy = TD.specific_enthalpy.(thermo_params, ᶜts),
-            buoyancy = CAP.grav(params) .* (p.ᶜρ_ref .- ᶜρ) ./ ᶜρ,
-        )
-        if !(p.atmos.moisture_model isa DryModel)
-            diagnostics = (;
-                diagnostics...,
-                q_vap = TD.vapor_specific_humidity.(thermo_params, ᶜts),
-                q_liq = TD.liquid_specific_humidity.(thermo_params, ᶜts),
-                q_ice = TD.ice_specific_humidity.(thermo_params, ᶜts),
-                q_tot = TD.total_specific_humidity.(thermo_params, ᶜts),
-                relative_humidity = TD.relative_humidity.(thermo_params, ᶜts),
-                cloud_fraction_gm = get_cloud_fraction.(
-                    thermo_params,
-                    env_thermo_quad,
-                    ᶜp,
-                    ᶜts,
-                ),
-            )
-        end
-        return diagnostics
-    end
-
     set_precomputed_quantities!(Y, p, t) # sets ᶜu, ᶜK, ᶜts, ᶜp, & SGS analogues
 
     (; ᶜu, ᶜK, ᶜts, ᶜp, sfc_conditions) = p
     dycore_diagnostic = (;
-        common_diagnostics(ᶜu, ᶜts)...,
+        common_diagnostics(p, ᶜu, ᶜts)...,
         pressure = ᶜp,
         kinetic_energy = ᶜK,
         sfc_temperature = TD.air_temperature.(thermo_params, sfc_conditions.ts),
@@ -281,19 +291,11 @@ NVTX.@annotate function compute_diagnostics(integrator)
         precip_diagnostic = NamedTuple()
     end
 
-    # Adds a prefix to the front of each name in the named tuple. This function
-    # is not type stable, but that probably doesn't matter for diagnostics.
-    add_prefix(diagnostics::NamedTuple{names}, prefix) where {names} =
-        NamedTuple{Symbol.(prefix, names)}(values(diagnostics))
-
-    cloud_fraction(ts, area::FT) where {FT} =
-        TD.has_condensate(thermo_params, ts) && area > 1e-3 ? FT(1) : FT(0)
-
     if p.atmos.turbconv_model isa EDMFX
         (; ᶜspecific⁰, ᶜu⁰, ᶜts⁰, ᶜmixing_length) = p
         (; ᶜu⁺, ᶜts⁺, ᶜa⁺, ᶜa⁰) = output_sgs_quantities(Y, p, t)
         env_diagnostics = (;
-            common_diagnostics(ᶜu⁰, ᶜts⁰)...,
+            common_diagnostics(p, ᶜu⁰, ᶜts⁰)...,
             area = ᶜa⁰,
             cloud_fraction = get_cloud_fraction.(
                 thermo_params,
@@ -305,9 +307,9 @@ NVTX.@annotate function compute_diagnostics(integrator)
             mixing_length = ᶜmixing_length,
         )
         draft_diagnostics = (;
-            common_diagnostics(ᶜu⁺, ᶜts⁺)...,
+            common_diagnostics(p, ᶜu⁺, ᶜts⁺)...,
             area = ᶜa⁺,
-            cloud_fraction = cloud_fraction.(ᶜts⁺, ᶜa⁺),
+            cloud_fraction = cloud_fraction.(thermo_params, ᶜts⁺, ᶜa⁺),
         )
         turbulence_convection_diagnostic = (;
             add_prefix(env_diagnostics, :env_)...,
@@ -318,7 +320,7 @@ NVTX.@annotate function compute_diagnostics(integrator)
                 env_thermo_quad,
                 ᶜp,
                 ᶜts⁰,
-            ) .+ ᶜa⁺ .* cloud_fraction.(ᶜts⁺, ᶜa⁺),
+            ) .+ ᶜa⁺ .* cloud_fraction.(thermo_params, ᶜts⁺, ᶜa⁺),
         )
     elseif p.atmos.turbconv_model isa DiagnosticEDMFX
         (; ᶜtke⁰, ᶜmixing_length) = p
@@ -334,9 +336,9 @@ NVTX.@annotate function compute_diagnostics(integrator)
             mixing_length = ᶜmixing_length,
         )
         draft_diagnostics = (;
-            common_diagnostics(ᶜu⁺, ᶜts⁺)...,
+            common_diagnostics(p, ᶜu⁺, ᶜts⁺)...,
             area = ᶜa⁺,
-            cloud_fraction = cloud_fraction.(ᶜts⁺, ᶜa⁺),
+            cloud_fraction = cloud_fraction.(thermo_params, ᶜts⁺, ᶜa⁺),
         )
         turbulence_convection_diagnostic = (;
             add_prefix(env_diagnostics, :env_)...,
@@ -346,7 +348,7 @@ NVTX.@annotate function compute_diagnostics(integrator)
                 env_thermo_quad,
                 ᶜp,
                 ᶜts,
-            ) .+ ᶜa⁺ .* cloud_fraction.(ᶜts⁺, ᶜa⁺),
+            ) .+ ᶜa⁺ .* cloud_fraction.(thermo_params, ᶜts⁺, ᶜa⁺),
         )
     elseif p.atmos.turbconv_model isa TC.EDMFModel
         tc_cent(p) = p.edmf_cache.aux.c.turbconv