diff --git a/.dev/Manifest.toml b/.dev/Manifest.toml index 08985d3afa..414f21880d 100644 --- a/.dev/Manifest.toml +++ b/.dev/Manifest.toml @@ -79,9 +79,9 @@ version = "0.21.4" [[deps.JuliaFormatter]] deps = ["CSTParser", "CommonMark", "DataStructures", "Glob", "Pkg", "PrecompileTools", "Tokenize"] -git-tree-sha1 = "c0ac2f93058fd148267154c42dd7dc79869f7923" +git-tree-sha1 = "80031f6e58b09b0de4553bf63d9a36ec5db57967" uuid = "98e50ef6-434e-11e9-1051-2b60c6c9e899" -version = "1.0.37" +version = "1.0.39" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 788a63abab..3c866c0e0d 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -92,9 +92,9 @@ version = "0.1.29" [[deps.ArrayLayouts]] deps = ["FillArrays", "LinearAlgebra"] -git-tree-sha1 = "0d61921af2799487b80453a44abb57db7a0c1381" +git-tree-sha1 = "9a731850434825d183af39c6e6cd0a1c32dd7e20" uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" -version = "1.4.1" +version = "1.4.2" weakdeps = ["SparseArrays"] [deps.ArrayLayouts.extensions] @@ -266,9 +266,9 @@ version = "0.10.51" [[deps.ClimaTimeSteppers]] deps = ["CUDA", "ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"] -git-tree-sha1 = "1b437bf71f2e35925e957715361bcfa60f68351a" +git-tree-sha1 = "6b8068a0d888e1ec83e3001526cb8050f2ef39fe" uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" -version = "0.7.10" +version = "0.7.13" [[deps.CloseOpenIntervals]] deps = ["Static", "StaticArrayInterface"] @@ -1233,9 +1233,9 @@ version = "6.57.0" [[deps.PDMats]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "b7c4f29f93b548caa58f703580f4d79ab753c8ac" +git-tree-sha1 = "528664265c9c36b3ecdb6d721d47aaab52ddf267" uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.21" +version = "0.11.24" [[deps.PackageExtensionCompat]] git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518" diff --git a/examples/Manifest.toml b/examples/Manifest.toml index 136765dac3..cf978d6059 100644 --- a/examples/Manifest.toml +++ b/examples/Manifest.toml @@ -98,9 +98,9 @@ version = "0.1.29" [[deps.ArrayLayouts]] deps = ["FillArrays", "LinearAlgebra"] -git-tree-sha1 = "0d61921af2799487b80453a44abb57db7a0c1381" +git-tree-sha1 = "9a731850434825d183af39c6e6cd0a1c32dd7e20" uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" -version = "1.4.1" +version = "1.4.2" weakdeps = ["SparseArrays"] [deps.ArrayLayouts.extensions] @@ -335,9 +335,9 @@ version = "0.7.1" [[deps.ClimaTimeSteppers]] deps = ["CUDA", "ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"] -git-tree-sha1 = "1b437bf71f2e35925e957715361bcfa60f68351a" +git-tree-sha1 = "6b8068a0d888e1ec83e3001526cb8050f2ef39fe" uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" -version = "0.7.10" +version = "0.7.13" [[deps.CloseOpenIntervals]] deps = ["Static", "StaticArrayInterface"] @@ -1446,9 +1446,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "0d097476b6c381ab7906460ef1ef1638fbce1d91" +git-tree-sha1 = "c1dd6d7978c12545b4179fb6153b9250c96b0075" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.0.2" +version = "1.0.3" [[deps.LoopVectorization]] deps = ["ArrayInterface", "ArrayInterfaceCore", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] @@ -1767,9 +1767,9 @@ version = "10.42.0+0" [[deps.PDMats]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "b7c4f29f93b548caa58f703580f4d79ab753c8ac" +git-tree-sha1 = "528664265c9c36b3ecdb6d721d47aaab52ddf267" uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.21" +version = "0.11.24" [[deps.PNGFiles]] deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] diff --git a/perf/Manifest.toml b/perf/Manifest.toml index 2a6ff98a5e..d647fabe06 100644 --- a/perf/Manifest.toml +++ b/perf/Manifest.toml @@ -98,9 +98,9 @@ version = "0.1.29" [[deps.ArrayLayouts]] deps = ["FillArrays", "LinearAlgebra"] -git-tree-sha1 = "0d61921af2799487b80453a44abb57db7a0c1381" +git-tree-sha1 = "9a731850434825d183af39c6e6cd0a1c32dd7e20" uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" -version = "1.4.1" +version = "1.4.2" weakdeps = ["SparseArrays"] [deps.ArrayLayouts.extensions] @@ -346,9 +346,9 @@ version = "0.7.1" [[deps.ClimaTimeSteppers]] deps = ["CUDA", "ClimaComms", "Colors", "DataStructures", "DiffEqBase", "DiffEqCallbacks", "KernelAbstractions", "Krylov", "LinearAlgebra", "LinearOperators", "NVTX", "SciMLBase", "StaticArrays"] -git-tree-sha1 = "1b437bf71f2e35925e957715361bcfa60f68351a" +git-tree-sha1 = "6b8068a0d888e1ec83e3001526cb8050f2ef39fe" uuid = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" -version = "0.7.10" +version = "0.7.13" [[deps.CloseOpenIntervals]] deps = ["Static", "StaticArrayInterface"] @@ -1498,9 +1498,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.LoggingExtras]] deps = ["Dates", "Logging"] -git-tree-sha1 = "0d097476b6c381ab7906460ef1ef1638fbce1d91" +git-tree-sha1 = "c1dd6d7978c12545b4179fb6153b9250c96b0075" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.0.2" +version = "1.0.3" [[deps.LoopVectorization]] deps = ["ArrayInterface", "ArrayInterfaceCore", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] @@ -1813,9 +1813,9 @@ version = "10.42.0+0" [[deps.PDMats]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "b7c4f29f93b548caa58f703580f4d79ab753c8ac" +git-tree-sha1 = "528664265c9c36b3ecdb6d721d47aaab52ddf267" uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.11.21" +version = "0.11.24" [[deps.PNGFiles]] deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] diff --git a/perf/flame.jl b/perf/flame.jl index 80ab8d826b..c946aad3c3 100644 --- a/perf/flame.jl +++ b/perf/flame.jl @@ -58,7 +58,7 @@ allocs_limit["flame_perf_target"] = 12864 allocs_limit["flame_perf_target_tracers"] = 212496 allocs_limit["flame_perf_target_edmfx"] = 304064 allocs_limit["flame_perf_diagnostics"] = 3024344 -allocs_limit["flame_perf_target_diagnostic_edmfx"] = 762784 +allocs_limit["flame_perf_target_diagnostic_edmfx"] = 862576 allocs_limit["flame_perf_target_edmf"] = 12459299664 allocs_limit["flame_perf_target_threaded"] = 6175664 allocs_limit["flame_perf_target_callbacks"] = 46413904 diff --git a/src/ClimaAtmos.jl b/src/ClimaAtmos.jl index 752198d1b2..1f0c7fcac3 100644 --- a/src/ClimaAtmos.jl +++ b/src/ClimaAtmos.jl @@ -114,7 +114,6 @@ include(joinpath("diagnostics", "Diagnostics.jl")) import .Diagnostics as CAD include(joinpath("solver", "model_getters.jl")) # high-level (using parsed_args) model getters -include(joinpath("time_stepper", "time_stepper.jl")) include(joinpath("solver", "type_getters.jl")) include(joinpath("solver", "yaml_helper.jl")) include(joinpath("solver", "solve.jl")) diff --git a/src/prognostic_equations/implicit/implicit_tendency.jl b/src/prognostic_equations/implicit/implicit_tendency.jl index de6dadd30a..c7c97a68f5 100644 --- a/src/prognostic_equations/implicit/implicit_tendency.jl +++ b/src/prognostic_equations/implicit/implicit_tendency.jl @@ -7,7 +7,6 @@ import ClimaCore: Fields, Geometry NVTX.@annotate function implicit_tendency!(Yₜ, Y, p, t) fill_with_nans!(p) Yₜ .= zero(eltype(Yₜ)) - set_precomputed_quantities!(Y, p, t) Fields.bycolumn(axes(Y.c)) do colidx implicit_vertical_advection_tendency!(Yₜ, Y, p, t, colidx) if p.turbconv_model isa TurbulenceConvection.EDMFModel diff --git a/src/prognostic_equations/implicit/wfact.jl b/src/prognostic_equations/implicit/wfact.jl index 8144801093..e5d9c587c5 100644 --- a/src/prognostic_equations/implicit/wfact.jl +++ b/src/prognostic_equations/implicit/wfact.jl @@ -60,7 +60,6 @@ end NVTX.@annotate function Wfact!(W, Y, p, dtγ, t) fill_with_nans!(p) - set_precomputed_quantities!(Y, p, t) Fields.bycolumn(axes(Y.c)) do colidx Wfact!(W, Y, p, dtγ, t, colidx) end diff --git a/src/prognostic_equations/limited_tendencies.jl b/src/prognostic_equations/limited_tendencies.jl index a08da15c89..58c8e8cb27 100644 --- a/src/prognostic_equations/limited_tendencies.jl +++ b/src/prognostic_equations/limited_tendencies.jl @@ -14,7 +14,6 @@ import ClimaCore.Fields: ColumnField NVTX.@annotate function limited_tendency!(Yₜ, Y, p, t) Yₜ .= zero(eltype(Yₜ)) - set_precomputed_quantities!(Y, p, t) horizontal_tracer_advection_tendency!(Yₜ, Y, p, t) tracer_hyperdiffusion_tendency!(Yₜ, Y, p, t) return nothing diff --git a/src/solver/type_getters.jl b/src/solver/type_getters.jl index 6e21f7c74b..59f393c94a 100644 --- a/src/solver/type_getters.jl +++ b/src/solver/type_getters.jl @@ -719,7 +719,7 @@ function args_integrator(parsed_args, Y, p, tspan, ode_algo, callback) tgrad = (∂Y∂t, Y, p, t) -> (∂Y∂t .= 0), ) if is_cts_algo(ode_algo) - CA.AtmosODEFunction(; + CTS.ClimaODEFunction(; T_lim! = limited_tendency!, T_exp! = remaining_tendency!, T_imp! = implicit_func, diff --git a/src/time_stepper/imex_ark.jl b/src/time_stepper/imex_ark.jl deleted file mode 100644 index f9ad5cb6be..0000000000 --- a/src/time_stepper/imex_ark.jl +++ /dev/null @@ -1,121 +0,0 @@ -function CTS.step_u!( - integrator, - cache::CTS.IMEXARKCache, - f::AtmosODEFunction, - name, -) - (; u, p, t, dt, alg) = integrator - (; T_lim!, T_exp!, T_imp!, lim!, dss!) = f - (; tableau, newtons_method) = alg - (; a_exp, b_exp, a_imp, b_imp, c_exp, c_imp) = tableau - (; U, T_lim, T_exp, T_imp, temp, γ, newtons_method_cache) = cache - s = length(b_exp) - - if !isnothing(newtons_method) - (; update_j) = newtons_method - jacobian = newtons_method_cache.j - if (!isnothing(jacobian)) && - CTS.needs_update!(update_j, CTS.NewTimeStep(t)) - if γ isa Nothing - sdirk_error(name) - else - T_imp!.Wfact(jacobian, u, p, dt * γ, t) - end - end - end - - for i in 1:s - NVTX.@range "stage" payload = i begin - t_exp = t + dt * c_exp[i] - t_imp = t + dt * c_imp[i] - - @. U = u - - for j in 1:(i - 1) - iszero(a_exp[i, j]) && continue - @. U += dt * a_exp[i, j] * T_lim[j] - end - lim!(U, p, t_exp, u) - - for j in 1:(i - 1) - iszero(a_exp[i, j]) && continue - @. U += dt * a_exp[i, j] * T_exp[j] - end - - for j in 1:(i - 1) - iszero(a_imp[i, j]) && continue - @. U += dt * a_imp[i, j] * T_imp[j] - end - - dss!(U, p, t_exp) - - if !iszero(a_imp[i, i]) # Implicit solve - @assert !isnothing(newtons_method) - @. temp = U - # TODO: can/should we remove these closures? - implicit_equation_residual! = - (residual, Ui) -> begin - T_imp!(residual, Ui, p, t_imp) - @. residual = - temp + dt * a_imp[i, i] * residual - Ui - end - implicit_equation_jacobian! = - (jacobian, Ui) -> - T_imp!.Wfact(jacobian, Ui, p, dt * a_imp[i, i], t_imp) - - CTS.solve_newton!( - newtons_method, - newtons_method_cache, - U, - implicit_equation_residual!, - implicit_equation_jacobian!, - ) - end - - # We do not need to DSS U again because the implicit solve should - # give the same results for redundant columns (as long as the implicit - # tendency only acts in the vertical direction). - - if !all(iszero, a_imp[:, i]) || !iszero(b_imp[i]) - if iszero(a_imp[i, i]) - # If its coefficient is 0, T_imp[i] is effectively being - # treated explicitly. - T_imp!(T_imp[i], U, p, t_imp) - else - # If T_imp[i] is being treated implicitly, ensure that it - # exactly satisfies the implicit equation. - @. T_imp[i] = (U - temp) / (dt * a_imp[i, i]) - end - end - - if !all(iszero, a_exp[:, i]) || !iszero(b_exp[i]) - T_lim!(T_lim[i], U, p, t_exp) - T_exp!(T_exp[i], U, p, t_exp) - end - end - end - - t_final = t + dt - - @. temp = u - for j in 1:s - iszero(b_exp[j]) && continue - @. temp += dt * b_exp[j] * T_lim[j] - end - lim!(temp, p, t_final, u) - @. u = temp - - for j in 1:s - iszero(b_exp[j]) && continue - @. u += dt * b_exp[j] * T_exp[j] - end - - for j in 1:s - iszero(b_imp[j]) && continue - @. u += dt * b_imp[j] * T_imp[j] - end - - dss!(u, p, t_final) - - return u -end diff --git a/src/time_stepper/time_stepper.jl b/src/time_stepper/time_stepper.jl deleted file mode 100644 index 4134def718..0000000000 --- a/src/time_stepper/time_stepper.jl +++ /dev/null @@ -1,18 +0,0 @@ -import NVTX -import ClimaTimeSteppers as CTS -import Krylov - -Base.@kwdef struct AtmosODEFunction{TL, TE, TI, L, D, PE, PI} <: - CTS.AbstractClimaODEFunction - T_lim!::TL = (uₜ, u, p, t) -> nothing - T_exp!::TE = (uₜ, u, p, t) -> nothing - T_imp!::TI = (uₜ, u, p, t) -> nothing - lim!::L = (u, p, t, u_ref) -> nothing - dss!::D = (u, p, t) -> nothing - post_explicit!::PE = (u, p, t) -> nothing - post_implicit!::PI = (u, p, t) -> nothing -end - -include("imex_ark.jl") -include("imex_ssprk.jl") -include("hc_ars343.jl")