Merge branch 'main' into sk/try_dss_cov123

.buildkite/longruns/pipeline.yml

@@ -132,9 +132,9 @@ steps:
       - label: ":computer: aquaplanet (ρe_tot) equilmoist clearsky high resolution hightop rayleigh sponge(35e3, 10) Float64"
         command:
           - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
-          - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl --data_dir JOB_NAME --out_dir JOB_NAME
-          - julia --color=yes --project=examples post_processing/plot/plot_pipeline.jl --nc_dir JOB_NAME --fig_dir JOB_NAME --case_name aquaplanet
-        artifact_paths: "JOB_NAME/*"
+          - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl --data_dir $$JOB_NAME --out_dir $$JOB_NAME
+          - julia --color=yes --project=examples post_processing/plot/plot_pipeline.jl --nc_dir $$JOB_NAME --fig_dir $$JOB_NAME --case_name aquaplanet
+        artifact_paths: "$$JOB_NAME/*"
         agents:
           slurm_ntasks: 64
           slurm_time: 24:00:00
@@ -222,3 +222,35 @@ steps:
           slurm_time: 24:00:00
         env: 
           JOB_NAME: "longrun_compressible_edmf_trmm"
+
+  - group: "DYAMOND"
+
+    steps:
+
+      - label: ":computer: aquaplanet dyamond"
+        command:
+          - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
+          - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl --data_dir $$JOB_NAME --out_dir $$JOB_NAME
+        artifact_paths: "$$JOB_NAME/*"
+        agents:
+          slurm_ntasks: 64
+          slurm_mem_per_cpu: 32GB
+          slurm_time: 24:00:00
+        env:
+          JOB_NAME: "longrun_aquaplanet_dyamond"
+
+  - group: "AMIP"
+
+    steps:
+
+      - label: ":computer: aquaplanet amip"
+        command:
+          - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
+          - julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl --data_dir $$JOB_NAME --out_dir $$JOB_NAME
+        artifact_paths: "$$JOB_NAME/*"
+        agents:
+          slurm_ntasks: 64
+          slurm_mem_per_cpu: 16GB
+          slurm_time: 24:00:00
+        env:
+          JOB_NAME: "longrun_aquaplanet_amip"

.buildkite/pipeline.yml

@@ -632,6 +632,7 @@ steps:
           julia --color=yes --project=examples examples/hybrid/driver.jl
           --config_file $CONFIG_PATH/diagnostic_edmfx_aquaplanet_tke.yml
         artifact_paths: "diagnostic_edmfx_aquaplanet_tke/*"
+        soft_fail: true
         agents:
           slurm_mem: 20GB
 

.buildkite/scaling/pipeline.sh

@@ -67,7 +67,7 @@ done
 # set up environment and agents
 cat << EOM
 env:
-  JULIA_VERSION: "1.8.5"
+  JULIA_VERSION: "1.9.3"
   MPICH_VERSION: "4.0.0"
   OPENMPI_VERSION: "4.1.1"
   MPI_IMPL: "$mpi_impl"

config/default_configs/default_edmf_config.yml

@@ -35,6 +35,9 @@ debugging_tc:
 entr_coeff:
   help: "Entrainment coefficient"
   value: 1.0
+entr_tau:
+  help: "Entrainment timescale"
+  value: 900.0
 detr_coeff:
   help: "Detrainment coefficient"
   value: 0.001

config/longrun_configs/longrun_aquaplanet_amip.yml

@@ -0,0 +1,25 @@
+h_elem: 16
+z_max: 55000.0
+z_elem: 63
+dz_bottom: 30.0
+dz_top: 3000.0
+kappa_4: 1e16
+moist: "equil" 
+precip_model: "0M" 
+rad: "allskywithclear" 
+idealized_insolation: false
+dt_rad: "1hours"
+surface_setup: "DefaultMoninObukhov" 
+turbconv: "diagnostic_edmfx" 
+prognostic_tke: true
+edmfx_upwinding: "first_order" 
+edmfx_entr_detr: true 
+edmfx_nh_pressure: true 
+edmfx_sgs_flux: true
+rayleigh_sponge: true
+dt_save_to_disk: "3hours"
+dt: "10secs" 
+t_end: "1days"
+FLOAT_TYPE: "Float64"
+job_id: "longrun_aquaplanet_amip" 
+toml: [toml/longrun_aquaplanet_amip.toml]

config/longrun_configs/longrun_aquaplanet_dyamond.yml

@@ -0,0 +1,20 @@
+h_elem: 60
+z_max: 55000.0
+z_elem: 63
+dz_bottom: 30.0
+dz_top: 3000.0
+kappa_4: 1e14
+moist: "equil" 
+precip_model: "0M" 
+rad: "allskywithclear" 
+idealized_insolation: false
+dt_rad: "1hours"
+vert_diff: "true" 
+surface_setup: "DefaultMoninObukhov" 
+rayleigh_sponge: true
+dt_save_to_disk: "3hours"
+dt: "50secs" 
+t_end: "1days"
+FLOAT_TYPE: "Float64"
+job_id: "longrun_aquaplanet_dyamond" 
+toml: [toml/longrun_aquaplanet_dyamond.toml]

config/longrun_configs/longrun_aquaplanet_rhoe_equil_gray_55km_nz63_0M.yml

@@ -5,6 +5,8 @@ h_elem: 16
 z_max: 55000.0
 rayleigh_sponge: true
 z_elem: 63
+dz_bottom: 30.0
+dz_top: 3000.0
 vert_diff: "true" 
 kappa_4: 2e16
 surface_setup: "DefaultExchangeCoefficients" 

perf/flame.jl

@@ -58,7 +58,7 @@ allocs_limit["flame_perf_target"] = 4384
 allocs_limit["flame_perf_target_tracers"] = 204016
 allocs_limit["flame_perf_target_edmfx"] = 304064
 allocs_limit["flame_perf_target_diagnostic_edmfx"] = 685456
-allocs_limit["flame_perf_target_edmf"] = 9015243600
+allocs_limit["flame_perf_target_edmf"] = 12459299664
 allocs_limit["flame_perf_target_threaded"] = 6175664
 allocs_limit["flame_perf_target_callbacks"] = 49850536
 allocs_limit["flame_perf_gw"] = 4985829472

regression_tests/ref_counter.jl

@@ -1 +1 @@
-123
+124

src/TurbulenceConvection_deprecated/dycore_api.jl

@@ -7,8 +7,8 @@ struct CentField <: FieldLocation end
 struct FaceField <: FieldLocation end
 struct SingleValuePerColumn <: FieldLocation end
 
-field_loc(::CentField) = :cent
-field_loc(::FaceField) = :face
+field_loc(::CentField) = :c
+field_loc(::FaceField) = :f
 field_loc(::SingleValuePerColumn) = :svpc
 
 #=

src/TurbulenceConvection_deprecated/types.jl

@@ -206,27 +206,28 @@ struct State{P, A, T, CACHE, C, SC}
     surface_conditions::SC
 end
 
-Grid(state::State) = Grid(axes(state.prog.cent))
+Grid(state::State) = Grid(axes(state.prog.c))
 
 float_type(state::State) = eltype(state.prog)
 # float_type(field::CC.Fields.Field) = CC.Spaces.undertype(axes(field))
 float_type(field::CC.Fields.Field) = eltype(parent(field))
 
+import ClimaCore.Fields as Fields
+import ClimaCore.Spaces as Spaces
+
+
+Base.@propagate_inbounds function field_vector_column(
+    fv::Fields.FieldVector{T},
+    colidx::Fields.ColumnIndex,
+) where {T}
+    values = map(x -> x[colidx], Fields._values(fv))
+    return Fields.FieldVector{T, typeof(values)}(values)
+end
+
 function tc_column_state(prog, p, tendencies, colidx, t)
-    prog_cent_column = CC.column(prog.c, colidx)
-    prog_face_column = CC.column(prog.f, colidx)
-    aux_cent_column = CC.column(p.edmf_cache.aux.cent, colidx)
-    aux_face_column = CC.column(p.edmf_cache.aux.face, colidx)
-    tends_cent_column = CC.column(tendencies.c, colidx)
-    tends_face_column = CC.column(tendencies.f, colidx)
-    prog_column =
-        CC.Fields.FieldVector(cent = prog_cent_column, face = prog_face_column)
-    aux_column =
-        CC.Fields.FieldVector(cent = aux_cent_column, face = aux_face_column)
-    tends_column = CC.Fields.FieldVector(
-        cent = tends_cent_column,
-        face = tends_face_column,
-    )
+    prog_column = field_vector_column(prog, colidx)
+    aux_column = field_vector_column(p.edmf_cache.aux, colidx)
+    tends_column = field_vector_column(tendencies, colidx)
     surface_conditions = CC.column(p.sfc_conditions, colidx)[]
     return State(
         prog_column,
@@ -239,14 +240,8 @@ function tc_column_state(prog, p, tendencies, colidx, t)
 end
 
 function tc_column_state(prog, p, tendencies::Nothing, colidx, t)
-    prog_cent_column = CC.column(prog.c, colidx)
-    prog_face_column = CC.column(prog.f, colidx)
-    aux_cent_column = CC.column(p.edmf_cache.aux.cent, colidx)
-    aux_face_column = CC.column(p.edmf_cache.aux.face, colidx)
-    prog_column =
-        CC.Fields.FieldVector(cent = prog_cent_column, face = prog_face_column)
-    aux_column =
-        CC.Fields.FieldVector(cent = aux_cent_column, face = aux_face_column)
+    prog_column = field_vector_column(prog, colidx)
+    aux_column = field_vector_column(p.edmf_cache.aux, colidx)
     tends_column = nothing
     surface_conditions = CC.column(p.sfc_conditions, colidx)[]
     return State(

src/cache/cache.jl

@@ -132,19 +132,12 @@ function additional_cache(
 )
     (; precip_model, forcing_type, radiation_mode, turbconv_model) = atmos
 
-    idealized_insolation = parsed_args["idealized_insolation"]
-    @assert idealized_insolation in (true, false)
-    idealized_clouds = parsed_args["idealized_clouds"]
-    @assert idealized_clouds in (true, false)
-
     radiation_cache = if radiation_mode isa RRTMGPI.AbstractRRTMGPMode
         radiation_model_cache(
             Y,
             default_cache,
             params,
             radiation_mode;
-            idealized_insolation,
-            idealized_clouds,
             data_loader = rrtmgp_data_loader,
         )
     else

src/cache/diagnostic_edmf_precomputed_quantities.jl

@@ -205,7 +205,7 @@ function set_diagnostic_edmf_precomputed_quantities!(Y, p, t)
         ρʲ_int_level = Fields.field_values(Fields.level(ᶜρʲ, 1))
 
         @. u³ʲ_int_halflevel = CT3(
-            Geometry.WVector($(FT(0.01)), local_geometry_int_halflevel),
+            Geometry.WVector($(FT(0)), local_geometry_int_halflevel),
             local_geometry_int_halflevel,
         )
         @. h_totʲ_int_level = sgs_scalar_first_interior_bc(

src/cache/precomputed_quantities.jl

@@ -275,7 +275,7 @@ Note: If you need to use any of the precomputed quantities, please call this
 function instead of recomputing the value yourself. Otherwise, it will be
 difficult to ensure that the duplicated computations are consistent.
 """
-function set_precomputed_quantities!(Y, p, t)
+NVTX.@annotate function set_precomputed_quantities!(Y, p, t)
     (; energy_form, moisture_model, turbconv_model) = p.atmos
     thermo_params = CAP.thermodynamics_params(p.params)
     n = n_mass_flux_subdomains(turbconv_model)

src/cache/temporary_quantities.jl

@@ -49,7 +49,6 @@ function temporary_quantities(atmos, center_space, face_space)
         ᶠtemp_CT3 = Fields.Field(CT3{FT}, face_space), # ᶠuₕ³
         ᶠtemp_CT12 = Fields.Field(CT12{FT}, face_space), # ᶠω¹²
         ᶠtemp_CT12ʲs = Fields.Field(NTuple{n, CT12{FT}}, face_space), # ᶠω¹²ʲs
-        ᶜtemp_C123 = Fields.Field(C123{FT}, center_space), # χ₁₂₃
         ᶠtemp_C123 = Fields.Field(C123{FT}, face_space), # χ₁₂₃
         ᶜtemp_UVWxUVW = Fields.Field(
             typeof(UVW(FT(0), FT(0), FT(0)) * UVW(FT(0), FT(0), FT(0))'),

src/callbacks/callbacks.jl

@@ -16,18 +16,16 @@ import ClimaCore.Fields: ColumnField
 
 include("callback_helpers.jl")
 
-function dss_callback!(integrator)
+NVTX.@annotate function dss_callback!(integrator)
     Y = integrator.u
     ghost_buffer = integrator.p.ghost_buffer
     if integrator.p.do_dss
-        NVTX.@range "dss callback" color = colorant"yellow" begin
-            Spaces.weighted_dss_start2!(Y.c, ghost_buffer.c)
-            Spaces.weighted_dss_start2!(Y.f, ghost_buffer.f)
-            Spaces.weighted_dss_internal2!(Y.c, ghost_buffer.c)
-            Spaces.weighted_dss_internal2!(Y.f, ghost_buffer.f)
-            Spaces.weighted_dss_ghost2!(Y.c, ghost_buffer.c)
-            Spaces.weighted_dss_ghost2!(Y.f, ghost_buffer.f)
-        end
+        Spaces.weighted_dss_start2!(Y.c, ghost_buffer.c)
+        Spaces.weighted_dss_start2!(Y.f, ghost_buffer.f)
+        Spaces.weighted_dss_internal2!(Y.c, ghost_buffer.c)
+        Spaces.weighted_dss_internal2!(Y.f, ghost_buffer.f)
+        Spaces.weighted_dss_ghost2!(Y.c, ghost_buffer.c)
+        Spaces.weighted_dss_ghost2!(Y.f, ghost_buffer.f)
     end
     return nothing
 end
@@ -51,7 +49,7 @@ function flux_accumulation!(integrator)
     return nothing
 end
 
-function turb_conv_affect_filter!(integrator)
+NVTX.@annotate function turb_conv_affect_filter!(integrator)
     p = integrator.p
     (; edmf_cache) = p
     (; edmf, param_set) = edmf_cache
@@ -73,7 +71,7 @@ function turb_conv_affect_filter!(integrator)
     SciMLBase.u_modified!(integrator, false)
 end
 
-function rrtmgp_model_callback!(integrator)
+NVTX.@annotate function rrtmgp_model_callback!(integrator)
     Y = integrator.u
     p = integrator.p
     t = integrator.t
@@ -201,16 +199,15 @@ function rrtmgp_model_callback!(integrator)
             1000 * Y.c.ρ * TD.liquid_specific_humidity(thermo_params, ᶜts) * ᶜΔz
         @. ᶜiwp =
             1000 * Y.c.ρ * TD.ice_specific_humidity(thermo_params, ᶜts) * ᶜΔz
-        @. ᶜfrac =
-            get_cloud_fraction(thermo_params, env_thermo_quad, FT(ᶜp), ᶜts)
+        @. ᶜfrac = ifelse(TD.has_condensate(thermo_params, ᶜts), 1, 0 * ᶜΔz)
     end
 
     RRTMGPI.update_fluxes!(radiation_model)
     RRTMGPI.field2array(ᶠradiation_flux) .= radiation_model.face_flux
     return nothing
 end
 
-function compute_diagnostics(integrator)
+NVTX.@annotate function compute_diagnostics(integrator)
     (; t, u, p) = integrator
     Y = u
     (; params, env_thermo_quad) = p
@@ -352,8 +349,8 @@ function compute_diagnostics(integrator)
             ) .+ ᶜa⁺ .* cloud_fraction.(ᶜts⁺, ᶜa⁺),
         )
     elseif p.atmos.turbconv_model isa TC.EDMFModel
-        tc_cent(p) = p.edmf_cache.aux.cent.turbconv
-        tc_face(p) = p.edmf_cache.aux.face.turbconv
+        tc_cent(p) = p.edmf_cache.aux.c.turbconv
+        tc_face(p) = p.edmf_cache.aux.f.turbconv
         turbulence_convection_diagnostic = (;
             bulk_up_area = tc_cent(p).bulk.area,
             bulk_up_h_tot = tc_cent(p).bulk.h_tot,

src/dycore_equations_deprecated/sgs_flux_tendencies.jl

@@ -124,7 +124,7 @@ function turbconv_aux(atmos, edmf, Y, ::Type{FT}) where {FT}
 
     aux_cent_fields = cent_aux_vars.(FT, ᶜlocal_geometry, atmos, edmf)
     aux_face_fields = face_aux_vars.(FT, ᶠlocal_geometry, atmos, edmf)
-    aux = Fields.FieldVector(cent = aux_cent_fields, face = aux_face_fields)
+    aux = Fields.FieldVector(; c = aux_cent_fields, f = aux_face_fields)
     return aux
 end
 
@@ -174,8 +174,8 @@ function implicit_sgs_flux_tendency!(Yₜ, Y, p, t, colidx, ::TC.EDMFModel)
 
     grid = TC.Grid(state)
     if test_consistency
-        parent(state.aux.face) .= NaN
-        parent(state.aux.cent) .= NaN
+        parent(state.aux.f) .= NaN
+        parent(state.aux.c) .= NaN
     end
 
     assign_thermo_aux!(state, edmf.moisture_model, thermo_params)
@@ -213,8 +213,8 @@ function explicit_sgs_flux_tendency!(Yₜ, Y, p, t, colidx, ::TC.EDMFModel)
 
     grid = TC.Grid(state)
     if test_consistency
-        parent(state.aux.face) .= NaN
-        parent(state.aux.cent) .= NaN
+        parent(state.aux.f) .= NaN
+        parent(state.aux.c) .= NaN
     end
 
     assign_thermo_aux!(state, edmf.moisture_model, thermo_params)
@@ -237,14 +237,9 @@ function explicit_sgs_flux_tendency!(Yₜ, Y, p, t, colidx, ::TC.EDMFModel)
     # Note: This "filter relaxation tendency" can be scaled down if needed, but
     # it must be present in order to prevent Y and Y_filtered from diverging
     # during each timestep.
-    Yₜ_turbconv =
-        Fields.FieldVector(c = Yₜ.c.turbconv[colidx], f = Yₜ.f.turbconv[colidx])
-    Y_filtered_turbconv = Fields.FieldVector(
-        c = Y_filtered.c.turbconv[colidx],
-        f = Y_filtered.f.turbconv[colidx],
-    )
-    Y_turbconv =
-        Fields.FieldVector(c = Y.c.turbconv[colidx], f = Y.f.turbconv[colidx])
+    Yₜ_turbconv = TC.field_vector_column(Yₜ, colidx)
+    Y_filtered_turbconv = TC.field_vector_column(Y_filtered, colidx)
+    Y_turbconv = TC.field_vector_column(Y, colidx)
     Yₜ_turbconv .+= (Y_filtered_turbconv .- Y_turbconv) ./ Δt
     return nothing
 end