wip

.buildkite/longruns/pipeline.yml

@@ -26,13 +26,14 @@ steps:
       - echo "--- Instantiate"
       - "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=examples -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'"
+      # - "julia --project=examples -e 'using CUDA; CUDA.precompile_runtime()'"
       - "julia --project=examples -e 'using Pkg; Pkg.status()'"
       - "julia --project=examples artifacts/download_artifacts.jl"
 
     agents:
-      slurm_cpus_per_task: 8
-      slurm_gpus: 1
+      # slurm_cpus_per_task: 8
+      # slurm_gpus: 1
+      slurm_ntasks: 1
     env:
       JULIA_NUM_PRECOMPILE_TASKS: 8
 
@@ -54,40 +55,52 @@ steps:
       #   env:
       #     JOB_NAME: "longrun_zalesak_tracer_energy_bw_rhoe_equil_highres"
 
-      - label: ":computer: SSP baroclinic wave (ρe_tot) equilmoist high resolution centered diff"
-        command:
-          - "srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml"
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_ntasks: 32
-          slurm_nodes: 2
-          slurm_time: 24:00:00
-        env:
-          JOB_NAME: "longrun_ssp_bw_rhoe_equil_highres"
+      # - label: ":computer: SSP baroclinic wave (ρe_tot) equilmoist high resolution centered diff"
+      #   command:
+      #     - "srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml"
+      #   artifact_paths: "$$JOB_NAME/output_active/*"
+      #   agents:
+      #     slurm_ntasks: 32
+      #     slurm_nodes: 2
+      #     slurm_time: 24:00:00
+      #   env:
+      #     JOB_NAME: "longrun_ssp_bw_rhoe_equil_highres"
+
+      # - label: ":computer: aquaplanet equilmoist clearsky radiation + prognostic edmf diffusion only + 0M microphysics"
+      #   command:
+      #     - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
+      #   artifact_paths: "$$JOB_NAME/output_active/*"
+      #   agents:
+      #     slurm_ntasks: 64
+      #     slurm_nodes: 4
+      #     slurm_mem_per_cpu: 16GB
+      #     slurm_time: 24:00:00
+      #   env:
+      #     JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_diffonly_0M"
 
-      - label: ":computer: aquaplanet equilmoist clearsky radiation + prognostic edmf diffusion only + 0M microphysics"
+      - label: ":computer: aquaplanet equilmoist clearsky radiation + prognostic edmf + 0M microphysics"
         command:
           - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
         artifact_paths: "$$JOB_NAME/output_active/*"
         agents:
-          slurm_ntasks: 64
-          slurm_nodes: 4
+          slurm_ntasks: 128
+          slurm_nodes: 8
           slurm_mem_per_cpu: 16GB
           slurm_time: 24:00:00
         env:
-          JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_diffonly_0M"
+          JOB_NAME: "longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M"
 
-  - group: "Low resolution long runs"
+  # - group: "Low resolution long runs"
 
-    steps:
+  #   steps:
 
-      - label: ":computer: low resolution aquaplanet equilmoist clearsky radiation + time-varying insolation + slab ocean"
-        command:
-          - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
-        artifact_paths: "$$JOB_NAME/output_active/*"
-        agents:
-          slurm_ntasks: 8
-          slurm_mem_per_cpu: 16GB
-          slurm_time: 24:00:00
-        env:
-          JOB_NAME: "longrun_aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"
+  #     - label: ":computer: low resolution aquaplanet equilmoist clearsky radiation + time-varying insolation + slab ocean"
+  #       command:
+  #         - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml
+  #       artifact_paths: "$$JOB_NAME/output_active/*"
+  #       agents:
+  #         slurm_ntasks: 8
+  #         slurm_mem_per_cpu: 16GB
+  #         slurm_time: 24:00:00
+  #       env:
+  #         JOB_NAME: "longrun_aquaplanet_rhoe_equil_clearsky_tvinsol_0M_slabocean"

config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M.yml

@@ -0,0 +1,24 @@
+h_elem: 12
+z_max: 55000.0
+z_elem: 30
+dz_bottom: 100.0
+dz_top: 3000.0
+moist: "equil"
+precip_model: "0M"
+override_τ_precip: false
+rad: "clearsky"
+dt_rad: "6hours"
+surface_setup: "DefaultMoninObukhov"
+vert_diff: "FriersonDiffusion"
+rayleigh_sponge: true
+dt_save_state_to_disk: "10days"
+dt: "200secs"
+t_end: "100days"
+job_id: "longrun_aquaplanet_equil_helem12_clearsky_0M"
+toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_0M.toml]
+netcdf_output_at_levels: true
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
+    period: 10days
+  - short_name: [lmix, bgrad, strain, edt, evu]
+    period: 10days

config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.yml

@@ -0,0 +1,25 @@
+h_elem: 12
+z_max: 55000.0
+z_elem: 30
+dz_bottom: 100.0
+dz_top: 3000.0
+moist: "equil"
+precip_model: "0M"
+override_τ_precip: false
+rad: "clearsky"
+dt_rad: "6hours"
+surface_setup: "DefaultMoninObukhov"
+vert_diff: "FriersonDiffusion"
+rayleigh_sponge: true
+viscous_sponge: true
+dt_save_state_to_disk: "10days"
+dt: "200secs"
+t_end: "100days"
+job_id: "longrun_aquaplanet_equil_helem12_clearsky_0M_viscous"
+toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_0M_viscous.toml]
+netcdf_output_at_levels: true
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
+    period: 10days
+  - short_name: [lmix, bgrad, strain, edt, evu]
+    period: 10days

config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.yml

@@ -0,0 +1,40 @@
+h_elem: 12
+z_max: 55000.0
+z_elem: 30
+dz_bottom: 100.0
+dz_top: 3000.0
+moist: "equil"
+precip_model: "0M"
+override_τ_precip: false
+rad: "clearsky"
+dt_rad: "6hours"
+surface_setup: "DefaultMoninObukhov"
+turbconv: "prognostic_edmfx"
+implicit_sgs_advection: true
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+max_newton_iters_ode: 3
+prognostic_tke: true
+edmfx_upwinding: "first_order"
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
+edmfx_nh_pressure: true
+edmfx_velocity_relaxation: true
+edmfx_sgs_mass_flux: true
+edmfx_sgs_diffusive_flux: true
+rayleigh_sponge: true
+viscous_sponge: true
+restart_file: "/central/groups/esm/zhaoyi/climaatmos/longrun_1514/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M/output_active/day20.0.hdf5"
+dt_save_state_to_disk: "3hours"
+dt: "40secs"
+t_end: "100days"
+job_id: "longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M"
+toml: [toml/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M.toml]
+netcdf_output_at_levels: true
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
+    period: 3hours
+  - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke]
+    period: 3hours
+  - short_name: [entr, detr, lmix, bgrad, strain, edt, evu]
+    period: 3hours

config/longrun_configs/longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M_asym.yml

@@ -0,0 +1,39 @@
+h_elem: 12
+z_max: 55000.0
+z_elem: 30
+dz_bottom: 100.0
+dz_top: 3000.0
+moist: "equil"
+precip_model: "0M"
+override_τ_precip: false
+surface_temperature: "ZonallyAsymmetric"
+rad: "clearsky"
+dt_rad: "6hours"
+surface_setup: "DefaultMoninObukhov"
+turbconv: "prognostic_edmfx"
+implicit_sgs_advection: true
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+max_newton_iters_ode: 3
+prognostic_tke: true
+edmfx_upwinding: "first_order"
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
+edmfx_nh_pressure: true
+edmfx_velocity_relaxation: true
+edmfx_sgs_mass_flux: true
+edmfx_sgs_diffusive_flux: true
+rayleigh_sponge: true
+dt_save_state_to_disk: "10days"
+dt: "40secs"
+t_end: "100days"
+job_id: "longrun_aquaplanet_equil_helem12_clearsky_progedmf_0M_asym"
+toml: [toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml]
+netcdf_output_at_levels: true
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
+    period: 10days
+  - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke]
+    period: 10days
+  - short_name: [entr, detr, lmix, bgrad, strain, edt, evu]
+    period: 10days

config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.yml

@@ -0,0 +1,38 @@
+h_elem: 16
+z_max: 55000.0
+z_elem: 63
+dz_bottom: 30.0
+dz_top: 3000.0
+moist: "equil"
+precip_model: "0M"
+override_τ_precip: false
+rad: "clearsky"
+dt_rad: "6hours"
+surface_setup: "DefaultMoninObukhov"
+turbconv: "prognostic_edmfx"
+implicit_sgs_advection: true
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+max_newton_iters_ode: 3
+prognostic_tke: true
+edmfx_upwinding: "first_order"
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
+edmfx_nh_pressure: true
+edmfx_velocity_relaxation: true
+edmfx_sgs_mass_flux: true
+edmfx_sgs_diffusive_flux: true
+rayleigh_sponge: true
+dt_save_state_to_disk: "3hours"
+dt: "40secs"
+t_end: "10days"
+job_id: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M"
+toml: [toml/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_progedmf_0M.toml]
+netcdf_output_at_levels: true
+diagnostics:
+  - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]
+    period: 1days
+  - short_name: [arup, waup, taup, thetaaup, haup, husup, hurup, clwup, cliup, waen, taen, thetaaen, haen, husen, huren, clwen, clien, tke]
+    period: 1days
+  - short_name: [entr, detr, lmix, bgrad, strain, edt, evu]
+    period: 1days

post_processing/ci_plots.jl

@@ -1042,26 +1042,26 @@ function make_plots(
 
     short_names = [
         "ua",
-        "wa",
+        "va",
+        "tke",
+        "lmix",
+        "bgrad",
+        "strain",
         "thetaa",
         "thetaaup",
         "ta",
         "taup",
         "ha",
         "haup",
+        "wa",
         "waup",
-        "tke",
-        "lmix",
         "arup",
         "hus",
         "husup",
         "hur",
         "hurup",
-        "cl",
         "clw",
         "clwup",
-        "cli",
-        "cliup",
         precip_names...,
     ]
     reduction = "inst"
@@ -1083,6 +1083,13 @@ function make_plots(
         ([slice(v, time = LAST_SNAP) for v in group]...,) for
         group in var_groups_zt
     ]
+    # var_groups_z = [
+    #     (
+    #         [
+    #             ClimaAnalysis.window(v, "time", left = 14400, right = 21600) |> ClimaAnalysis.average_time for v in group
+    #         ]...,
+    #     ) for group in var_groups_zt
+    # ]
 
     tmp_file = make_plots_generic(
         output_paths,

src/cache/precomputed_quantities.jl

@@ -38,7 +38,7 @@ function precomputed_quantities(Y, atmos)
             !(atmos.turbconv_model isa DiagnosticEDMFX)
     @assert !(atmos.moisture_model isa DryModel) ||
             !(atmos.turbconv_model isa PrognosticEDMFX)
-    @assert isnothing(atmos.turbconv_model) || isnothing(atmos.vert_diff)
+    # @assert isnothing(atmos.turbconv_model) || isnothing(atmos.vert_diff)
     TST = thermo_state_type(atmos.moisture_model, FT)
     SCT = SurfaceConditions.surface_conditions_type(atmos, FT)
     cspace = axes(Y.c)
@@ -53,6 +53,8 @@ function precomputed_quantities(Y, atmos)
         ᶜp = similar(Y.c, FT),
         ᶜh_tot = similar(Y.c, FT),
         ᶜmixing_length = similar(Y.c, FT),
+        ᶜlinear_buoygrad = similar(Y.c, FT),
+        ᶜstrain_rate_norm = similar(Y.c, FT),
         sfc_conditions = Fields.Field(SCT, Spaces.level(axes(Y.f), half)),
     )
     cloud_diagnostics = (; ᶜcloud_fraction = similar(Y.c, FT),)
@@ -83,8 +85,6 @@ function precomputed_quantities(Y, atmos)
             ᶜq_tot⁰ = similar(Y.c, FT),
             ᶜts⁰ = similar(Y.c, TST),
             ᶜρ⁰ = similar(Y.c, FT),
-            ᶜlinear_buoygrad = similar(Y.c, FT),
-            ᶜstrain_rate_norm = similar(Y.c, FT),
             ᶜK_u = similar(Y.c, FT),
             ᶜK_h = similar(Y.c, FT),
             ρatke_flux = similar(Fields.level(Y.f, half), C3{FT}),

src/cache/prognostic_edmf_precomputed_quantities.jl

@@ -256,7 +256,7 @@ NVTX.@annotate function set_prognostic_edmf_precomputed_quantities_closures!(
             ᶜentrʲs.:($$j),
             ᶜvert_div,
             ᶜmassflux_vert_div,
-            ᶜtke⁰,
+            max(ᶜtke⁰, 0),
             p.atmos.edmfx_detr_model,
         )
         @. ᶜdetrʲs.:($$j) = limit_detrainment(
@@ -324,7 +324,7 @@ NVTX.@annotate function set_prognostic_edmf_precomputed_quantities_closures!(
 
     turbconv_params = CAP.turbconv_params(params)
     c_m = CAP.tke_ed_coeff(turbconv_params)
-    @. ᶜK_u = c_m * ᶜmixing_length * sqrt(max(ᶜtke⁰, 0))
+    @. ᶜK_u = max(c_m * ᶜmixing_length * sqrt(max(ᶜtke⁰, 0)), 0)
     @. ᶜK_h = ᶜK_u / ᶜprandtl_nvec
 
     ρatke_flux_values = Fields.field_values(ρatke_flux)

src/diagnostics/core_diagnostics.jl

@@ -246,6 +246,38 @@ add_diagnostic_variable!(
     end,
 )
 
+###
+# Buoyancy gradient (3d)
+###
+add_diagnostic_variable!(
+    short_name = "bgrad",
+    long_name = "Linearized Buoyancy Gradient",
+    units = "s^-2",
+    compute! = (out, state, cache, time) -> begin
+        if isnothing(out)
+            return copy(cache.precomputed.ᶜlinear_buoygrad)
+        else
+            out .= cache.precomputed.ᶜlinear_buoygrad
+        end
+    end,
+)
+
+###
+# Strain rate magnitude (3d)
+###
+add_diagnostic_variable!(
+    short_name = "strain",
+    long_name = "String Rate Magnitude",
+    units = "s^-2",
+    compute! = (out, state, cache, time) -> begin
+        if isnothing(out)
+            return copy(cache.precomputed.ᶜstrain_rate_norm)
+        else
+            out .= cache.precomputed.ᶜstrain_rate_norm
+        end
+    end,
+)
+
 ###
 # Relative humidity (3d)
 ###