add docs

rev

rebase fixes

format

README.md

@@ -24,3 +24,13 @@ Provides coupled system time stepping control and support for mapping import and
 boundary information between components.
 
 Recommended Julia Version: Stable release v1.10.1. CI no longer tests earlier versions of Julia.
+
+## Start Up
+Before starting Julia, ensure your environment is properly set up. This includes loading the correct Julia version (specified in the `Project.toml` file), modules and setting the correct environment variables.
+
+Typically (e.g., when running on your local machine) the installation of the required module libraries and specification of environment variables should be automatically handled by the Julia package manager upon running `]instantiate` and `]build` in the Julia REPL,
+which reads the `Project.toml` file. Each experiment has its own `Project.toml` file and `Manifest.toml` file, which are used to specify the exact environment for the experiment. Once instantiated, the experiment run scripts should be able to run without any further environment setup (e.g., with `julia --project --threads 8 experiments/<your_experiment>.jl`).
+
+For computationally intensive jobs, it is recommended to run the jobs on a cluster using MPI and/or GPU triggered by a SLURM job script.
+In this case, we need to load specific MPI and GPU compatible modules and set the correct environment variables. For an example of a SLURM job script, see `test/mpi_tests/local_checks.sh`.
+

experiments/AMIP/components/atmosphere/climaatmos_extra_diags.jl

@@ -24,7 +24,7 @@ add_diagnostic_variable!(
         (; ᶜts) = cache.precomputed
         c_space = axes(state.c)
         thermo_params = CAP.thermodynamics_params(params)
-        e_pot = CAP.grav(params) .* Fields.coordinate_field(c_space).z
+        e_pot = CAP.grav(params) .* ClimaCore.Fields.coordinate_field(c_space).z
         if isnothing(out)
             return TD.moist_static_energy.(thermo_params, ᶜts, e_pot)
         else
@@ -62,11 +62,11 @@ add_diagnostic_variable!(
     compute! = (out, state, cache, time) -> begin
         (; ᶜp) = cache.precomputed
         (; C_E) = cache.atmos.vert_diff
-        interior_uₕ = Fields.level(state.c.uₕ, 1)
+        interior_uₕ = ClimaCore.Fields.level(state.c.uₕ, 1)
         ᶠp = ᶠK_E = cache.scratch.ᶠtemp_scalar
-        Fields.bycolumn(axes(ᶜp)) do colidx
+        ClimaCore.Fields.bycolumn(axes(ᶜp)) do colidx
             @. ᶠp[colidx] = CAD.ᶠinterp(ᶜp[colidx])
-            ᶜΔz_surface = Fields.Δz_field(interior_uₕ)
+            ᶜΔz_surface = ClimaCore.Fields.Δz_field(interior_uₕ)
             @. ᶠK_E[colidx] = CA.eddy_diffusivity_coefficient(
                 C_E,
                 CA.norm(interior_uₕ[colidx]),

experiments/AMIP/components/atmosphere/climaatmos_init.jl

@@ -1,18 +1,24 @@
 # atmos_init: for ClimaAtmos pre-AMIP interface
+using StaticArrays
+using Statistics: mean
+
 import ClimaAtmos as CA
 import ClimaAtmos: CT1, CT2, CT12, CT3, C3, C12, unit_basis_vector_data, ⊗
 import SurfaceFluxes as SF
+using ClimaCore
+using ClimaCore.Utilities: half
 
+import ClimaCoupler.Interfacer: AtmosModelSimulation
 import ClimaCoupler.FluxCalculator:
     atmos_turbulent_fluxes!,
     calculate_surface_air_density,
     PartitionedStateFluxes,
     extrapolate_ρ_to_sfc,
     get_surface_params
-using ClimaCore: Fields.level, Geometry
 import ClimaCoupler.Interfacer: get_field, update_field!, name
 import ClimaCoupler.Checkpointer: get_model_prog_state
-using StaticArrays
+import ClimaCoupler.FieldExchanger: update_sim!, step!, reinit!
+import ClimaCoupler.Utilities: swap_space!
 
 include("climaatmos_extra_diags.jl")
 
@@ -87,9 +93,9 @@ function atmos_init(::Type{FT}, atmos_config_dict::Dict) where {FT}
     ᶜ3d_snow = integrator.p.precipitation.ᶜ3d_snow
 
     # set initial fluxes to zero
-    @. ρ_flux_h_tot = Geometry.Covariant3Vector(FT(0.0))
-    @. ρ_flux_q_tot = Geometry.Covariant3Vector(FT(0.0))
-    @. ᶠradiation_flux = Geometry.WVector(FT(0))
+    @. ρ_flux_h_tot = ClimaCore.Geometry.Covariant3Vector(FT(0.0))
+    @. ρ_flux_q_tot = ClimaCore.Geometry.Covariant3Vector(FT(0.0))
+    @. ᶠradiation_flux = ClimaCore.Geometry.WVector(FT(0))
     ρ_flux_uₕ.components .= Ref(SMatrix{1, 2}([FT(0), FT(0)]))
     col_integrated_rain .= FT(0)
     col_integrated_snow .= FT(0)
@@ -106,22 +112,23 @@ end
 
 # extensions required by the Interfacer
 get_field(sim::ClimaAtmosSimulation, ::Val{:radiative_energy_flux_sfc}) =
-    Fields.level(sim.integrator.p.radiation.ᶠradiation_flux, half)
+    ClimaCore.Fields.level(sim.integrator.p.radiation.ᶠradiation_flux, half)
 get_field(sim::ClimaAtmosSimulation, ::Val{:liquid_precipitation}) = sim.integrator.p.precipitation.col_integrated_rain # kg/m^2/s
 get_field(sim::ClimaAtmosSimulation, ::Val{:snow_precipitation}) = sim.integrator.p.precipitation.col_integrated_snow  # kg/m^2/s
 get_field(sim::ClimaAtmosSimulation, ::Val{:turbulent_energy_flux}) =
-    Geometry.WVector.(sim.integrator.p.precomputed.sfc_conditions.ρ_flux_h_tot)
+    ClimaCore.Geometry.WVector.(sim.integrator.p.precomputed.sfc_conditions.ρ_flux_h_tot)
 get_field(sim::ClimaAtmosSimulation, ::Val{:turbulent_moisture_flux}) =
-    Geometry.WVector.(sim.integrator.p.precomputed.sfc_conditions.ρ_flux_q_tot)
-get_field(sim::ClimaAtmosSimulation, ::Val{:thermo_state_int}) = Spaces.level(sim.integrator.p.precomputed.ᶜts, 1)
+    ClimaCore.Geometry.WVector.(sim.integrator.p.precomputed.sfc_conditions.ρ_flux_q_tot)
+get_field(sim::ClimaAtmosSimulation, ::Val{:thermo_state_int}) =
+    ClimaCore.Spaces.level(sim.integrator.p.precomputed.ᶜts, 1)
 
 # extensions required by FluxCalculator (partitioned fluxes)
 get_field(sim::ClimaAtmosSimulation, ::Val{:height_int}) =
-    Spaces.level(Fields.coordinate_field(sim.integrator.u.c).z, 1)
+    ClimaCore.Spaces.level(ClimaCore.Fields.coordinate_field(sim.integrator.u.c).z, 1)
 get_field(sim::ClimaAtmosSimulation, ::Val{:height_sfc}) =
-    Spaces.level(Fields.coordinate_field(sim.integrator.u.f).z, half)
+    ClimaCore.Spaces.level(ClimaCore.Fields.coordinate_field(sim.integrator.u.f).z, half)
 function get_field(sim::ClimaAtmosSimulation, ::Val{:uv_int})
-    uₕ_int = Geometry.UVVector.(Spaces.level(sim.integrator.u.c.uₕ, 1))
+    uₕ_int = ClimaCore.Geometry.UVVector.(ClimaCore.Spaces.level(sim.integrator.u.c.uₕ, 1))
     return @. StaticArrays.SVector(uₕ_int.components.data.:1, uₕ_int.components.data.:2)
 end
 get_field(sim::ClimaAtmosSimulation, ::Val{:air_density}) =
@@ -156,7 +163,7 @@ function update_field!(sim::ClimaAtmosSimulation, ::Val{:turbulent_fluxes}, fiel
     (; F_turb_energy, F_turb_moisture, F_turb_ρτxz, F_turb_ρτyz) = fields
 
     Y = sim.integrator.u
-    surface_local_geometry = Fields.level(Fields.local_geometry_field(Y.f), Fields.half)
+    surface_local_geometry = ClimaCore.Fields.level(ClimaCore.Fields.local_geometry_field(Y.f), ClimaCore.Fields.half)
     surface_normal = @. C3(unit_basis_vector_data(C3, surface_local_geometry))
 
     # get template objects for the contravariant components of the momentum fluxes (required by Atmos boundary conditions)
@@ -204,7 +211,7 @@ struct CoupledMoninObukhov end
 """
     coupler_surface_setup(::CoupledMoninObukhov, p, csf_sfc = (; T = nothing, z0m = nothing, z0b = nothing, beta = nothing, q_vap = nothing))
 
-Sets up `surface_setup` as a `Fields.Field` of `SurfaceState`s.
+Sets up `surface_setup` as a `ClimaCore.Fields.Field` of `SurfaceState`s.
 """
 function coupler_surface_setup(
     ::CoupledMoninObukhov,
@@ -281,11 +288,11 @@ Returns the thermodynamic parameters from the atmospheric model simulation objec
 get_thermo_params(sim::ClimaAtmosSimulation) = CAP.thermodynamics_params(sim.integrator.p.params)
 
 """
-    calculate_surface_air_density(atmos_sim::ClimaAtmosSimulation, T_S::Fields.Field)
+    calculate_surface_air_density(atmos_sim::ClimaAtmosSimulation, T_S::ClimaCore.Fields.Field)
 
 Extension for this  to to calculate surface density.
 """
-function calculate_surface_air_density(atmos_sim::ClimaAtmosSimulation, T_S::Fields.Field)
+function calculate_surface_air_density(atmos_sim::ClimaAtmosSimulation, T_S::ClimaCore.Fields.Field)
     thermo_params = get_thermo_params(atmos_sim)
     ts_int = get_field(atmos_sim, Val(:thermo_state_int))
     extrapolate_ρ_to_sfc.(Ref(thermo_params), ts_int, swap_space!(ones(axes(ts_int.ρ)), T_S))
@@ -311,15 +318,15 @@ function get_field(atmos_sim::ClimaAtmosSimulation, ::Val{:radiative_energy_flux
     # save radiation source
     if atmos_sim.integrator.p.radiation.radiation_model != nothing
         face_space = axes(atmos_sim.integrator.u.f)
-        nz_faces = length(Spaces.vertical_topology(face_space).mesh.faces)
+        nz_faces = length(ClimaCore.Spaces.vertical_topology(face_space).mesh.faces)
 
         (; face_lw_flux_dn, face_lw_flux_up, face_sw_flux_dn, face_sw_flux_up) =
             atmos_sim.integrator.p.radiation.radiation_model
 
-        LWd_TOA = Fields.level(CA.RRTMGPI.array2field(FT.(face_lw_flux_dn), face_space), nz_faces - half)
-        LWu_TOA = Fields.level(CA.RRTMGPI.array2field(FT.(face_lw_flux_up), face_space), nz_faces - half)
-        SWd_TOA = Fields.level(CA.RRTMGPI.array2field(FT.(face_sw_flux_dn), face_space), nz_faces - half)
-        SWu_TOA = Fields.level(CA.RRTMGPI.array2field(FT.(face_sw_flux_up), face_space), nz_faces - half)
+        LWd_TOA = ClimaCore.Fields.level(CA.RRTMGPI.array2field(FT.(face_lw_flux_dn), face_space), nz_faces - half)
+        LWu_TOA = ClimaCore.Fields.level(CA.RRTMGPI.array2field(FT.(face_lw_flux_up), face_space), nz_faces - half)
+        SWd_TOA = ClimaCore.Fields.level(CA.RRTMGPI.array2field(FT.(face_sw_flux_dn), face_space), nz_faces - half)
+        SWu_TOA = ClimaCore.Fields.level(CA.RRTMGPI.array2field(FT.(face_sw_flux_up), face_space), nz_faces - half)
 
         return @. -(LWd_TOA + SWd_TOA - LWu_TOA - SWu_TOA) # [W/m^2]
     else
@@ -361,7 +368,7 @@ function dss_state!(sim::ClimaAtmosSimulation)
     Y = sim.integrator.u
     for key in propertynames(Y)
         field = getproperty(Y, key)
-        buffer = Spaces.create_dss_buffer(field)
-        Spaces.weighted_dss!(field, buffer)
+        buffer = ClimaCore.Spaces.create_dss_buffer(field)
+        ClimaCore.Spaces.weighted_dss!(field, buffer)
     end
 end

experiments/AMIP/components/land/bucket_init.jl

@@ -1,11 +1,12 @@
 # slab_rhs!
+using Dates: DateTime
+using Statistics: mean
+import SciMLBase: ODEProblem, init
+
 using ClimaCore
 import ClimaTimeSteppers as CTS
 import Thermodynamics as TD
-using Dates: DateTime
-using ClimaComms: AbstractCommsContext
 import CLIMAParameters
-
 import ClimaLand
 using ClimaLand.Bucket: BucketModel, BucketModelParameters, AbstractAtmosphericDrivers, AbstractRadiativeDrivers
 import ClimaLand.Bucket: BulkAlbedoTemporal, BulkAlbedoStatic, BulkAlbedoFunction

experiments/AMIP/components/land/bucket_utils.jl

@@ -79,7 +79,7 @@ step!(sim::BucketSimulation, t) = step!(sim.integrator, t - sim.integrator.t, tr
 reinit!(sim::BucketSimulation) = reinit!(sim.integrator)
 
 # extensions required by FluxCalculator (partitioned fluxes)
-function update_turbulent_fluxes_point!(sim::BucketSimulation, fields::NamedTuple, colidx::Fields.ColumnIndex)
+function update_turbulent_fluxes_point!(sim::BucketSimulation, fields::NamedTuple, colidx::ClimaCore.Fields.ColumnIndex)
     (; F_turb_energy, F_turb_moisture) = fields
     turbulent_fluxes = sim.integrator.p.bucket.turbulent_fluxes
     turbulent_fluxes.shf[colidx] .= F_turb_energy

experiments/AMIP/components/ocean/eisenman_seaice_init.jl

@@ -1,10 +1,14 @@
-import ClimaCoupler: FluxCalculator
+import SciMLBase: ODEProblem, init
+
+using ClimaCore
+using ClimaCore.Fields: getindex
 import ClimaTimeSteppers as CTS
+
+import ClimaCoupler: FluxCalculator
 import ClimaCoupler.FluxCalculator:
     update_turbulent_fluxes_point!, differentiate_turbulent_fluxes!, surface_thermo_state
 import ClimaCoupler.Interfacer: get_field, update_field!
-using ClimaCore.Fields: getindex
-import SciMLBase: step!, reinit!, init, ODEProblem
+import ClimaCoupler.FieldExchanger: step!, reinit!
 
 """
     EisenmanIceSimulation{P, Y, D, I}
@@ -46,14 +50,14 @@ end
 
 Initialize the state vectors for the Eisenman-Zhang sea ice model.
 """
-function state_init(p::EisenmanIceParameters, space::Spaces.AbstractSpace)
-    Y = Fields.FieldVector(
+function state_init(p::EisenmanIceParameters, space::ClimaCore.Spaces.AbstractSpace)
+    Y = ClimaCore.Fields.FieldVector(
         T_sfc = ones(space) .* p.T_freeze,
         h_ice = zeros(space),
         T_ml = ones(space) .* 277,
         q_sfc = ClimaCore.Fields.zeros(space),
     )
-    Ya = Fields.FieldVector(
+    Ya = ClimaCore.Fields.FieldVector(
         F_turb = ClimaCore.Fields.zeros(space),
         ∂F_turb_energy∂T_sfc = ClimaCore.Fields.zeros(space),
         F_rad = ClimaCore.Fields.zeros(space),
@@ -179,7 +183,7 @@ function ∑tendencies(dY, Y, cache, _)
     @. dY.T_sfc = -Y.T_sfc / Δt
     @. dY.q_sfc = -Y.q_sfc / Δt
 
-    Fields.bycolumn(axes(Y.T_sfc)) do colidx
+    ClimaCore.Fields.bycolumn(axes(Y.T_sfc)) do colidx
         solve_eisenman_model!(Y[colidx], Ya[colidx], p, thermo_params, Δt)
     end
 
@@ -254,7 +258,7 @@ get_field(sim::EisenmanIceSimulation, ::Val{:surface_albedo}) =
 get_field(sim::EisenmanIceSimulation, ::Val{:area_fraction}) = sim.integrator.p.area_fraction
 get_field(sim::EisenmanIceSimulation, ::Val{:air_density}) = sim.integrator.p.Ya.ρ_sfc
 
-function update_field!(sim::EisenmanIceSimulation, ::Val{:area_fraction}, field::Fields.Field)
+function update_field!(sim::EisenmanIceSimulation, ::Val{:area_fraction}, field::ClimaCore.Fields.Field)
     sim.integrator.p.area_fraction .= field
 end
 function update_field!(sim::EisenmanIceSimulation, ::Val{:turbulent_energy_flux}, field)
@@ -272,7 +276,11 @@ step!(sim::EisenmanIceSimulation, t) = step!(sim.integrator, t - sim.integrator.
 reinit!(sim::EisenmanIceSimulation) = reinit!(sim.integrator)
 
 # extensions required by FluxCalculator (partitioned fluxes)
-function update_turbulent_fluxes_point!(sim::EisenmanIceSimulation, fields::NamedTuple, colidx::Fields.ColumnIndex)
+function update_turbulent_fluxes_point!(
+    sim::EisenmanIceSimulation,
+    fields::NamedTuple,
+    colidx::ClimaCore.Fields.ColumnIndex,
+)
     (; F_turb_energy) = fields
     @. sim.integrator.p.Ya.F_turb[colidx] = F_turb_energy
 end

experiments/AMIP/components/ocean/prescr_seaice_init.jl

@@ -1,9 +1,15 @@
+import SciMLBase: ODEProblem, init
+
+using ClimaCore
 import ClimaTimeSteppers as CTS
+import Thermodynamics as TD
+
 import ClimaCoupler.Interfacer: SeaIceModelSimulation, get_field, update_field!, name
 import ClimaCoupler.FieldExchanger: step!, reinit!
 import ClimaCoupler.FluxCalculator: update_turbulent_fluxes_point!
 using ClimaCoupler: Regridder
-import Thermodynamics as TD
+import ClimaCoupler.Utilities: swap_space!
+import ClimaCoupler.BCReader: float_type_bcf
 
 include("../slab_utils.jl")
 
@@ -51,7 +57,7 @@ name(::IceSlabParameters) = "IceSlabParameters"
 
 # init simulation
 function slab_ice_space_init(::Type{FT}, space, p) where {FT}
-    Y = Fields.FieldVector(T_sfc = ones(space) .* p.T_freeze)
+    Y = ClimaCore.Fields.FieldVector(T_sfc = ones(space) .* p.T_freeze)
     return Y
 end
 
@@ -143,7 +149,7 @@ get_field(sim::PrescribedIceSimulation, ::Val{:surface_albedo}) = sim.integrator
 get_field(sim::PrescribedIceSimulation, ::Val{:area_fraction}) = sim.integrator.p.area_fraction
 get_field(sim::PrescribedIceSimulation, ::Val{:air_density}) = sim.integrator.p.ρ_sfc
 
-function update_field!(sim::PrescribedIceSimulation, ::Val{:area_fraction}, field::Fields.Field)
+function update_field!(sim::PrescribedIceSimulation, ::Val{:area_fraction}, field::ClimaCore.Fields.Field)
     sim.integrator.p.area_fraction .= field
 end
 
@@ -162,7 +168,11 @@ step!(sim::PrescribedIceSimulation, t) = step!(sim.integrator, t - sim.integrato
 reinit!(sim::PrescribedIceSimulation) = reinit!(sim.integrator)
 
 # extensions required by FluxCalculator (partitioned fluxes)
-function update_turbulent_fluxes_point!(sim::PrescribedIceSimulation, fields::NamedTuple, colidx::Fields.ColumnIndex)
+function update_turbulent_fluxes_point!(
+    sim::PrescribedIceSimulation,
+    fields::NamedTuple,
+    colidx::ClimaCore.Fields.ColumnIndex,
+)
     (; F_turb_energy) = fields
     @. sim.integrator.p.F_turb_energy[colidx] = F_turb_energy
 end
@@ -199,6 +209,6 @@ function dss_state!(sim::PrescribedIceSimulation)
     for key in propertynames(Y)
         field = getproperty(Y, key)
         buffer = get_dss_buffer(axes(field), p)
-        Spaces.weighted_dss!(field, buffer)
+        ClimaCore.Spaces.weighted_dss!(field, buffer)
     end
 end

experiments/AMIP/components/ocean/slab_ocean_init.jl

@@ -1,9 +1,12 @@
-using ClimaCore
+import SciMLBase: ODEProblem, init
 
+using ClimaCore
 import ClimaTimeSteppers as CTS
 import ClimaCoupler.Interfacer: OceanModelSimulation, get_field, update_field!, name
 import ClimaCoupler.FieldExchanger: step!, reinit!
 import ClimaCoupler.FluxCalculator: update_turbulent_fluxes_point!
+import ClimaCoupler.Utilities: swap_space!
+import ClimaCoupler.BCReader: float_type_bcf
 
 include("../slab_utils.jl")
 
@@ -141,7 +144,7 @@ get_field(sim::SlabOceanSimulation, ::Val{:surface_albedo}) = sim.integrator.p.p
 get_field(sim::SlabOceanSimulation, ::Val{:area_fraction}) = sim.integrator.p.area_fraction
 get_field(sim::SlabOceanSimulation, ::Val{:air_density}) = sim.integrator.p.ρ_sfc
 
-function update_field!(sim::SlabOceanSimulation, ::Val{:area_fraction}, field::Fields.Field)
+function update_field!(sim::SlabOceanSimulation, ::Val{:area_fraction}, field::ClimaCore.Fields.Field)
     sim.integrator.p.area_fraction .= field
 end
 
@@ -161,7 +164,11 @@ step!(sim::SlabOceanSimulation, t) = step!(sim.integrator, t - sim.integrator.t,
 reinit!(sim::SlabOceanSimulation) = reinit!(sim.integrator)
 
 # extensions required by FluxCalculator (partitioned fluxes)
-function update_turbulent_fluxes_point!(sim::SlabOceanSimulation, fields::NamedTuple, colidx::Fields.ColumnIndex)
+function update_turbulent_fluxes_point!(
+    sim::SlabOceanSimulation,
+    fields::NamedTuple,
+    colidx::ClimaCore.Fields.ColumnIndex,
+)
     (; F_turb_energy) = fields
     @. sim.integrator.p.F_turb_energy[colidx] = F_turb_energy
 end
@@ -198,6 +205,6 @@ function dss_state!(sim::SlabOceanSimulation)
     for key in propertynames(Y)
         field = getproperty(Y, key)
         buffer = get_dss_buffer(axes(field), p)
-        Spaces.weighted_dss!(field, buffer)
+        ClimaCore.Spaces.weighted_dss!(field, buffer)
     end
 end