Update to Atmos v0.20.0

CliMA · Feb 4, 2024 · 9f9117b · 9f9117b
1 parent 7514bb9
commit 9f9117b
Show file tree

Hide file tree

Showing 15 changed files with 779 additions and 370 deletions.
diff --git a/Project.toml b/Project.toml
@@ -16,9 +16,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [compat]
-ClimaAtmos = "0.17"
-ClimaComms = "0.5"
-ClimaCore = "0.11"
+ClimaAtmos = "0.20.0"
 Distributions = "0.25"
 EnsembleKalmanProcesses = "1"
 JLD2 = "0.4"

diff --git a/experiments/Manifest.toml b/experiments/Manifest.toml
diff --git a/experiments/Project.toml b/experiments/Project.toml
@@ -12,16 +12,16 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [extras]
-CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
 CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc"
+CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
 MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [preferences.CUDA_Driver_jll]
 compat = false
 
 [preferences.CUDA_Runtime_jll]
-version = "12.2"
 local = "true"
+version = "12.2"
 
 [preferences.MPIPreferences]
 _format = "1.0"

diff --git a/experiments/common_env.sh b/experiments/common_env.sh
@@ -0,0 +1,10 @@
+# Configure for Resnick HPCC
+module load julia/1.10.0 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
+export OPENBLAS_NUM_THREADS=1
+export OMPI_MCA_opal_warn_on_missing_libcuda=0
+export JULIA_MAX_NUM_PRECOMPILE_FILES=100
+export JULIA_CPU_TARGET="broadwell;skylake"
+export JULIA_CUDA_MEMORY_POOL=none
+export JULIA_MPI_HAS_CUDA="true"
+export MPITRAMPOLINE_LIB=/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so
+export MPITRAMPOLINE_MPIEXEC=/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec
diff --git a/experiments/initialize.sbatch b/experiments/initialize.sbatch
@@ -7,9 +7,9 @@
 experiment_id=$1
 
 echo "Initializing calibration for experiment $experiment_id"
-julia --project=experiments -e 'using Pkg; Pkg.instantiate(;verbose=true)'
+julia --color=no --project=experiments -e 'using Pkg; Pkg.instantiate(;verbose=true)'
 
-julia --project=experiments -e '
+julia --color=no --project=experiments -e '
     import CalibrateAtmos
     CalibrateAtmos.initialize("'$experiment_id'")
 '

diff --git a/experiments/model_run.sbatch b/experiments/model_run.sbatch
@@ -13,7 +13,7 @@ member=$(printf "member_%03d" "$SLURM_ARRAY_TASK_ID")
 output=output/$experiment_id/$format_i/$member/model_log.out
 
 # Run the forward model
-srun --output=$output julia --project=experiments -e "
+srun --output=$output julia --color=no --project=experiments -e "
     using ClimaComms
     ClimaComms.init(ClimaComms.context())
     import CalibrateAtmos

diff --git a/experiments/pipeline.sh b/experiments/pipeline.sh
@@ -1,15 +1,7 @@
 #!/bin/bash
 
-# Configure for Resnick HPCC
-module load julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
-export OPENBLAS_NUM_THREADS=1
-export OMPI_MCA_opal_warn_on_missing_libcuda=0
-export JULIA_MAX_NUM_PRECOMPILE_FILES=100
-export JULIA_CPU_TARGET="broadwell;skylake"
-export JULIA_CUDA_MEMORY_POOL=none
-export JULIA_MPI_HAS_CUDA="true"
-export MPITRAMPOLINE_LIB=/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so
-export MPITRAMPOLINE_MPIEXEC=/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec
+# Configure the environment
+source experiments/common_env.sh
 
 # Parse command line
 experiment_id=${1?Error: no experiment ID given}

diff --git a/experiments/sphere_held_suarez_rhoe_equilmoist/atmos_config.yml b/experiments/sphere_held_suarez_rhoe_equilmoist/atmos_config.yml
@@ -1,7 +1,6 @@
 dt: 500secs
 t_end: 560days 
 ode_algo: ARS343 
-dt_save_to_disk: 50days 
 moist: equil 
 forcing: held_suarez 
 precip_model: 0M 

diff --git a/experiments/sphere_held_suarez_rhoe_equilmoist/generate_observations.sbatch b/experiments/sphere_held_suarez_rhoe_equilmoist/generate_observations.sbatch
@@ -4,10 +4,8 @@
 #SBATCH --cpus-per-task=8
 #SBATCH --output="experiments/sphere_held_suarez_rhoe_equilmoist/truth_simulation/model_log.out"
 
-module load julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415
-export OMPI_MCA_opal_warn_on_missing_libcuda=0
-export JULIA_NUM_PRECOMPILE_TASKS=8
-export JULIA_CPU_TARGET='broadwell;skylake'
+# Configure the environment
+source experiments/common_env.sh
 
 echo "Generating truth observations."
 
@@ -30,11 +28,9 @@ config = CA.AtmosConfig(config_dict)
 simulation = CA.get_simulation(config)
 CA.solve_atmos!(simulation)
 
-ta = ncread(joinpath(output_dir, "ta_60.0d_average.nc"), "ta")
+ta = ncread(joinpath(output_dir, "ta_60d_average.nc"), "ta")
 include(joinpath(experiment_dir, "observation_map.jl"))
 (; observation, variance) = process_member_data(ta; output_variance = true)
-@show observation
-@show variance
 JLD2.save_object(joinpath(experiment_dir, "obs_mean.jld2"), observation)
 JLD2.save_object(joinpath(experiment_dir, "obs_noise_cov.jld2"), variance)
 '
diff --git a/experiments/sphere_held_suarez_rhoe_equilmoist/obs_mean.jld2 b/experiments/sphere_held_suarez_rhoe_equilmoist/obs_mean.jld2
diff --git a/experiments/sphere_held_suarez_rhoe_equilmoist/obs_noise_cov.jld2 b/experiments/sphere_held_suarez_rhoe_equilmoist/obs_noise_cov.jld2
diff --git a/experiments/sphere_held_suarez_rhoe_equilmoist/observation_map.jl b/experiments/sphere_held_suarez_rhoe_equilmoist/observation_map.jl
@@ -43,7 +43,7 @@ function observation_map(iteration)
         YAML.load_file(joinpath("experiments", experiment_id, "ekp_config.yml"))
     output_dir = config["output_dir"]
     ensemble_size = config["ensemble_size"]
-    model_output = "ta_60.0d_average.nc"
+    model_output = "ta_60d_average.nc"
 
     dims = 1
     G_ensemble = Array{Float64}(undef, dims..., ensemble_size)

diff --git a/experiments/sphere_held_suarez_rhoe_equilmoist/truth_simulation/graceful_exit.dat b/experiments/sphere_held_suarez_rhoe_equilmoist/truth_simulation/graceful_exit.dat
@@ -0,0 +1 @@
+0