Tweak interface for integration into atmos (#77)

* Add backends for deploying calibration pipeline * Add simple slurm controller * Remove sphere HS experiment * Add hooks for reading information from ENV variables in Julia * Update dependencies
CliMA · May 3, 2024 · fe403ea · fe403ea
1 parent 2e30ffb
commit fe403ea
Show file tree

Hide file tree

Showing 51 changed files with 990 additions and 3,365 deletions.
diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -0,0 +1,24 @@
+agents:
+  queue: new-central
+  modules: climacommon/2024_04_30
+
+env:
+  OPENBLAS_NUM_THREADS: 1
+  SLURM_KILL_BAD_EXIT: 1
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
+  JULIA_DEPOT_PATH: "${BUILDKITE_BUILD_PATH}/${BUILDKITE_PIPELINE_SLUG}/depot/default"
+
+steps:
+  - label: "init :computer:"
+    key: "init_cpu_env"
+    command:
+      - echo "--- Instantiate SurfaceFluxes calibration project"
+      - julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.build("CalibrateAtmos"); Pkg.precompile()'
+
+  - wait
+  - label: "SurfaceFluxes perfect model calibration"
+    command: julia --project=experiments/surface_fluxes_perfect_model test/caltech_hpc_e2e.jl
+    artifact_paths: output/surface_fluxes_perfect_model/*
+
+  - label: "Slurm job controller unit tests"
+    command: julia --project=experiments/surface_fluxes_perfect_model test/slurm_unit_tests.jl
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -36,13 +36,9 @@ jobs:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
       - uses: julia-actions/julia-buildpkg@v1
-      - name: Surface Fluxes Perfect Model Test
-        run: |
-          julia --project=experiments/surface_fluxes_perfect_model -e 'using Pkg; Pkg.instantiate(;verbose=true)'
-          julia --project=experiments/surface_fluxes_perfect_model test/test_sf.jl
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           file: lcov.info
           token: ${{secrets.CODECOV_TOKEN}}
diff --git a/Project.toml b/Project.toml
@@ -4,20 +4,16 @@ authors = ["Climate Modeling Alliance"]
 version = "0.1.0"
 
 [deps]
-CalibrateEmulateSample = "95e48a1f-0bec-4818-9538-3db4340308e3"
-ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
-ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 EnsembleKalmanProcesses = "aa8a2aa5-91d8-4396-bcef-d4f2ec43552d"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+CalibrateEmulateSample = "95e48a1f-0bec-4818-9538-3db4340308e3"
 
 [compat]
 CalibrateEmulateSample = "0.5"
-ClimaComms = "0.5"
-ClimaParams = "0.10"
 Distributions = "0.25"
 EnsembleKalmanProcesses = "1"
 JLD2 = "0.4"

diff --git a/README.md b/README.md
diff --git a/calibrate.sh b/calibrate.sh
diff --git a/docs/make.jl b/docs/make.jl
@@ -24,7 +24,7 @@ makedocs(
     pages = [
         "Home" => "index.md",
         "Getting Started" => "quickstart.md",
-        "Experiment Setup Guide" => "experiment_setup_guide.md",
+        "ClimaAtmos Setup Guide" => "atmos_setup_guide.md",
         "Emulate and Sample" => "emulate_sample.md",
         "Precompilation" => "precompilation.md",
         "API" => "api.md",

diff --git a/docs/src/api.md b/docs/src/api.md
@@ -1,16 +1,24 @@
 # API
 
 ## Model Interface
+
 ```@docs
 CalibrateAtmos.get_config
 CalibrateAtmos.run_forward_model
 CalibrateAtmos.get_forward_model
 CalibrateAtmos.observation_map
 ```
 
-## EnsembleKalmanProcesses Interface
+## Backend Interface
+
 ```@docs
 CalibrateAtmos.calibrate
+CalibrateAtmos.sbatch_model_run
+```
+
+## EnsembleKalmanProcesses Interface
+
+```@docs
 CalibrateAtmos.initialize
 CalibrateAtmos.save_G_ensemble
 CalibrateAtmos.update_ensemble