Fix scaling pipeline (again)

.buildkite/scaling/pipeline.sh

@@ -65,32 +65,28 @@ for i in "${!resolutions[@]}"; do
 done
 
 # set up environment and agents
-cat << EOM
-env:
-  JULIA_VERSION: "1.9.3"
-  MPICH_VERSION: "4.0.0"
-  OPENMPI_VERSION: "4.1.1"
-  MPI_IMPL: "$mpi_impl"
-  CUDA_VERSION: "11.3"
-  OPENBLAS_NUM_THREADS: 1
-  CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED: "true"
-
+cat << 'EOM'
 agents:
-  config: cpu
   queue: central
+  modules: julia/1.9.3 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+
+env:
+  JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
+  OPENBLAS_NUM_THREADS: 1
+  JULIA_NVTX_CALLBACKS: gc
+  OMPI_MCA_opal_warn_on_missing_libcuda: 0
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
+  JULIA_CPU_TARGET: 'broadwell;skylake'
+  SLURM_KILL_BAD_EXIT: 1
 
 steps:
   - label: "init :computer:"
     key: "init_cpu_env"
     command:
-      - echo "--- Configure MPI"
-      - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
-
       - echo "--- Instantiate"
       - "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=examples -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=examples -e 'using Pkg; Pkg.status()'"
-
     agents:
       slurm_cpus_per_task: 8
     env:
@@ -129,11 +125,7 @@ if [[ "$profiling" == "enable" ]]; then
 else
     cpus_per_proc=1
 fi
-if [[ "$mpi_impl" == "mpich" ]]; then
-    launcher="srun --cpu-bind=cores"
-else
-    launcher="mpiexec --map-by node:PE=$cpus_per_proc --bind-to core"
-fi
+launcher="srun --cpu-bind=cores"
 
 if [[ "$res" == "low" ]]; then
     time="04:00:00"
@@ -160,7 +152,6 @@ cat << EOM
     - label: "$nprocs"
       key: "$job_id"
       command:
-        - "module load cuda/11.3 nsight-systems/2022.2.1"
         - "$launcher $command"
         - "find ${job_id} -iname '*.nsys-rep' -printf '%f\\\\n' | sort -V | jq --raw-input --slurp 'split(\"\n\") | .[0:-1] | {files: .} + {\"extension\": \"nsys-view\", \"version\": \"1.0\"}' > ${job_id}/${job_id}.nsys-view"
         - "find ${job_id} -iname '*.nsys-*' | sort -V | tar cvzf ${job_id}-nsys.tar.gz -T -"
@@ -170,8 +161,6 @@ cat << EOM
       env:
         CLIMACORE_DISTRIBUTED: "MPI"
       agents:
-        config: cpu
-        queue: central
         slurm_time: $time
 EOM
 
@@ -209,8 +198,6 @@ cat << EOM
         - "${res}-*.png"
         - "${res}-*.pdf"
       agents:
-        config: cpu
-        queue: central
         slurm_nodes: 1
         slurm_tasks_per_node: 1
 

post_processing/plot_scaling_results.jl

@@ -89,7 +89,8 @@ ax1 = Axis(
     xgridvisible = true,
     ygridvisible = false,
 )
-scatterlines!(ax1, nprocs_clima_atmos, sypd_clima_atmos)
+scatterlines!(nprocs_clima_atmos, sypd_clima_atmos)
+# Plot a second axis to display tick labels clearly
 ax1 = Axis(
     fig[1, 1],
     yaxisposition = :right,
@@ -103,15 +104,7 @@ ax1 = Axis(
     yscale = log10,
     ytickformat = "{:.2f}",
 )
-hlines!(
-    ax1,
-    sypd_clima_atmos,
-    xscale = log10,
-    yscale = log10,
-    color = :gray,
-    alpha = 0.5,
-    linestyle = :dash,
-)
+scatterlines!(nprocs_clima_atmos, sypd_clima_atmos)
 
 ax2 = Axis(
     fig[2, 1],