diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml
index 7dbe8fda32..da72856021 100644
--- a/.buildkite/longruns/pipeline.yml
+++ b/.buildkite/longruns/pipeline.yml
@@ -2,7 +2,7 @@ env:
   JULIA_VERSION: "1.9.3"
   MPI_IMPL: "openmpi"
   OPENMPI_VERSION: "4.1.1"
-  CUDA_VERSION: "11.2"
+  CUDA_VERSION: "12.2"
   OPENBLAS_NUM_THREADS: 1
   CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED: "true"
   BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 32f106e19b..0773cef946 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -2,7 +2,7 @@ agents:
   queue: central
   slurm_mem: 8G
   slurm_time: 24:00:00
-  modules: julia/1.9.3 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+  modules: julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
diff --git a/test/mpi_tests/run_mpi_tests.jl b/test/mpi_tests/run_mpi_tests.jl
index e3402063a0..8459ec8aa5 100644
--- a/test/mpi_tests/run_mpi_tests.jl
+++ b/test/mpi_tests/run_mpi_tests.jl
@@ -2,8 +2,7 @@ using MPI
 #=
 # if running locally:
 module purge
-# module load julia/1.9.3 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
-module load julia/1.9.3 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+module load julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
 export OPENBLAS_NUM_THREADS=1
 export JULIA_NVTX_CALLBACKS=gc
 export OMPI_MCA_opal_warn_on_missing_libcuda=0