Merge pull request #419 from lanl/rberger/ats4

Changes for ATS4
lanl · Sep 30, 2024 · 1307c49 · 1307c49
2 parents 494ccaf + 924752a
commit 1307c49
Show file tree

Hide file tree

Showing 9 changed files with 559 additions and 149 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,4 @@ spack-configure-*
 *.sublime*
 CMakeUserPresets.json
 build/
-
+goldfiles.tar.gz
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,62 +1,14 @@
-###################
-# Global Settings #
-###################
-
 stages:
   - prepare
   - build_n_test
 
-default:
-  interruptible: true
-
 variables:
-  ENABLED_CLUSTERS: "darwin,rocinante"
+  PROJECT_NAME: singularity-eos
+  ENABLE_CDASH: false
   GIT_SUBMODULE_STRATEGY: normal
 
-.ascgit_job:
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
-
-.darwin_job:
-  rules:
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "pipeline"
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "schedule"
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_COMMIT_TAG
-  variables:
-    CLUSTER: darwin
-    SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p general,skylake-gold,skylake-platinum --constraint=\"(cpu_family:skylake)&ib:edr\""
-  tags:
-    - darwin-slurm-shared
-
-.rocinante_job:
-  rules:
-    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "pipeline"
-    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "schedule"
-    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_COMMIT_TAG
-  variables:
-    CLUSTER: rocinante
-    SCHEDULER_PARAMETERS: "-N 1 -A asc-ci -p ci --reservation ci --time=00:45:00"
-  tags:
-    - rocinante
-    - batch
-
-.report_tests:
-  artifacts:
-    expire_in: 2 weeks
-    paths:
-      - ${CI_PROJECT_DIR}/build/tests.xml
-    reports:
-      junit: ${CI_PROJECT_DIR}/build/tests.xml
-
-.build_and_test:
-  stage: build_n_test
-  script:
-    - source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}
+include:
+ - .gitlab/common.yml
 
 
 ########
@@ -65,7 +17,13 @@ variables:
 
 download_prereq:
   stage: prepare
-  extends: [.ascgit_job, .darwin_job]
+  extends: [.ascgit_job]
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "push"
+  needs: []
   variables:
     GIT_SUBMODULE_STRATEGY: none
   script:
@@ -75,41 +33,41 @@ download_prereq:
       - goldfiles.tar.gz
 
 openmpi_gcc:
-  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: openmpi-gcc
 
 openmpi_fortran_gcc:
-  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: openmpi-fortran-gcc
 
 openmpi_cuda_gcc_ampere:
-  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: openmpi-cuda-gcc-ampere
     SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere"
 
 openmpi_cuda_gcc_volta:
-  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: openmpi-cuda-gcc-volta
     SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p volta-x86 -C cpu_family:haswell"
 
 # currently disabled due to very long compilation times
 #openmpi_fortran_cuda_gcc_ampere:
-#  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+#  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
 #  needs: [download_prereq]
 #  variables:
 #    SPACK_ENV_NAME: openmpi-fortran-cuda-gcc-ampere
 #    SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere"
 #
 #openmpi_fortran_cuda_gcc_volta:
-#  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+#  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
 #  needs: [download_prereq]
 #  variables:
 #    SPACK_ENV_NAME: openmpi-fortran-cuda-gcc-volta
@@ -118,7 +76,7 @@ openmpi_cuda_gcc_volta:
 # EXAMPLE of how to modify an existing Spack environment and/or set CMake options
 #openmpi_cuda_gcc_volta_noclosure:
 #  stage: build_n_test
-#  extends: [.ascgit_job, .darwin_job, .report_tests,]
+#  extends: [.ascgit_job, .darwin_job, .darwin_regular_job,]
 #  needs: [download_prereq]
 #  variables:
 #    SPACK_ENV_NAME: openmpi-cuda-gcc-volta
@@ -134,39 +92,51 @@ openmpi_cuda_gcc_volta:
 #   - cmake_install
 
 openmpi_fortran_kokkos_openmp_gcc:
-  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: openmpi-fortran-kokkos-openmp-gcc
 
 openmpi_fortran_kokkos_static_openmp_gcc:
-  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: openmpi-fortran-kokkos-static-openmp-gcc
 
 #openmpi_rocm_gcc:
-#  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+#  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
 #  needs: [download_prereq]
 #  variables:
 #    spack_env_name: openmpi-rocm-gcc
 #    scheduler_parameters: "-n 1 --qos=debug -p shared-gpu-amd-mi250"
-
+#
 #openmpi_fortran_rocm_gcc:
-#  extends: [.ascgit_job, .darwin_job, .report_tests, .build_and_test]
+#  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
 #  needs: [download_prereq]
 #  variables:
 #    spack_env_name: openmpi-fortran-rocm-gcc
 #    scheduler_parameters: "-n 1 --qos=debug -p shared-gpu-amd-mi250"
 
-craympich_gcc:
-  extends: [.ascgit_job, .rocinante_job, .report_tests, .build_and_test]
+rocinante_craympich_gcc:
+  extends: [.ascgit_job, .rocinante_job, .rocinante_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: craympich-gcc
 
-craympich_fortran_gcc:
-  extends: [.ascgit_job, .rocinante_job, .report_tests, .build_and_test]
+rocinante_craympich_fortran_gcc:
+  extends: [.ascgit_job, .rocinante_job, .rocinante_regular_job, .build_and_test]
   needs: [download_prereq]
   variables:
     SPACK_ENV_NAME: craympich-fortran-gcc
+
+rzvernal_craympich_rocm_mi250_cce:
+  extends: [.ascgit_job, .rzadams_job, .rzvernal_regular_job, .build_and_test]
+  needs: [download_prereq]
+  variables:
+    SPACK_ENV_NAME: craympich-rocm-gfx90a-cce
+
+rzadams_craympich_rocm_mi300_cce:
+  extends: [.ascgit_job, .rzadams_job, .rzadams_regular_job, .build_and_test]
+  needs: [download_prereq]
+  variables:
+    SPACK_ENV_NAME: craympich-rocm-gfx942-cce
diff --git a/.gitlab/build_and_test.cmake b/.gitlab/build_and_test.cmake
@@ -0,0 +1,75 @@
+set(CTEST_SOURCE_DIRECTORY "$ENV{SOURCE_DIR}")
+set(CTEST_BINARY_DIRECTORY "$ENV{BUILD_DIR}")
+set(CTEST_PROJECT_NAME "$ENV{PROJECT_NAME}")
+
+
+set(CTEST_SITE "$ENV{SYSTEM_NAME}")
+set(CTEST_BUILD_NAME "$ENV{SPACK_ENV_NAME}")
+
+set(CTEST_SUBMIT_URL "$ENV{CDASH_SERVER_URL}/submit.php?project=${CTEST_PROJECT_NAME}")
+set(CTEST_CURL_OPTIONS CURLOPT_SSL_VERIFYPEER_OFF CURLOPT_SSL_VERIFYHOST_OFF)
+set(CTEST_NIGHTLY_START_TIME "22:00:00 MDT")
+set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1024000)
+set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1024000)
+
+set(CTEST_UPDATE_COMMAND "git")
+set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "Skipping git update (no-op).")
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+set(CTEST_BUILD_CONFIGURATION "RelWithDebInfo")
+
+set(CTEST_MODE "$ENV{CTEST_MODE}")
+
+set(CTEST_OUTPUT_ON_FAILURE ON)
+set(CTEST_USE_LAUNCHERS TRUE)
+
+cmake_host_system_information(RESULT NUM_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES)
+
+if(${NUM_PHYSICAL_CORES} EQUAL 1)
+  # workaround for GraceHopper
+  include(ProcessorCount)
+  ProcessorCount(NUM_PHYSICAL_CORES)
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Configure)
+  ctest_start(${CTEST_MODE})
+else()
+  ctest_start(${CTEST_MODE} APPEND)
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Configure)
+  ctest_update(SOURCE "${CTEST_SOURCE_DIRECTORY}")
+  ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE configure_error)
+
+  if(configure_error)
+    if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+        ctest_submit()
+    endif()
+    message(FATAL_ERROR "configure error")
+  endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Build)
+  ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" PARALLEL_LEVEL ${NUM_PHYSICAL_CORES} RETURN_VALUE build_error)
+
+  if(build_error)
+    if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+        ctest_submit()
+    endif()
+    message(FATAL_ERROR "build error")
+  endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Test)
+  ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" OUTPUT_JUNIT tests.xml RETURN_VALUE test_error)
+
+  if(test_error)
+    if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+        ctest_submit()
+    endif()
+    message(FATAL_ERROR "test error")
+  endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Submit)
+  ctest_submit()
+endif()