Merge pull request #59 from lanl/rberger/ats4

Add ATS4 CI

.gitlab-ci.yml

@@ -1,49 +1,42 @@
 stages:
+  - static_analysis
   - build_n_test
 
-default:
-  interruptible: true
-
 variables:
-  ENABLED_CLUSTERS: "darwin"
-  GIT_SUBMODULE_STRATEGY: normal
+  PROJECT_NAME: ports-of-call
+  ENABLE_CDASH: false
 
-.ascgit_job:
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
+include:
+ - .gitlab/common.yml
 
-.darwin_job:
-  rules:
-    - if: $ENABLED_CLUSTERS =~ /darwin/
-  variables:
-    CLUSTER: darwin
-    SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p general,skylake-gold,skylake-platinum --constraint=\"(cpu_family:skylake)&ib:edr\""
-  tags:
-    - darwin-slurm-shared
-
-.build_and_test:
-  stage: build_n_test
-  script:
-    - source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}
-  artifacts:
-    expire_in: 2 weeks
-    paths:
-      - ${CI_PROJECT_DIR}/build/tests.xml
-    reports:
-      junit: ${CI_PROJECT_DIR}/build/tests.xml
 
 ########
 # Jobs #
 ########
 
 openmpi_gcc:
-  extends: [.ascgit_job, .darwin_job, .build_and_test]
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   variables:
     SPACK_ENV_NAME: openmpi-gcc
 
+openmpi_cuda_gcc_volta:
+  extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
+  variables:
+    SPACK_ENV_NAME: openmpi-cuda-gcc-volta
+    SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p volta-x86 -C cpu_family:haswell --time=02:00:00"
+
 openmpi_cuda_gcc_ampere:
   extends: [.ascgit_job, .darwin_job, .build_and_test]
   variables:
     SPACK_ENV_NAME: openmpi-cuda-gcc-ampere
     SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere"
+
+rzvernal_craympich_rocm_mi250_gcc:
+  extends: [.ascgit_job, .rzadams_job, .rzvernal_regular_job, .build_and_test]
+  variables:
+    SPACK_ENV_NAME: craympich-rocm-gfx90a-gcc
+
+rzadams_craympich_rocm_mi300_gcc:
+  extends: [.ascgit_job, .rzadams_job, .rzadams_regular_job, .build_and_test]
+  variables:
+    SPACK_ENV_NAME: craympich-rocm-gfx942-gcc

.gitlab/build_and_test.cmake

@@ -0,0 +1,75 @@
+set(CTEST_SOURCE_DIRECTORY "$ENV{SOURCE_DIR}")
+set(CTEST_BINARY_DIRECTORY "$ENV{BUILD_DIR}")
+set(CTEST_PROJECT_NAME "$ENV{PROJECT_NAME}")
+
+
+set(CTEST_SITE "$ENV{SYSTEM_NAME}")
+set(CTEST_BUILD_NAME "$ENV{SPACK_ENV_NAME}")
+
+set(CTEST_SUBMIT_URL "$ENV{CDASH_SERVER_URL}/submit.php?project=${CTEST_PROJECT_NAME}")
+set(CTEST_CURL_OPTIONS CURLOPT_SSL_VERIFYPEER_OFF CURLOPT_SSL_VERIFYHOST_OFF)
+set(CTEST_NIGHTLY_START_TIME "22:00:00 MDT")
+set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1024000)
+set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1024000)
+
+set(CTEST_UPDATE_COMMAND "git")
+set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "Skipping git update (no-op).")
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+set(CTEST_BUILD_CONFIGURATION "RelWithDebInfo")
+
+set(CTEST_MODE "$ENV{CTEST_MODE}")
+
+set(CTEST_OUTPUT_ON_FAILURE ON)
+set(CTEST_USE_LAUNCHERS TRUE)
+
+cmake_host_system_information(RESULT NUM_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES)
+
+if(${NUM_PHYSICAL_CORES} EQUAL 1)
+  # workaround for GraceHopper
+  include(ProcessorCount)
+  ProcessorCount(NUM_PHYSICAL_CORES)
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Configure)
+  ctest_start(${CTEST_MODE})
+else()
+  ctest_start(${CTEST_MODE} APPEND)
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Configure)
+  ctest_update(SOURCE "${CTEST_SOURCE_DIRECTORY}")
+  ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE configure_error)
+
+  if(configure_error)
+    if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+        ctest_submit()
+    endif()
+    message(FATAL_ERROR "configure error")
+  endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Build)
+  ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" PARALLEL_LEVEL ${NUM_PHYSICAL_CORES} RETURN_VALUE build_error)
+
+  if(build_error)
+    if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+        ctest_submit()
+    endif()
+    message(FATAL_ERROR "build error")
+  endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Test)
+  ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}" OUTPUT_JUNIT tests.xml RETURN_VALUE test_error)
+
+  if(test_error)
+    if(${CTEST_SCRIPT_ARG} MATCHES ReportErrors)
+        ctest_submit()
+    endif()
+    message(FATAL_ERROR "test error")
+  endif()
+endif()
+
+if(${CTEST_SCRIPT_ARG} MATCHES Submit)
+  ctest_submit()
+endif()

.gitlab/build_and_test.sh

@@ -1,12 +1,33 @@
 #!/bin/bash
-PROJECT_NAME=ports-of-call
-PROJECT_DEFAULT_BRANCH=main
-PROJECT_GROUP=oss
-BUILD_DIR=${BUILD_DIR:-build}
-SOURCE_DIR=${CI_PROJECT_DIR:-$PWD}
-PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/oss/current}
+###############################################################################
+export PROJECT_NAME=ports-of-call
+export PROJECT_DEFAULT_BRANCH=main
+export PROJECT_TYPE=oss
+export PROJECT_GROUP=oss
+###############################################################################
+
+export BUILD_DIR=${BUILD_DIR:-build}
+export SOURCE_DIR=${CI_PROJECT_DIR:-$PWD}
+TMPDIR=${TMPDIR:-/tmp/xcap/${PROJECT_GROUP}/${PROJECT_NAME}}
 CI_SPACK_ENV=${CI_SPACK_ENV:-$TMPDIR/$USER-ci-env}
-UNTIL=${UNTIL:-install}
+export CTEST_MODE=${CTEST_MODE:-Continuous}
+SUBMIT_TO_CDASH=${SUBMIT_TO_CDASH:-false}
+BUILD_WITH_CTEST=${BUILD_WITH_CTEST:-${SUBMIT_TO_CDASH}}
+SUBMIT_ON_ERROR=${SUBMIT_ON_ERROR:-${SUBMIT_TO_CDASH}}
+SHOW_HELP_MESSAGE=${SHOW_HELP_MESSAGE:-true}
+
+if ${SUBMIT_TO_CDASH}; then
+  UNTIL=${UNTIL:-submit}
+else
+  UNTIL=${UNTIL:-install}
+fi
+
+if ${SUBMIT_ON_ERROR}; then
+  REPORT_ERRORS="ReportErrors"
+else
+  REPORT_ERRORS=""
+fi
+PROJECT_SPACK_ENV_VERSION=${PROJECT_SPACK_ENV_VERSION:-current}
 
 # colors
 COLOR_BLUE='\033[1;34m'
@@ -47,7 +68,7 @@ print_usage() {
   echo "  environment    name of the Spack environment"
   echo
   echo "options:"
-  echo "  -u {spack,env,cmake,build,test,install}, --until {spack,env,cmake,build,test,install}"
+  echo "  -u {spack,env,cmake,build,test,install,submit}, --until {spack,env,cmake,build,test,install,submit}"
   echo "                 run script until the given phase"
   echo "  -h, --help     show this help message and exit"
 }
@@ -67,11 +88,34 @@ fi
 
 if ! $VALID_CMD; then
     print_usage
+    false
     return
 fi
 
-SYSTEM_NAME=$1
-SPACK_ENV_NAME=$2
+export SYSTEM_NAME=$1
+export SPACK_ENV_NAME=$2
+
+if [[ "$SYSTEM_NAME" == "darwin" || "$SYSTEM_NAME" == "rocinante" || "$SYSTEM_NAME" == "venado" ]]; then
+  PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/${PROJECT_TYPE}/${PROJECT_SPACK_ENV_VERSION}}
+elif [[ "$SYSTEM_NAME" == "rzadams" || "${SYSTEM_NAME}" == "rzansel" || "$SYSTEM_NAME" == "rzvernal" ]]; then
+  PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/workspace/xcap/spack-env/${PROJECT_TYPE}/${PROJECT_SPACK_ENV_VERSION}}
+else
+  echo "Unkown system '${SYSTEM_NAME}'"
+  false
+  return
+fi
+
+if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]] && [[ -z "${SPACK_ENV_SPEC}" ]]; then
+  echo "Spack environment 'custom-spec' requires SPACK_ENV_SPEC environment variable to be set!"
+  false
+  return
+fi
+
+if [[ "${SPACK_ENV_NAME}" == "custom-file" ]] && [[ -z "${SPACK_ENV_FILE}" ]]; then
+  echo "Spack environment 'custom-file' requires SPACK_ENV_FILE environment variable to be set!"
+  false
+  return
+fi
 
 ###############################################################################
 # Generic steps
@@ -83,6 +127,7 @@ SPACK_ENV_NAME=$2
 prepare_spack() {
   section start "prepare_spack[collapsed=true]" "Prepare Spack"
   umask 0007
+  mkdir -p $TMPDIR
   source $PROJECT_SPACK_ENV/replicate.sh $CI_SPACK_ENV
   section end "prepare_spack"
 }
@@ -93,12 +138,27 @@ prepare_spack() {
 prepare_env() {
   section start "prepare_env[collapsed=true]" "Create environment"
   echo "Activating ${SPACK_ENV_NAME} environment on ${SYSTEM_NAME}"
+
+  if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]]; then
+    SPACK_ENV_FILE=${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/custom/spack.yaml
+  fi
+
+  if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]] || [[ "${SPACK_ENV_NAME}" == "custom-file" ]]; then
+    mkdir -p ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}
+    cp ${SPACK_ENV_FILE} ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}/spack.yaml
+  fi
+
   source ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/activate.sh ${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}
   if [ -d ${SOURCE_DIR}/spack-repo ]; then
     spack repo add ${SOURCE_DIR}/spack-repo
   fi
 
   spack develop -b ${BUILD_DIR} -p ${SOURCE_DIR} --no-clone ${PROJECT_NAME}@${PROJECT_DEFAULT_BRANCH}
+
+  if [[ "${SPACK_ENV_NAME}" == "custom-spec" ]]; then
+    spack add ${SPACK_ENV_SPEC}
+  fi
+
   spack install --include-build-deps --only dependencies -j $(nproc)
   section end "prepare_env"
 }
@@ -122,7 +182,11 @@ cmake_build() {
   (
   source ${BUILD_ENV}
   cmake -DCMAKE_VERBOSE_MAKEFILE=off -DCMAKE_INSTALL_PREFIX=$PWD/${BUILD_DIR}/install $@ ${BUILD_DIR}
-  cmake --build ${BUILD_DIR} --parallel
+  if ${BUILD_WITH_CTEST}; then
+    ctest -VV -S .gitlab/build_and_test.cmake,Configure,Build,$REPORT_ERRORS
+  else
+    cmake --build ${BUILD_DIR} --parallel
+  fi
   )
   section end "cmake_build"
 }
@@ -135,11 +199,27 @@ cmake_test() {
   (
   source ${BUILD_ENV}
   export CTEST_OUTPUT_ON_FAILURE=1
-  ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@
+  if ${BUILD_WITH_CTEST}; then
+    ctest -V -S .gitlab/build_and_test.cmake,Test,$REPORT_ERRORS
+  else
+    ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@
+  fi
   )
   section end "testing"
 }
 
+#------------------------------------------------------------------------------
+# Submit results to CDash
+#------------------------------------------------------------------------------
+cmake_submit() {
+  section start "submit[collapsed=false]" "Submit to CDash"
+  (
+  source ${BUILD_ENV}
+  ctest -V -S .gitlab/build_and_test.cmake,Submit
+  )
+  section end "submit"
+}
+
 #------------------------------------------------------------------------------
 # Install project
 #------------------------------------------------------------------------------
@@ -162,23 +242,37 @@ activate_build_env() {
 ###############################################################################
 # Run script
 ###############################################################################
-echo "Running on $(hostname)"
-
-if [ ${CI} ]; then
-  echo " "
-  echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
-  echo " "
-  echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}"
-  echo " "
-  echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}"
-  echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}"
-  echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
-  echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh --until ${UNTIL} ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
-  echo " "
-  echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}"
-  echo " "
-  echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
-  echo " "
+if ${SHOW_HELP_MESSAGE}; then
+  echo "Running on $(hostname)"
+
+  if [ ${CI} ]; then
+    echo " "
+    echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
+    echo " "
+    echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}"
+    echo " "
+    echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}"
+    echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}"
+    if [[ ! -z "${LLNL_FLUX_SCHEDULER_PARAMETERS}" ]]; then
+      echo -e "${COLOR_BLUE}flux alloc ${LLNL_FLUX_SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+    elif [[ ! -z "${LLNL_LSF_SCHEDULER_PARAMETERS}" ]]; then
+      echo -e "${COLOR_BLUE}bsub -I ${LLNL_LSF_SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+    else
+      echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+    fi
+    if [[ ! -z "${SPACK_ENV_SPEC}" ]]; then
+      echo -e "${COLOR_BLUE}export SPACK_ENV_SPEC='${SPACK_ENV_SPEC}'${COLOR_PLAIN}"
+    fi
+    if [[ ! -z "${SPACK_ENV_FILE}" ]]; then
+      echo -e "${COLOR_BLUE}export SPACK_ENV_FILE='${SPACK_ENV_FILE}'${COLOR_PLAIN}"
+    fi
+    echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
+    echo " "
+    echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}"
+    echo " "
+    echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
+    echo " "
+  fi
 fi
 
 prepare_spack
@@ -198,3 +292,7 @@ if [[ "${UNTIL}" == "test" ]]; then return; fi
 
 cmake_install
 if [[ "${UNTIL}" == "install" ]]; then return; fi
+
+if ${SUBMIT_TO_CDASH}; then
+  cmake_submit
+fi