.gitlab-ci.yml

variables:
  GIT_STRATEGY: fetch
  GIT_SUBMODULE_STRATEGY: recursive

  RSYNC_OPTS: --recursive --ignore-missing-args --chown ${DEPLOY_USER}:${DEPLOY_USER}

  DOCKER_IMAGE: sogno/dpsim
  DOCKER_IMAGE_DEV: sogno/dpsim-dev
  DOCKER_IMAGE_DEV_ROCKY: sogno/dpsim-dev-rocky

  MAKE_OPTS: -j16 # for HYPER buildbots

stages:
  - prepare
  - build
  - test
  - generate
#  - deploy

# moved to github actions
#.docker-fedora-template: &docker-fedora-template
#  stage: prepare
#  image: docker:20.10
#  needs: []
#  script:
#    - docker build
#        --build-arg MAKE_OPTS=${MAKE_OPTS}
#        --file packaging/Docker/Dockerfile.dev
#        ${TAGS}
#        ${BUILD_ARGS} .
#  tags:
#    - docker
#
#docker:fedora-cgmes:
#  <<: *docker-fedora-template
#  variables:
#    TAGS:        --tag ${DOCKER_IMAGE_DEV}:${CI_COMMIT_REF_NAME}-cgmes --tag ${DOCKER_IMAGE_DEV}:latest
#    BUILD_ARGS:  --build-arg CIM_VERSION=CGMES_2.4.15_16FEB2016

# CGMES is the preferred CIM flavor
#docker:fedora-cim:
#  <<: *docker-fedora-template
#  variables:
#    TAGS:       --tag ${DOCKER_IMAGE_DEV}:${CI_COMMIT_REF_NAME}-cim
#    BUILD_ARGS: --build-arg CIM_VERSION=IEC61970_16v29a

docker:rocky:
  stage: prepare
  needs: []
  script:
    - docker build .
        --build-arg MAKE_OPTS=${MAKE_OPTS}
        --file packaging/Docker/Dockerfile.dev-rocky
        --tag ${DOCKER_IMAGE_DEV_ROCKY}:${CI_COMMIT_REF_NAME}
  tags:
    - docker

# moved to github actions
#build:linux-cgmes:
#  stage: build
#  needs: ["docker:fedora-cgmes"]
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DCIM_VERSION=CGMES_2.4.15_16FEB2016 ..
#    - make -j 32
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-cgmes
#  cache:
#  artifacts:
#    paths:
#      - build
#  tags:
#    - docker

# CGMES is the preferred CIM flavor
#build:linux-cim:
#  stage: build
#  needs: ["docker:fedora-cim"]
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DCIM_VERSION=IEC61970_16v29a ..
#    - make -j 32
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-cim
#  cache:
#    paths:
#      - build
#    key: build-linux
#  tags:
#    - docker

# moved to github actions
#build:linux-profiling:
#  stage: build
#  needs: ["docker:centos"]
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DWITH_PROFILING=ON -DWITH_ASAN=ON -DWITH_CUDA=OFF -DFETCH_SPDLOG=ON ..
#    - make -j 32
#  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
#  cache:
#  artifacts:
#    paths:
#      - build
#  tags:
#    - docker

build:linux-cuda:
  stage: build
  needs: ["docker:rocky"]
  image: ${DOCKER_IMAGE_DEV_ROCKY}:${CI_COMMIT_REF_NAME}
  artifacts:
    paths:
      - build
  tags:
    - docker
  script:
    - mkdir -p build
    - cd build
    - cmake -DWITH_CUDA=ON -DWITH_SPARSE=ON -DFETCH_SPDLOG=ON -DWITH_MAGMA=ON ..
    - make ${MAKE_OPTS}
  cache:

# moved to GitHub actions
#build:windows:
#  stage: build
#  needs: []
#  script:
#    - New-Item -ItemType Directory -Force -Path build
#    - Set-Location -Path build
#    - cmake ..
#    - cmake --build . -j 24

# moved to GitHub actions
#build:docker:
#  stage: build
#  needs: []
#  script:
#    - docker build
#        --file packaging/Docker/Dockerfile
#        --build-arg DOCKER_TAG=${DOCKER_TAG}
#        --build-arg MAKE_OPTS=${MAKE_OPTS}
#        --tag ${DOCKER_IMAGE}:${DOCKER_TAG}
#        --tag ${DOCKER_IMAGE}:latest .
#  tags:
#    - docker
#  only:
#    - tags

#build:macos:
#  stage: build
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DCMAKE_C_COMPILER=/usr/local/bin/gcc-9
#            -DCMAKE_CXX_COMPILER=/usr/local/bin/g++-9
#            -DFETCH_CIMPP=ON
#            -DWITH_SUNDIALS=OFF ..
#    - make -j $(sysctl -n hw.ncpu)
#  cache:
#    paths:
#      - build
#    key: build-macos
#  artifacts:
#    paths:
#      - build
#  tags:
#    - mac

#test:pytest:
#  stage: test
#  variables:
#    PYTHONPATH: "build/Source/Python:Source/Python"
#  script:
#    - pytest -v examples/CIM/test_CIM.yml
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
#  dependencies:
#    - build:linux
#  tags:
#    - docker

##############################################################################
# Stage test
##############################################################################

# moved to GitHub actions
#test:jupyter-cgmes:
#  stage: test
#  needs: ["build:linux-cgmes"]
#  variables:
#    PYTHONPATH: "build:build/Source/Python:Source/Python"
#  script:
#    - pytest -v examples/Notebooks
#  dependencies:
#    - build:linux-cgmes
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-cgmes
#  tags:
#    - docker
#  artifacts:
#    paths:
#      - outputs/examples/Notebooks/
#
#test:cppcheck 1/2:
#  stage: test
#  needs: ["docker:centos"]
#  script:
#    - set -o pipefail
#    - cppcheck --max-configs=32 -j 32 --error-exitcode=1 -q --std=c++11 -I Include/ -I models/Include/ Source/ examples/ models/Source/ | tee cppcheck-error.log
#  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
#  dependencies:
#    - docker:centos
#  tags:
#    - docker
#  artifacts:
#    when: on_failure
#    paths:
#      - cppcheck-error.log
#    expose_as: 'cppcheck-error'
#
#test:cppcheck 2/2:
#  stage: test
#  needs: ["docker:centos"]
#  allow_failure: true
#  script:
#    - set -o pipefail
#    - cppcheck --max-configs=32 -j 32 --error-exitcode=1 -q --enable=warning,performance,portability,information,missingInclude --std=c++11 -I Include/ -I models/Include/ Source/ examples/ models/Source/ | tee cppcheck-warn.log
#  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
#  dependencies:
#    - docker:centos
#  tags:
#    - docker
#  artifacts:
#    when: on_failure
#    paths:
#      - cppcheck-warn.log
#    expose_as: 'cppcheck-warn'
#
#.test-examples:
#  stage: test
#  needs: ["build:linux-profiling"]
#  script:
#    - $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM 2>&1 | tee output.log
#    - cat output.log |
#      sed -n -E -e 's/^.*Average step time. ([0-9]+\.[0-9]+)$/step-time \1/p' |
#      tee -a metrics.txt
#    - gprof $TEST_BINARY_PATH/$TEST_BINARY | gprof2dot -s | dot -Tpng -o profiling.png || true
#  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
#  dependencies:
#    - build:linux-profiling
#  tags:
#    - docker
#  artifacts:
#    reports:
#        metrics: metrics.txt
#    paths:
#      - profiling.png
#    expose_as: 'test-examples'
#
#test:examples 1/2:
#    extends: .test-examples
#    variables:
## Name of the test binary
#      TEST_BINARY: 'WSCC_9bus_mult_decoupled'
## Path where the test binary is located
#      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
## Command line parameters for the test binary
#      TEST_PARAM: ''
#
#test:examples 2/2:
#    extends: .test-examples
#    variables:
## Name of the test binary
#      TEST_BINARY: 'WSCC_9bus_mult_coupled'
## Path where the test binary is located
#      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
## Command line parameters for the test binary
#      TEST_PARAM: ''


.benchmark-script: &benchmark-script |
    for ((copies=0;copies<=$TEST_NUM_COPIES;copies++))
    do
      echo "Running with $copies copies"
      $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM -o copies=$copies 2>&1 | tee output.log
      cat output.log |
      sed -n -E -e 's/^.*Average step time. ([0-9]+\.[0-9]+)$/\1/p' |
      tee -a ${METRICS_ID}-metrics.txt
    done

.benchmark-step-times:
  stage: test
  script:
    - *benchmark-script
  image: ${DOCKER_IMAGE_DEV_ROCKY}:${CI_COMMIT_REF_NAME}
  tags:
    - docker

test:solver-benchmark 1/2:
    extends: .benchmark-step-times
    needs: ["build:linux-cuda"]
    dependencies:
      - build:linux-cuda
    variables:
# Metrics-filename
      METRICS_ID: 'cpu-dense'
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
# Up to how many copies should be benchmarked (WSCC_9bus - Example)
      TEST_NUM_COPIES: 6
      TEST_PARAM: "-U EigenDense"
    artifacts:
      reports:
          metrics: cpu-dense-metrics.txt
      paths:
        - cpu-dense-metrics.txt
      expose_as: 'step-times-benchmark'


test:solver-benchmark 2/2:
    extends: .benchmark-step-times
    needs: ["build:linux-cuda"]
    dependencies:
      - build:linux-cuda
    variables:
# Metrics-filename
      METRICS_ID: 'cpu-sparse'
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
# Up to how many copies should be benchmarked (WSCC_9bus - Example)
      TEST_NUM_COPIES: 6
      TEST_PARAM: "-U EigenSparse"
    artifacts:
      reports:
          metrics: cpu-sparse-metrics.txt
      paths:
        - cpu-sparse-metrics.txt
      expose_as: 'step-times-benchmark'


.gpu-init-script: &gpu-init-script |
    mkdir ~/.ssh &&
      echo "-----BEGIN OPENSSH PRIVATE KEY-----" > ~/.ssh/id_rsa &&
      echo $SSH_PRIVATE | sed "s/\S\{64\}/&\n/g" >> ~/.ssh/id_rsa &&
      echo "-----END OPENSSH PRIVATE KEY-----" >> ~/.ssh/id_rsa &&
      echo $SSH_PUBLIC > ~/.ssh/id_rsa.pub &&
      echo $KNOWN_HOSTS > ~/.ssh/known_hosts && chmod 600 ~/.ssh/id_rsa
    echo "$TEST_BINARY_PATH/$TEST_BINARY $TEST_FILES" | tr -d '\n' | xargs -r -d " " -I % cp % $LDIR
    ldd $LDIR/$TEST_BINARY |
      sed -n -E -e 's/^\s+\S+ => (\S+) \S+$/\1/p' |
      grep -Ev "$(echo $REMOTE_LIBS | tr ' ' '|')|not" |
      xargs -I % cp % $LDIR/
    ssh $GPU_TARGET mkdir -p $RDIR
    scp -r $LDIR/* $GPU_TARGET:$RDIR/

.gpu-run-script: &gpu-run-script |
  ssh $GPU_TARGET "LD_LIBRARY_PATH=$RDIR:${LD_LIBRARY_PATH} LD_PRELOAD=$RDIR/cricket-server.so $RDIR/$TEST_BINARY" &
  sleep 2
  PATH=$LDIR:${PATH} REMOTE_GPU_ADDRESS=ghost.acs-lab.eonerc.rwth-aachen.de LD_PRELOAD=$LDIR/cricket-client.so $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM 2>&1 | tee output.log
  cat output.log |
  sed -n -E -e 's/^.*Average step time. ([0-9]+\.[0-9]+)$/\1/p' |
  tee -a ${METRICS_ID}-metrics.txt

.remote-gpu:
  stage: test
  variables:
    GPU_TARGET: 'gitlab-runner@ghost'
    RDIR: '/tmp/gitlab-jobs/$CI_PROJECT_NAME/$CI_JOB_ID'
    CI_DIR: '$CI_BUILDS_DIR/$CI_PROJECT_PATH'
    LDIR: '/cricket/bin'
    TEST_FILES: ''
    TEST_PARAM: ''
    REMOTE_LIBS: 'linux-vdso.so.1
                  librt.so.1
                  libdl.so.2
                  libcudart.so.10.2
                  libcusolver.so.10
                  libstdc\+\+.so.6
                  libm.so.6
                  libgcc_s.so.1
                  libc.so.6
                  /lib64/ld-linux-x86-64.so.2
                  libutil.so.1
                  libpthread.so.0'
  script:
    - *gpu-init-script
    - *gpu-run-script
  after_script:
    - ssh $GPU_TARGET rm -rf $RDIR
    - ssh $GPU_TARGET pkill -fe -2 $RDIR/$TEST_BINARY
  image: ${DOCKER_IMAGE_DEV_ROCKY}:${CI_COMMIT_REF_NAME}
  tags:
    - docker

test:cuda 1/3:
    extends: .remote-gpu
    needs: ["build:linux-cuda"]
    dependencies:
    - build:linux-cuda
    variables:
# Metrics-filename
      METRICS_ID: 'cuda'
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: "-U CUDADense"
    artifacts:
      reports:
          metrics: cuda-metrics.txt
      paths:
        - cuda-metrics.txt
      expose_as: 'step-times-benchmark'

test:cuda 2/3:
    extends: .remote-gpu
    needs: ["build:linux-cuda"]
    dependencies:
    - build:linux-cuda
    allow_failure: true
    variables:
# Metrics-filename
      METRICS_ID: 'cuda'
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: '-U CUDASparse'
    artifacts:
      reports:
          metrics: cuda-metrics.txt
      paths:
        - cuda-metrics.txt
      expose_as: 'step-times-benchmark'

test:cuda 3/3:
    extends: .remote-gpu
    needs: ["build:linux-cuda"]
    allow_failure: true
    dependencies:
    - build:linux-cuda
    variables:
# Metrics-filename
      METRICS_ID: 'cuda'
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/dpsim/examples/cxx'
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: '-U CUDAMagma'
    artifacts:
      reports:
          metrics: cuda-metrics.txt
      paths:
        - cuda-metrics.txt
      expose_as: 'step-times-benchmark'


##############################################################################
# Stage generate
##############################################################################

# moved to GitHub actions
#generate:docs:
#  stage: generate
#  needs: ["build:linux-cgmes"]
#  script:
#    - make ${MAKE_OPTS} -C build docs
#    - make ${MAKE_OPTS} -C build docs_cxx
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-cgmes
#  dependencies:
#    - build:linux-cgmes
#  artifacts:
#    paths:
#      - build/docs/sphinx/html
#      - build/docs/doxygen/html
#  tags:
#    - docker
#
#generate:packages:
#  stage: generate
#  needs: ["build:linux-cgmes"]
#  script:
#    - make ${MAKE_OPTS} -C build package
#  only:
#    - tags
#  tags:
#    - docker
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-cgmes
#  dependencies:
#    - build:linux-cgmes
#  artifacts:
#    paths:
#      - build/*.rpm
#      - build/*.tar.gz

generate:metrics:
  stage: generate
  needs: ["test:solver-benchmark 1/2", "test:solver-benchmark 2/2"]
  script:
    - echo "set terminal svg size 800, 500; set output 'metrics.svg'; set title 'Speed of different Solver-Configurations'; set style data lines; set key outside; set xlabel \"Copies (WSCC-9bus Example)\"; set ylabel \"Average steptime (µs)\"; set xtics 1; set xtics nomirror; set ytics nomirror; set grid; plot \"cpu-dense-metrics.txt\" using (\$1 * 1000000) title \"Solving with CPU and dense matrices\", \"gpu-dense-metrics.txt\" using (\$1 * 1000000) title \"Solving with GPU and dense matrices\", \"cpu-sparse-metrics.txt\" using (\$1 * 1000000) title \"Solving with CPU and sparse matrices\"" > script.p
    - gnuplot script.p
  image: ${DOCKER_IMAGE_DEV_ROCKY}:${CI_COMMIT_REF_NAME}
  artifacts:
    reports:
      metrics: metrics.svg
    paths:
      - metrics.svg
    expose_as: 'WSCC-9bus-benchmark'
  tags:
    - docker


##############################################################################
# Stage deploy
##############################################################################

# moved to GitHub actions
#pages:
#  stage: deploy
#  needs: ["generate:docs"]
#  script:
#    - cp -r build/docs/sphinx/html/. public/sphinx
#    - cp -r build/docs/doxygen/html/. public/doxygen
#  artifacts:
#    paths:
#      - public
#  dependencies:
#    - generate:docs
#  only:
#    - deploy
#  tags:
#    - docker
#
#deploy:docker:
#  stage: deploy
#  script:
#    - docker push ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-cgmes
#    - docker push ${DOCKER_IMAGE_DEV}:latest
#    - docker push ${DOCKER_IMAGE}:${DOCKER_TAG}-cgmes
#    - docker push ${DOCKER_IMAGE}:latest
#  dependencies:
#    - build:docker
#  tags:
#    - docker
#  only:
#    - tags