ci: Stability to CI/CD (#10694)

* ci: Switch to `--runtime=nvidia` Signed-off-by: Oliver Koenig <okoenig@nvidia.com> * fix success marker Signed-off-by: Oliver Koenig <okoenig@nvidia.com> * f Signed-off-by: Oliver Koenig <okoenig@nvidia.com> --------- Signed-off-by: Oliver Koenig <okoenig@nvidia.com>
NVIDIA · Oct 1, 2024 · 32503fd · 32503fd
1 parent c0a05f6
commit 32503fd
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/.github/workflows/_test_template.yml b/.github/workflows/_test_template.yml
@@ -59,7 +59,7 @@ jobs:
  ( 
  set -e
 
- docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=64g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.SCRIPT }}'
+ docker run --rm --runtime=nvidia --gpus all --shm-size=64g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.SCRIPT }}'
  ) 2> >(tee err.log)
 
  EXIT_CODE=$?
@@ -73,4 +73,4 @@ jobs:
  - name: after_script
  if: always() && inputs.AFTER_SCRIPT != ':'
  run: |
- docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=64g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.AFTER_SCRIPT }}'
+ docker run --rm --runtime=nvidia --gpus all --shm-size=64g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.AFTER_SCRIPT }}'
diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml
@@ -5409,7 +5409,7 @@ jobs:
  echo "FAILED=$FAILED" >> $GITHUB_OUTPUT
 
  # Mark as successful if no job was cancelled:
- SUCCESS=${{ !contains(needs.*.result, 'cancelled') }}
+ SUCCESS=${{ !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'skipped') }}
  echo "SUCCESS=$SUCCESS" >> $GITHUB_OUTPUT
 
  # This should depend on all the tests so we block/unblock based on all tests passing