Skip to content

Commit

Permalink
sep script
Browse files Browse the repository at this point in the history
  • Loading branch information
LenkaNovak committed Mar 29, 2024
1 parent 6077e21 commit 3bdc5ab
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 5 deletions.
7 changes: 2 additions & 5 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -433,11 +433,8 @@ steps:
steps:
- label: "Submit and Monitor sbatch Job on Caltech HPC"
# check that (1) the script can be succesfully submitted, (2) it runs successfully
command:
- |
# Submit the sbatch script and capture its job ID
# JOB_ID=$(sbatch test/mpi_tests/local_checks.sh | awk '{print $4}')
agents:
command: "./../test/mpi_tests/test_sbatch_script.sh"
agents:
slurm_ntasks: 1

- wait
Expand Down
25 changes: 25 additions & 0 deletions test/mpi_tests/test_sbatch_script.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash

Submit the sbatch script and capture its job ID
JOB_ID=$(sbatch test/mpi_tests/local_checks.sh | awk '{print $4}')
echo "Submitted job with ID: $JOB_ID, output log: slurm-$JOB_ID.out"
START_TIME=$(date +%s)
# Loop until the job finishes
while true; do
# Check the status of the job
STATUS=$(squeue -j $JOB_ID | grep $JOB_ID | awk '{print $5}')
ELAPSED_TIME=$(( $(date +%s) - $START_TIME ))
# If the job status is 'PD' (pending) or 'R' (running), wait and continue checking
if [ "$STATUS" == "PD" ] || [ "$STATUS" == "R" ]; then
sleep 60
echo "Job is still running... Elapsed time: $ELAPSED_TIME seconds."
# If the job status is 'CF' (completed successfully), print success message and exit
elif [ "$STATUS" == "CF" ]; then
echo "Job completed successfully."
exit 0
# If the job status is anything else, print error message and exit
else
echo "Error: Job failed or terminated. See slurm-$JOB_ID.out for more information."
exit 1
fi
done

0 comments on commit 3bdc5ab

Please sign in to comment.