update to use $TARGET

laszewsk · Sep 29, 2023 · 5f8f67b · 5f8f67b
1 parent ea5bd01
commit 5f8f67b
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 9 deletions.
diff --git a/benchmarks/cloudmask/target/greene_v0.5/README.md b/benchmarks/cloudmask/target/greene_v0.5/README.md
@@ -123,7 +123,14 @@ greene>
   python3 visualizer.py mlperf_cloudmask_200.log cloudmask_200.log
 ```
 
+## Killing all jobs
 
+To kill all jobs in the queue, please use
+
+```bash
+greene>
+    squeue -u $USER -h | awk '{print $1}' | xargs scancel
+```
 
 
 ---

diff --git a/benchmarks/cloudmask/target/greene_v0.5/config_simple.yaml b/benchmarks/cloudmask/target/greene_v0.5/config_simple.yaml
@@ -86,15 +86,15 @@ train_dir: /scratch/rg3515/data/one-day
 inference_dir: /scratch/rg3515/data/ssts
 
 # Model file
-model_file: /scratch/rg3515/github-fork/mlcommons/benchmarks/cloudmask/target/greene_v0.5/outputs/slstr_cloud/cloudModel.h5
+model_file: /scratch/rg3515/github/mlcommons/benchmarks/cloudmask/target/greene_v0.5/outputs/slstr_cloud/cloudModel.h5
 
 # training 
 training_loss: binary_crossentropy
 training_metrics: accuracy
 
 
 # Output directory
-output_dir: /scratch/rg3515/github-fork/mlcommons/benchmarks/cloudmask/target/greene_v0.5/outputs/slstr_cloud
+output_dir: /scratch/rg3515/github/mlcommons/benchmarks/cloudmask/target/greene_v0.5/outputs/slstr_cloud
 
 # Log file for recording runtimes
 log_file: ./cloudmask_final_1.log

diff --git a/benchmarks/cloudmask/target/greene_v0.5/killnote.txt b/benchmarks/cloudmask/target/greene_v0.5/killnote.txt
diff --git a/benchmarks/cloudmask/target/greene_v0.5/simple.slurm b/benchmarks/cloudmask/target/greene_v0.5/simple.slurm
@@ -8,10 +8,11 @@
 #SBATCH -o outputs/%u-%j.out
 #SBATCH -e outputs/%u-%j.err
 
-export USER_SCRATCH=/scratch/$USER/github-fork
+export USER_SCRATCH=/scratch/$USER/github
 export PROJECT_DIR=$USER_SCRATCH/mlcommons/benchmarks/cloudmask
 export PYTHON_DIR=$USER_SCRATCH/ENV3
 export PROJECT_DATA=/scratch/$USER/data
+export TARGET=PROJECT_DIR=/target/greene_v0.5
 
 module purge
 # module load anaconda3/2020.07
@@ -24,7 +25,7 @@ which python
 
 nvidia-smi
 
-cd $PROJECT_DIR/target/greene_v0.5
+cd $TARGET
 
 # mkdir -p outputs/slstr_cloud
 

diff --git a/benchmarks/cloudmask/target/greene_v0.5/tmptest-singularity.slurm b/benchmarks/cloudmask/target/greene_v0.5/tmptest-singularity.slurm
@@ -12,7 +12,7 @@ export USER_SCRATCH=/scratch/$USER/github-fork
 export PROJECT_DIR=$USER_SCRATCH/mlcommons/benchmarks/cloudmask
 export PYTHON_DIR=$USER_SCRATCH/ENV3
 export PROJECT_DATA=/scratch/$USER/data
-
+export TARGET=PROJECT_DIR=/target/greene_v0.5
 
 nvidia-smi
 cms gpu watch --gpu=0 --delay=0.5 --dense > outputs/gpu0.log &
@@ -21,12 +21,12 @@ cms gpu watch --gpu=0 --delay=0.5 --dense > outputs/gpu0.log &
 
 
 singularity exec --nv \
-    --overlay $PROJECT_DIR/target/greene_v0.5/tmptest-overlay-image:ro \
-    $PROJECT_DIR/target/greene_v0.5/cloudmask.sif \
+    --overlay $TARGET/tmptest-overlay-image:ro \
+    $TARGET/cloudmask.sif \
     /bin/bash \
     -c "source /ext3/env.sh; \
     source $PYTHON_DIR/bin/activate; \
-    cd $PROJECT_DIR/target/greene_v0.5; \
+    cd $TARGET; \
     python slstr_cloud.py --config config_simple.yaml"
 
 seff $SLURM_JOB_ID