update towards overlay

benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile

@@ -1,60 +1,9 @@
-NAME=cloudmask
-CODE_DIR=..
-BUILD=cloudmask.def
-IMAGE=cloudmask.sif
-DATA=/scratch2/data/cloudmask/data
-BIND=--bind ${DATA}:${DATA}
+ install h5py
+pip install pyyaml
+pip install git+https://github.com/mlperf/logging.git@1.0.0
 
-DATA_LOCALSCRATCH=/localscratch/${USER}/cloudmask/
-BIND_LOCALSCRATCH=--bind ${DATA}:${DATA}
+pip install cloudmesh-common
+pip install cloudmesh-gpu
+pip install cloudmesh-sbatch
 
-image:
-	time ../bin/singularity-image.py ${NAME}
-
-queue: watch
-
-delete:
-	rm -f *.out *.err
-	rm -f *_output.ipynb
-	rm *.log
-
-# TODO
-submit:
-	sbatch simple-a100-singularity.sh
-
-watch: status
-
-status:
-	watch squeue --format=\"%.18i %.9P %.50j %.8u %.8T %.10M %.9l %.6D %R\" --me
-
-run:
-	cd ${CODE_DIR}; mkdir -p outputs
-	cd ${CODE_DIR}; singularity exec ${BIND} --nv cloudmask.sif bash -c "python cloudmask_v2.py --config=config-new.yaml"
-
-#singularity exec --nv ${NAME}.sif papermill ${NAME}.ipynb ${NAME}_output.ipynb
-
-shell:
-	singularity ${BIND} shell --nv ${IMAGE}
-
-run-localscratch:
-	cd ${CODE_DIR}; mkdir -p outputs
-	cd ${CODE_DIR}; singularity exec ${BIND_LOCALSCRATCH} --nv cloudmask.sif bash -c "python cloudmask_v2.py --config=config-new.yaml"
-
-#singularity exec --nv ${NAME}.sif papermill ${NAME}.ipynb ${NAME}_output.ipynb
-
-shell-localscratch:
-	singularity ${BIND_LOCALSCRATCH} shell --nv ${IMAGE}
-
-shell-rivanna:
-	singularity shell --nv ${IMAGE}
-
-
-cancel: stop
-
-stop:
-	for i in "$$(squeue --user $$USER | awk 'NR>1{print $$1}')"; do scancel $$i ; done
-
-view:
-	watch tail -n 50 *.err
-
-all: delete clean image submit view
+EOF > a.sh~

benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile.1

@@ -0,0 +1,60 @@
+NAME=cloudmask
+CODE_DIR=..
+BUILD=cloudmask.def
+IMAGE=cloudmask.sif
+DATA=/scratch2/data/cloudmask/data
+BIND=--bind ${DATA}:${DATA}
+
+DATA_LOCALSCRATCH=/localscratch/${USER}/cloudmask/
+BIND_LOCALSCRATCH=--bind ${DATA}:${DATA}
+
+image:
+	time ../bin/singularity-image.py ${NAME}
+
+queue: watch
+
+delete:
+	rm -f *.out *.err
+	rm -f *_output.ipynb
+	rm *.log
+
+# TODO
+submit:
+	sbatch simple-a100-singularity.sh
+
+watch: status
+
+status:
+	watch squeue --format=\"%.18i %.9P %.50j %.8u %.8T %.10M %.9l %.6D %R\" --me
+
+run:
+	cd ${CODE_DIR}; mkdir -p outputs
+	cd ${CODE_DIR}; singularity exec ${BIND} --nv cloudmask.sif bash -c "python cloudmask_v2.py --config=config-new.yaml"
+
+#singularity exec --nv ${NAME}.sif papermill ${NAME}.ipynb ${NAME}_output.ipynb
+
+shell:
+	singularity ${BIND} shell --nv ${IMAGE}
+
+run-localscratch:
+	cd ${CODE_DIR}; mkdir -p outputs
+	cd ${CODE_DIR}; singularity exec ${BIND_LOCALSCRATCH} --nv cloudmask.sif bash -c "python cloudmask_v2.py --config=config-new.yaml"
+
+#singularity exec --nv ${NAME}.sif papermill ${NAME}.ipynb ${NAME}_output.ipynb
+
+shell-localscratch:
+	singularity ${BIND_LOCALSCRATCH} shell --nv ${IMAGE}
+
+shell-rivanna:
+	singularity shell --nv ${IMAGE}
+
+
+cancel: stop
+
+stop:
+	for i in "$$(squeue --user $$USER | awk 'NR>1{print $$1}')"; do scancel $$i ; done
+
+view:
+	watch tail -n 50 *.err
+
+all: delete clean image submit view

benchmarks/cloudmask/target/green_overlay/image-overlay/README.md

@@ -0,0 +1,45 @@
+Setting up Singularity
+-1. export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/rivanna_tmp
+0. 
+   mkdir /scratch/$USER/github/
+   cd /scratch/$USER/github/
+
+1. git clone https://github.com/laszewsk/mlcommons.git
+   cp -r mlcommons/benchmarks/cloudmask/target/rivanna/image-singularity mlcommons/benchmarks/cloudmask/target/rivanna_tmp/image-singularity
+
+2. cd mlcommons/benchmarks/cloudmask/target/rivanna_tmp/image-singularity
+3. singularity pull docker://nvcr.io/nvidia/tensorflow:22.10-tf2-py3
+4. cp tensorflow_22.10-tf2-py3.sif cloudmask.sif
+
+4*. cp -rp /scratch/work/public/overlay-fs-ext3/overlay-15GB-500K.ext3.gz .
+gunzip overlay-15GB-500K.ext3.gz
+
+
+5. singularity exec --overlay overlay-15GB-500K.ext3:rw cloudmask.sif /bin/bash
+unset -f which
+# source /ext3/env.sh
+which python
+which pip
+python --version
+
+
+6. Singuarity > 
+=====
+pip install pip -U ; python --version
+pip install scikit-learn
+# pip install aws-cli
+
+# install from requirements file in rivanna folder
+# pip install protobuf==3.20.0 numpy tensorflow  
+
+pip install h5py
+pip install pyyaml 
+pip install git+https://github.com/mlperf/logging.git@1.0.0
+
+pip install cloudmesh-common 
+pip install cloudmesh-gpu
+pip install cloudmesh-sbatch
+
+=====
+
+8. shell script (to set up singularity)

benchmarks/cloudmask/target/green_overlay/image-overlay/install.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+pip install pip -U ; python --version
+pip install scikit-learn
+# pip install aws-cli
+
+# install from requirements file in rivanna folder
+# pip install protobuf==3.20.0 numpy tensorflow
+
+pip install h5py
+pip install pyyaml
+pip install git+https://github.com/mlperf/logging.git@1.0.0
+
+pip install cloudmesh-common
+pip install cloudmesh-gpu
+pip install cloudmesh-sbatch