greene overlay

benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile

@@ -1,9 +1,71 @@
- install h5py
-pip install pyyaml
-pip install git+https://github.com/mlperf/logging.git@1.0.0
+# export $PROJECT_SOURCE=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay
+# export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay-test
+# export SCRTACH=/scratch/$USER/github/
+
+.ONESHELL:
+SHELL=/bin/sh
+
+info:
+	cd $(<D)
+	echo $PROJECT_SOURCE
+	echo $PROJECT_DIR
+	echo $SCRATCH
+
+setup: mkdir-scratch clone
+
+
+mkdir-scratch:
+	mkdir -p $SCRATCH
+
+clone:
+	cd $SCRATCH
+	git clone https://github.com/laszewsk/mlcommons.git
+	cp -r $PROJECT_SOURCE $PROJECT_DIR 
+
+image-pull: cloudmask.sif
+	cd $PROJECT_DIR
+	cd  singularity pull docker://nvcr.io/nvidia/tensorflow:22.10-tf2-py3
+	make -f Makefile image-copy
+
+image-copy:
+	cd $PROJECT_DIR
+	cp tensorflow_22.10-tf2-py3.sif cloudmask.sif
+
+image-create: image-copy
+	cd $PROJECT_DIR
+	cp -rp /scratch/work/public/overlay-fs-ext3/overlay-15GB-500K.ext3.gz .
+	gunzip overlay-15GB-500K.ext3.gz
+
+
+image-mod:
+	cd $PROJECT_DIR
+	singularity exec --overlay overlay-15GB-500K.ext3:rw cloudmask.sif /bin/bash
+
+image-script:
+	cd $PROJECT_DIR
+	echo <<EOF
+	unset -f which
+	which python
+	which pip
+	python --version
+
+	pip install pip -U ; python --version
+	pip install scikit-learn
+
+	# pip install aws-cli
+
+	# install from requirements file in rivanna folder
+	# pip install protobuf==3.20.0 numpy tensorflow
+
+	pip install h5py
+	pip install pyyaml
+	pip install git+https://github.com/mlperf/logging.git@1.0.0
+
+	pip install cloudmesh-common
+	pip install cloudmesh-gpu
+	pip install cloudmesh-sbatch
+
+	EOF > a.sh
+
 
-pip install cloudmesh-common
-pip install cloudmesh-gpu
-pip install cloudmesh-sbatch
 
-EOF > a.sh~

benchmarks/cloudmask/target/green_overlay/image-overlay/README.md

@@ -1,45 +1,55 @@
-Setting up Singularity
--1. export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/rivanna_tmp
-0. 
-   mkdir /scratch/$USER/github/
-   cd /scratch/$USER/github/
-
-1. git clone https://github.com/laszewsk/mlcommons.git
-   cp -r mlcommons/benchmarks/cloudmask/target/rivanna/image-singularity mlcommons/benchmarks/cloudmask/target/rivanna_tmp/image-singularity
-
-2. cd mlcommons/benchmarks/cloudmask/target/rivanna_tmp/image-singularity
-3. singularity pull docker://nvcr.io/nvidia/tensorflow:22.10-tf2-py3
-4. cp tensorflow_22.10-tf2-py3.sif cloudmask.sif
-
-4*. cp -rp /scratch/work/public/overlay-fs-ext3/overlay-15GB-500K.ext3.gz .
-gunzip overlay-15GB-500K.ext3.gz
-
-
-5. singularity exec --overlay overlay-15GB-500K.ext3:rw cloudmask.sif /bin/bash
-unset -f which
-# source /ext3/env.sh
-which python
-which pip
-python --version
-
-
-6. Singuarity > 
-=====
-pip install pip -U ; python --version
-pip install scikit-learn
-# pip install aws-cli
-
-# install from requirements file in rivanna folder
-# pip install protobuf==3.20.0 numpy tensorflow  
-
-pip install h5py
-pip install pyyaml 
-pip install git+https://github.com/mlperf/logging.git@1.0.0
-
-pip install cloudmesh-common 
-pip install cloudmesh-gpu
-pip install cloudmesh-sbatch
-
-=====
-
-8. shell script (to set up singularity)
+# Setting up cloudmask with Singularity overlay
+
+Gregor von Laszewski (laszewski@gmail.com)
+Ruochen Guo
+
+
+## Step 1: Environment
+
+```bash
+greene> 
+	export $PROJECT_SOURCE=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay
+	export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay-test
+	export SCRTACH=/scratch/$USER/github/
+```
+
+If you like to automatize the, please set it in your .bashrc file
+
+
+
+## Step 2. Setting up Singularity
+
+```bash
+greene>
+  export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/rivanna_overlay
+  mkdir -p /scratch/$USER/github/
+  cd /scratch/$USER/github/
+  git clone https://github.com/laszewsk/mlcommons.git
+  cd $PROJECT_DIR  
+  singularity pull docker://nvcr.io/nvidia/tensorflow:22.10-tf2-py3
+  cp tensorflow_22.10-tf2-py3.sif cloudmask.sif
+```
+
+## Step 3. Create overlay image
+
+TODO: use native calls rather then bredefined space
+
+```bash
+greene>
+  cp -rp /scratch/work/public/overlay-fs-ext3/overlay-15GB-500K.ext3.gz .
+  gunzip overlay-15GB-500K.ext3.gz
+  singularity exec --overlay overlay-15GB-500K.ext3:rw cloudmask.sif /bin/bash
+  unset -f which
+  which python
+  which pip
+  python --version
+```
+
+## Step 5. Download data if not there
+
+* check if data is there
+* download the data using globus
+
+## Step 4. Run a test run to see if it works
+
+todo

benchmarks/cloudmask/target/green_overlay/image-overlay/requirements.txt

@@ -0,0 +1,16 @@
+matplotlib
+#tensorflow==2.9.0
+protobuf==3.20.1
+numpy
+tqdm
+tensorflow
+tensorflow-addons
+cloudmesh-gpu
+cloudmesh-common
+cloudmesh-ee
+scikit-learn
+h5py
+pyyaml
+awscli
+# git+https://github.com/mlperf/logging.git@2.1.0
+git+https://github.com/mlperf/logging.git@1.0.0