diff --git a/benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile b/benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile index 8960c975..2450b676 100644 --- a/benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile +++ b/benchmarks/cloudmask/target/green_overlay/image-overlay/Makefile @@ -1,9 +1,71 @@ - install h5py -pip install pyyaml -pip install git+https://github.com/mlperf/logging.git@1.0.0 +# export $PROJECT_SOURCE=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay +# export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay-test +# export SCRTACH=/scratch/$USER/github/ + +.ONESHELL: +SHELL=/bin/sh + +info: + cd $( a.sh + -pip install cloudmesh-common -pip install cloudmesh-gpu -pip install cloudmesh-sbatch -EOF > a.sh~ diff --git a/benchmarks/cloudmask/target/green_overlay/image-overlay/README.md b/benchmarks/cloudmask/target/green_overlay/image-overlay/README.md index 224ba052..5c40ab35 100644 --- a/benchmarks/cloudmask/target/green_overlay/image-overlay/README.md +++ b/benchmarks/cloudmask/target/green_overlay/image-overlay/README.md @@ -1,45 +1,55 @@ -Setting up Singularity --1. export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/rivanna_tmp -0. - mkdir /scratch/$USER/github/ - cd /scratch/$USER/github/ - -1. git clone https://github.com/laszewsk/mlcommons.git - cp -r mlcommons/benchmarks/cloudmask/target/rivanna/image-singularity mlcommons/benchmarks/cloudmask/target/rivanna_tmp/image-singularity - -2. cd mlcommons/benchmarks/cloudmask/target/rivanna_tmp/image-singularity -3. singularity pull docker://nvcr.io/nvidia/tensorflow:22.10-tf2-py3 -4. cp tensorflow_22.10-tf2-py3.sif cloudmask.sif - -4*. cp -rp /scratch/work/public/overlay-fs-ext3/overlay-15GB-500K.ext3.gz . -gunzip overlay-15GB-500K.ext3.gz - - -5. singularity exec --overlay overlay-15GB-500K.ext3:rw cloudmask.sif /bin/bash -unset -f which -# source /ext3/env.sh -which python -which pip -python --version - - -6. Singuarity > -===== -pip install pip -U ; python --version -pip install scikit-learn -# pip install aws-cli - -# install from requirements file in rivanna folder -# pip install protobuf==3.20.0 numpy tensorflow - -pip install h5py -pip install pyyaml -pip install git+https://github.com/mlperf/logging.git@1.0.0 - -pip install cloudmesh-common -pip install cloudmesh-gpu -pip install cloudmesh-sbatch - -===== - -8. shell script (to set up singularity) \ No newline at end of file +# Setting up cloudmask with Singularity overlay + +Gregor von Laszewski (laszewski@gmail.com) +Ruochen Guo + + +## Step 1: Environment + +```bash +greene> + export $PROJECT_SOURCE=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay + export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/greene-overlay-test + export SCRTACH=/scratch/$USER/github/ +``` + +If you like to automatize the, please set it in your .bashrc file + + + +## Step 2. Setting up Singularity + +```bash +greene> + export $PROJECT_DIR=/scratch/$USER/github/mlcommons/benchmarks/cloudmask/target/rivanna_overlay + mkdir -p /scratch/$USER/github/ + cd /scratch/$USER/github/ + git clone https://github.com/laszewsk/mlcommons.git + cd $PROJECT_DIR + singularity pull docker://nvcr.io/nvidia/tensorflow:22.10-tf2-py3 + cp tensorflow_22.10-tf2-py3.sif cloudmask.sif +``` + +## Step 3. Create overlay image + +TODO: use native calls rather then bredefined space + +```bash +greene> + cp -rp /scratch/work/public/overlay-fs-ext3/overlay-15GB-500K.ext3.gz . + gunzip overlay-15GB-500K.ext3.gz + singularity exec --overlay overlay-15GB-500K.ext3:rw cloudmask.sif /bin/bash + unset -f which + which python + which pip + python --version +``` + +## Step 5. Download data if not there + +* check if data is there +* download the data using globus + +## Step 4. Run a test run to see if it works + +todo \ No newline at end of file diff --git a/benchmarks/cloudmask/target/green_overlay/image-overlay/requirements.txt b/benchmarks/cloudmask/target/green_overlay/image-overlay/requirements.txt new file mode 100644 index 00000000..3aef7cdf --- /dev/null +++ b/benchmarks/cloudmask/target/green_overlay/image-overlay/requirements.txt @@ -0,0 +1,16 @@ +matplotlib +#tensorflow==2.9.0 +protobuf==3.20.1 +numpy +tqdm +tensorflow +tensorflow-addons +cloudmesh-gpu +cloudmesh-common +cloudmesh-ee +scikit-learn +h5py +pyyaml +awscli +# git+https://github.com/mlperf/logging.git@2.1.0 +git+https://github.com/mlperf/logging.git@1.0.0 \ No newline at end of file