add singularity build

laszewsk · Oct 4, 2023 · 09542a5 · 09542a5
1 parent c2cfad1
commit 09542a5
Show file tree

Hide file tree

Showing 2 changed files with 106 additions and 0 deletions.
diff --git a/benchmarks/cloudmask/target/greene_v0.5/image-singularity/Makefile b/benchmarks/cloudmask/target/greene_v0.5/image-singularity/Makefile
@@ -0,0 +1,60 @@
+NAME=cloudmask
+CODE_DIR=..
+BUILD=cloudmask.def
+IMAGE=cloudmask.sif
+DATA=/scratch2/data/cloudmask/data
+BIND=--bind ${DATA}:${DATA}
+
+DATA_LOCALSCRATCH=/localscratch/${USER}/cloudmask/
+BIND_LOCALSCRATCH=--bind ${DATA}:${DATA}
+
+image:
+	time ../bin/singularity-image.py ${NAME}
+
+queue: watch
+
+delete:
+	rm -f *.out *.err
+	rm -f *_output.ipynb
+	rm *.log
+
+# TODO
+submit:
+	sbatch simple-a100-singularity.sh
+
+watch: status
+
+status:
+	watch squeue --format=\"%.18i %.9P %.50j %.8u %.8T %.10M %.9l %.6D %R\" --me
+
+run:
+	cd ${CODE_DIR}; mkdir -p outputs
+	cd ${CODE_DIR}; singularity exec ${BIND} --nv cloudmask.sif bash -c "python cloudmask_v2.py --config=config-new.yaml"
+
+#singularity exec --nv ${NAME}.sif papermill ${NAME}.ipynb ${NAME}_output.ipynb
+
+shell:
+	singularity ${BIND} shell --nv ${IMAGE}
+
+run-localscratch:
+	cd ${CODE_DIR}; mkdir -p outputs
+	cd ${CODE_DIR}; singularity exec ${BIND_LOCALSCRATCH} --nv cloudmask.sif bash -c "python cloudmask_v2.py --config=config-new.yaml"
+
+#singularity exec --nv ${NAME}.sif papermill ${NAME}.ipynb ${NAME}_output.ipynb
+
+shell-localscratch:
+	singularity ${BIND_LOCALSCRATCH} shell --nv ${IMAGE}
+
+shell-rivanna:
+	singularity shell --nv ${IMAGE}
+
+
+cancel: stop
+
+stop:
+	for i in "$$(squeue --user $$USER | awk 'NR>1{print $$1}')"; do scancel $$i ; done
+
+view:
+	watch tail -n 50 *.err
+
+all: delete clean image submit view
diff --git a/benchmarks/cloudmask/target/greene_v0.5/image-singularity/cloudmask.def b/benchmarks/cloudmask/target/greene_v0.5/image-singularity/cloudmask.def
@@ -0,0 +1,46 @@
+#Bootstrap: localimage
+#From: /share/resources/containers/singularity/tensorflow-2.10.0.sif
+#From: /share/resources/containers/singularity/tensorflow-2.4.1.sif
+#From: /share/resources/containers/singularity/tensorflow_23.03-tf1-py3.sif
+#From: /share/resources/containers/singularity/tensorflow_23.03-tf2-py3.sif
+
+Bootstrap: docker
+From:  nvcr.io/nvidia/tensorflow:22.10-tf2-py3
+
+%post
+    apt update
+    apt install -y python3-venv git
+    pip install pip -U ; python --version
+    # install from requirements file in rivanna folder
+    pip install protobuf==3.20.0 numpy tensorflow cloudmesh-gpu cloudmesh-common cloudmesh-sbatch scikit-learn h5py pyyaml awscli
+    pip install git+https://github.com/mlperf/logging.git@1.0.0
+
+
+
+    # protobuf solution
+
+    # # module purge ; module load gcc/11.2 openmpi/4.1.4 python/3.11.1 ; python -m venv ./EQVENV
+    # # source ./EQVENV/bin/activate ;
+    # # pip install --upgrade protobuf
+    # # python -m site
+    # # cp ~/.local/lib/python3.8/site-packages/google/protobuf/internal/builder.py ./builder.py
+    # pip install protobuf==3.20.3
+    # # cp ./builder.py ~/.local/lib/python3.8/site-packages/google/protobuf/internal/builder.py
+
+
+
+	# pip install git+https://github.com/mlperf/logging.git@1.0.0
+	# # pip install googleapis-common-protos
+	# # pip install --upgrade protobuf
+
+	pip install humanize
+	pip install numpy matplotlib pandas scikit-learn jupyter jupyterlab papermill
+
+    pip install tensorflow_datasets tensorflow-metadata
+    pip install tqdm wheel ipywidgets jupyter-autotime ipython-autotime
+
+	pip install cloudmesh-common cloudmesh-gpu
+
+
+
+