From 41986de3a5b0c111d38b1db5794b1b626f593bc4 Mon Sep 17 00:00:00 2001 From: Marcin Rogowski Date: Sun, 28 Apr 2024 14:34:58 +0200 Subject: [PATCH] Use venv (#32) Use python venv --- .github/workflows/containers.yml | 8 ++++---- docker/oshmem_fedora/Dockerfile | 5 +++-- docker/oshmem_ubuntu/Dockerfile | 7 +++++-- docker/oshmpi_fedora/Dockerfile | 5 +++-- docker/oshmpi_ubuntu/Dockerfile | 6 ++++-- docker/osss_fedora/Dockerfile | 5 +++-- docker/osss_ubuntu/Dockerfile | 6 ++++-- docker/sos_fedora/Dockerfile | 5 +++-- docker/sos_ubuntu/Dockerfile | 6 ++++-- 9 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 1b98b8c..e8ed747 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -25,10 +25,10 @@ jobs: - name: Build Docker container run: docker build -t local docker/${{ matrix.shmem }}/ - name: run shmem4py test-1 - run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "pip install . && make test-1 opt=-v" + run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "source /venv/bin/activate && pip install . && make test-1 opt=-v" - name: run shmem4py test-2 - run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "pip install . && make test-2 opt=-v" + run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "source /venv/bin/activate && pip install . && make test-2 opt=-v" - name: run shmem4py demo-test-1 - run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "pip install . && cd demo && make test-1 opt=-v" + run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "source /venv/bin/activate && pip install . && cd demo && make test-1 opt=-v" - name: run shmem4py demo-test-2 - run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "pip install . && cd demo && make test-2 opt=-v" + run: docker run -v $PWD:/repo -w/repo local /bin/bash -c "source /venv/bin/activate && pip install . && cd demo && make test-2 opt=-v" diff --git a/docker/oshmem_fedora/Dockerfile b/docker/oshmem_fedora/Dockerfile index 59ba3cd..e538832 100644 --- a/docker/oshmem_fedora/Dockerfile +++ b/docker/oshmem_fedora/Dockerfile @@ -1,4 +1,5 @@ FROM fedora:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -29,5 +30,5 @@ ENV PATH=/home/shmem/openmpi-4.1.6/install/bin:"${PATH}" \ OMPI_ALLOW_RUN_AS_ROOT=1 \ OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -RUN ln -s /usr/bin/python3 /usr/bin/python -RUN python -m pip install setuptools cffi numpy +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install setuptools cffi numpy diff --git a/docker/oshmem_ubuntu/Dockerfile b/docker/oshmem_ubuntu/Dockerfile index 8b8a610..681491a 100644 --- a/docker/oshmem_ubuntu/Dockerfile +++ b/docker/oshmem_ubuntu/Dockerfile @@ -1,4 +1,6 @@ FROM ubuntu:latest +SHELL ["/bin/bash", "-c"] + ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem ENV DEBIAN_FRONTEND=noninteractive @@ -10,7 +12,7 @@ RUN apt-get update -y && apt-get install -y \ automake \ libtool \ wget \ - python3 python3-pip python-is-python3 + python3 python3-pip python-is-python3 python3-venv RUN cd $INSTALL_DIR && \ wget https://github.com/openucx/ucx/archive/refs/tags/v1.15.0.tar.gz && \ @@ -33,4 +35,5 @@ ENV PATH=/home/shmem/openmpi-4.1.6/install/bin:"${PATH}" \ OMPI_ALLOW_RUN_AS_ROOT=1 \ OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -RUN python -m pip install numpy cffi +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install numpy cffi diff --git a/docker/oshmpi_fedora/Dockerfile b/docker/oshmpi_fedora/Dockerfile index 2e58cbe..1d7016b 100644 --- a/docker/oshmpi_fedora/Dockerfile +++ b/docker/oshmpi_fedora/Dockerfile @@ -1,4 +1,5 @@ FROM fedora:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -16,6 +17,6 @@ RUN cd $INSTALL_DIR make -j && make install ENV PATH="/home/shmem/oshmpi/install/bin/:/usr/lib64/mpich/bin/:${PATH}" -RUN ln -s /usr/bin/python3 /usr/bin/python -RUN python -m pip install numpy cffi +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install numpy cffi ENV PSM3_DEVICES="self,shm" diff --git a/docker/oshmpi_ubuntu/Dockerfile b/docker/oshmpi_ubuntu/Dockerfile index 0ff5658..d6712f0 100644 --- a/docker/oshmpi_ubuntu/Dockerfile +++ b/docker/oshmpi_ubuntu/Dockerfile @@ -1,4 +1,5 @@ FROM ubuntu:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -12,7 +13,7 @@ RUN apt-get update -y && apt-get install -y \ automake \ libtool \ mpich \ - python3 python3-pip python-is-python3 + python3 python3-pip python-is-python3 python3-venv RUN cd $INSTALL_DIR && \ git clone https://github.com/pmodels/oshmpi --recurse-submodules && \ @@ -22,4 +23,5 @@ RUN cd $INSTALL_DIR make -j && make install ENV PATH="/home/shmem/oshmpi/install/bin/:${PATH}" -RUN python -m pip install numpy cffi +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install numpy cffi diff --git a/docker/osss_fedora/Dockerfile b/docker/osss_fedora/Dockerfile index a1c93ca..4389c08 100644 --- a/docker/osss_fedora/Dockerfile +++ b/docker/osss_fedora/Dockerfile @@ -1,4 +1,5 @@ FROM fedora:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -36,5 +37,5 @@ ENV PATH=$INSTALL_DIR/osss-ucx/install/bin:/home/shmem/openmpi-4.1.6/install/bin OMPI_ALLOW_RUN_AS_ROOT=1 \ OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -RUN ln -s /usr/bin/python3 /usr/bin/python -RUN python -m pip install setuptools cffi numpy +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install setuptools cffi numpy diff --git a/docker/osss_ubuntu/Dockerfile b/docker/osss_ubuntu/Dockerfile index 61e1cf1..7c7b8ca 100644 --- a/docker/osss_ubuntu/Dockerfile +++ b/docker/osss_ubuntu/Dockerfile @@ -1,4 +1,5 @@ FROM ubuntu:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -13,7 +14,7 @@ RUN apt-get update -y && apt-get install -y \ libtool \ pkg-config \ libpmix-bin libpmix-dev \ - python3 python3-pip python-is-python3 + python3 python3-pip python-is-python3 python3-venv RUN cd $INSTALL_DIR && \ wget https://github.com/openucx/ucx/archive/refs/tags/v1.15.0.tar.gz && \ @@ -43,4 +44,5 @@ ENV PATH=$INSTALL_DIR/osss-ucx/install/bin:/home/shmem/openmpi-4.1.6/install/bin OMPI_ALLOW_RUN_AS_ROOT=1 \ OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -RUN python -m pip install setuptools cffi numpy +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install setuptools cffi numpy diff --git a/docker/sos_fedora/Dockerfile b/docker/sos_fedora/Dockerfile index 7c8291c..5ebc3ea 100644 --- a/docker/sos_fedora/Dockerfile +++ b/docker/sos_fedora/Dockerfile @@ -1,6 +1,7 @@ # adapted from https://github.com/Sandia-OpenSHMEM/SOS/blob/master/scripts/docker/Dockerfile FROM fedora:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -33,5 +34,5 @@ RUN cd $INSTALL_DIR make check TESTS= -j ENV PATH="/home/shmem/SOS/install/bin:/usr/lib64/mpich/bin:${PATH}" -RUN ln -s /usr/bin/python3 /usr/bin/python -RUN python -m pip install numpy cffi setuptools +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install numpy cffi setuptools diff --git a/docker/sos_ubuntu/Dockerfile b/docker/sos_ubuntu/Dockerfile index 4892f01..8c6660c 100644 --- a/docker/sos_ubuntu/Dockerfile +++ b/docker/sos_ubuntu/Dockerfile @@ -1,6 +1,7 @@ # adapted from https://github.com/Sandia-OpenSHMEM/SOS/blob/master/scripts/docker/Dockerfile FROM ubuntu:latest +SHELL ["/bin/bash", "-c"] ENV INSTALL_DIR=/home/shmem RUN mkdir /home/shmem @@ -19,7 +20,7 @@ RUN apt-get update -y && apt-get install -y \ libhwloc-dev \ libevent-dev \ mpich \ - python3 python3-pip python-is-python3 + python3 python3-pip python-is-python3 python3-venv # Build Libfabric RUN cd $INSTALL_DIR && \ @@ -41,4 +42,5 @@ RUN cd $INSTALL_DIR make check TESTS= -j ENV PATH="/home/shmem/SOS/install/bin:${PATH}" -RUN python -m pip install numpy cffi setuptools +RUN python3 -m venv /venv +RUN source /venv/bin/activate && pip install numpy cffi setuptools