From 6d02a0723cbc2cac643f9622b5388926832ea2d6 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 17 Jul 2024 09:17:49 -0400 Subject: [PATCH 1/5] Control the version of numpy used in workers. Updating Dockerfile and Compose build config file for ngen-based worker images (calibration, partitioning, testing, etc.) to utilize ARG sourced from environment config to control the version of numpy that is installed. --- docker/main/docker-build.yml | 5 +++++ docker/main/ngen/Dockerfile | 31 +++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docker/main/docker-build.yml b/docker/main/docker-build.yml index c587e19bb..55d0e0151 100644 --- a/docker/main/docker-build.yml +++ b/docker/main/docker-build.yml @@ -49,6 +49,7 @@ services: context: ./ngen target: rocky-ngen-deps args: + NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} DOCKER_INTERNAL_REGISTRY: ${DOCKER_INTERNAL_REGISTRY:?} #depends_on: # - base @@ -59,6 +60,7 @@ services: context: ./ngen target: rocky_ngen_build_testing args: + NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} @@ -77,6 +79,7 @@ services: context: ./ngen target: ngen_worker args: + NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} @@ -98,6 +101,7 @@ services: context: ./ngen target: partitioner_image args: + NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} @@ -114,6 +118,7 @@ services: context: ./ngen target: ngen_cal_worker args: + NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} diff --git a/docker/main/ngen/Dockerfile b/docker/main/ngen/Dockerfile index 6bedbd2f6..1e8069bd1 100644 --- a/docker/main/ngen/Dockerfile +++ b/docker/main/ngen/Dockerfile @@ -65,9 +65,11 @@ ARG BOOST_VERSION=1.82.0 #mpich 3.2 doesn't work well gfortran 11 it seems, an alignment error crops up, but 3.3.2 seems to work... ARG MPICH_VERSION="3.3.2" ARG MIN_PYTHON="3.8.0" -ARG MIN_NUMPY="1.18.0" ARG BLOSC2_VERSION +# Necessry to control specific numpy version after release of 2.0 and prior to fully support by all NextGen ecosystem +ARG NUMPY_CONSTRAINT + ARG NETCDF_C_VERSION=4.8.1 ARG NETCDF_CXX_VERSION=4.3.1 ARG NETCDF_FORTRAN_VERSION=4.6.0 @@ -290,6 +292,7 @@ RUN if [[ "${NETCDF_CXX_VERSION}" == "latest" ]]; then \ FROM rocky-base as rocky-ngen-packaged-deps ARG ROCKY_NGEN_DEPS_REQUIRED +ARG NUMPY_CONSTRAINT # TODO: later, go back and change all pip3/python3 to just pip/python (but leave for now to limit scope) # Note that this includes numpy, which is needed for Python BMI support, regardless of BMI module @@ -297,7 +300,7 @@ USER root RUN dnf update -y && dnf install -y ${ROCKY_NGEN_DEPS_REQUIRED} && dnf clean -y all \ && ln -s $(which python3) $(which python3 | sed 's/python3/python/') \ && pip install --no-cache-dir "pip>=23.0,<23.1" wheel packaging \ - && if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install --no-cache-dir numpy; fi + && if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install --no-cache-dir numpy${NUMPY_CONSTRAINT}; fi USER ${USER} ################################################################################################################ @@ -332,7 +335,6 @@ COPY --chown=${USER} --from=download_boost /boost ${WORKDIR}/boost ARG MPICH_VERSION ARG MIN_PYTHON -ARG MIN_NUMPY ARG BLOSC2_VERSION ARG ROCKY_NGEN_DEPS_REQUIRED @@ -652,6 +654,8 @@ ARG BUILD_TOPMODEL ARG BUILD_PET ARG BUILD_SLOTH +ARG NUMPY_CONSTRAINT + COPY --chown=${USER} --from=rocky_init_repo ${WORKDIR}/ngen ${WORKDIR}/ngen COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/wheels /tmp/t-route-wheels COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/src/kernel /tmp/t-route-kernels @@ -666,8 +670,9 @@ RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then \ USER ${USER} ENV VIRTUAL_ENV=/dmod/venv -RUN python3 -m venv $VIRTUAL_ENV && pip3 install numpy +RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install numpy${NUMPY_CONSTRAINT} ; fi RUN cd ${BOOST_ROOT} \ && tar -xf boost_tarball.blob --strip 1 \ @@ -783,7 +788,25 @@ FROM rocky-ngen-deps as rocky_ngen_build_testing #FROM rocky_build_ngen as rocky_ngen_build_testing #FROM ${DOCKER_INTERNAL_REGISTRY}/ngen-deps:latest as rocky_ngen_build_testing +ARG NUMPY_CONSTRAINT + COPY --chown=${USER} --from=rocky_init_repo ${WORKDIR}/ngen ${WORKDIR}/ngen + +ENV BOOST_ROOT=${WORKDIR}/boost + +USER root +RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then \ + chgrp -R ${USER} /usr/local/lib*/python3.* ; \ + chmod -R g+sw /usr/local/lib*/python3.* ; \ + fi +USER ${USER} + +ENV VIRTUAL_ENV=/dmod/venv +RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then python3 -m venv ${VIRTUAL_ENV} ; fi +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install numpy${NUMPY_CONSTRAINT} ; fi + +RUN cd ${BOOST_ROOT} && tar -xf boost_tarball.blob --strip 1 && rm boost_tarball.blob ENV BOOST_ROOT=${WORKDIR}/boost #COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/wheels /tmp/t-route-wheels #COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/requirements.txt /tmp/t-route-requirements.txt From 22b308b5cc09ef46e0de18bc995874bddad30b37 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 17 Jul 2024 09:25:07 -0400 Subject: [PATCH 2/5] Update example.env with numpy version constraint. Updating example for DMOD deployment environment config with section for numpy, new numpy constraint environmental variable used by worker image builds, safe default value for new variable, and comments explaining. --- example.env | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/example.env b/example.env index 16b2bc94e..60e635d9d 100644 --- a/example.env +++ b/example.env @@ -192,6 +192,17 @@ PYCHARM_REMOTE_DEBUG_SERVER_PORT_GUI=55875 ## Path string for Pycharm remote debug package egg #PYCHARM_REMOTE_DEBUG_EGG_PATH= +################################################################################ +################################################################################ +## Numpy Settings ## +################################################################################ +################################################################################ + +## Version of Numpy to select when installing (e.g., within Docker images) +# Note: mind the consecutive equal signs; the value being set here is '==1.26.4' +NUMPY_INSTALL_CONSTRAINT===1.26.4 + + ################################################################################ ################################################################################ ## Docker Settings ## From 318afe5f308612630656aa8921d23633372c0134 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Wed, 17 Jul 2024 09:29:47 -0400 Subject: [PATCH 3/5] Update requirements files for numpy constraint. Updating project requirements.txt and GUI service requirements file (python/gui/dependencies.txt) to constrain numpy version consistent with default used by worker image builds set up in example.env. --- example.env | 4 ++++ python/gui/dependencies.txt | 2 +- requirements.txt | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/example.env b/example.env index 60e635d9d..c0cf82129 100644 --- a/example.env +++ b/example.env @@ -199,6 +199,10 @@ PYCHARM_REMOTE_DEBUG_SERVER_PORT_GUI=55875 ################################################################################ ## Version of Numpy to select when installing (e.g., within Docker images) +## Note that this probably needs to stay in sync with what is in a few other places +## that don't read directly from this file: +## - requirement.txt +## - python/gui/dependencies.txt # Note: mind the consecutive equal signs; the value being set here is '==1.26.4' NUMPY_INSTALL_CONSTRAINT===1.26.4 diff --git a/python/gui/dependencies.txt b/python/gui/dependencies.txt index 82c66ba24..fb4a267a3 100644 --- a/python/gui/dependencies.txt +++ b/python/gui/dependencies.txt @@ -17,4 +17,4 @@ channels channels-redis djangorestframework psycopg2-binary # TODO: get source package in future. Note that psycopg2 cannot be used on Mac; psycopg2-binary must be used -numpy +numpy==1.26.4 diff --git a/requirements.txt b/requirements.txt index 67aa677e3..eac573be2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ python-dotenv djangorestframework git+https://github.com/noaa-owp/hypy@master#egg=hypy&subdirectory=python hydrotools.nwis-client -numpy +numpy==1.26.4 scikit-learn minio uri From d302fc3dec5165d1d5e9185974c00ca13ef671f9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 18 Jul 2024 10:17:27 -0400 Subject: [PATCH 4/5] Roll back initial #678 changes. Rolling back initial changes in favor of a different approach. --- docker/main/docker-build.yml | 5 ----- docker/main/ngen/Dockerfile | 17 ++++------------- example.env | 15 --------------- python/gui/dependencies.txt | 2 +- requirements.txt | 2 +- 5 files changed, 6 insertions(+), 35 deletions(-) diff --git a/docker/main/docker-build.yml b/docker/main/docker-build.yml index 55d0e0151..c587e19bb 100644 --- a/docker/main/docker-build.yml +++ b/docker/main/docker-build.yml @@ -49,7 +49,6 @@ services: context: ./ngen target: rocky-ngen-deps args: - NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} DOCKER_INTERNAL_REGISTRY: ${DOCKER_INTERNAL_REGISTRY:?} #depends_on: # - base @@ -60,7 +59,6 @@ services: context: ./ngen target: rocky_ngen_build_testing args: - NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} @@ -79,7 +77,6 @@ services: context: ./ngen target: ngen_worker args: - NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} @@ -101,7 +98,6 @@ services: context: ./ngen target: partitioner_image args: - NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} @@ -118,7 +114,6 @@ services: context: ./ngen target: ngen_cal_worker args: - NUMPY_CONSTRAINT: ${NUMPY_INSTALL_CONSTRAINT:?Please set global NUMPY_INSTALL_CONSTRAINT in environment config} REPO_URL: ${NGEN_REPO_URL?No NGen repo url configured} BRANCH: ${NGEN_BRANCH?No NGen branch configured} COMMIT: ${NGEN_COMMIT} diff --git a/docker/main/ngen/Dockerfile b/docker/main/ngen/Dockerfile index 1e8069bd1..bd7a59457 100644 --- a/docker/main/ngen/Dockerfile +++ b/docker/main/ngen/Dockerfile @@ -67,9 +67,6 @@ ARG MPICH_VERSION="3.3.2" ARG MIN_PYTHON="3.8.0" ARG BLOSC2_VERSION -# Necessry to control specific numpy version after release of 2.0 and prior to fully support by all NextGen ecosystem -ARG NUMPY_CONSTRAINT - ARG NETCDF_C_VERSION=4.8.1 ARG NETCDF_CXX_VERSION=4.3.1 ARG NETCDF_FORTRAN_VERSION=4.6.0 @@ -292,15 +289,14 @@ RUN if [[ "${NETCDF_CXX_VERSION}" == "latest" ]]; then \ FROM rocky-base as rocky-ngen-packaged-deps ARG ROCKY_NGEN_DEPS_REQUIRED -ARG NUMPY_CONSTRAINT # TODO: later, go back and change all pip3/python3 to just pip/python (but leave for now to limit scope) -# Note that this includes numpy, which is needed for Python BMI support, regardless of BMI module +# Note that this includes numpy, which is needed for Python BMI support, regardless of BMI module USER root RUN dnf update -y && dnf install -y ${ROCKY_NGEN_DEPS_REQUIRED} && dnf clean -y all \ && ln -s $(which python3) $(which python3 | sed 's/python3/python/') \ && pip install --no-cache-dir "pip>=23.0,<23.1" wheel packaging \ - && if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install --no-cache-dir numpy${NUMPY_CONSTRAINT}; fi + && if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install --no-cache-dir numpy; fi USER ${USER} ################################################################################################################ @@ -654,8 +650,6 @@ ARG BUILD_TOPMODEL ARG BUILD_PET ARG BUILD_SLOTH -ARG NUMPY_CONSTRAINT - COPY --chown=${USER} --from=rocky_init_repo ${WORKDIR}/ngen ${WORKDIR}/ngen COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/wheels /tmp/t-route-wheels COPY --chown=${USER} --from=rocky_build_troute ${WORKDIR}/t-route/src/kernel /tmp/t-route-kernels @@ -670,9 +664,8 @@ RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then \ USER ${USER} ENV VIRTUAL_ENV=/dmod/venv -RUN python3 -m venv $VIRTUAL_ENV +RUN python3 -m venv $VIRTUAL_ENV && pip3 install numpy ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install numpy${NUMPY_CONSTRAINT} ; fi RUN cd ${BOOST_ROOT} \ && tar -xf boost_tarball.blob --strip 1 \ @@ -788,8 +781,6 @@ FROM rocky-ngen-deps as rocky_ngen_build_testing #FROM rocky_build_ngen as rocky_ngen_build_testing #FROM ${DOCKER_INTERNAL_REGISTRY}/ngen-deps:latest as rocky_ngen_build_testing -ARG NUMPY_CONSTRAINT - COPY --chown=${USER} --from=rocky_init_repo ${WORKDIR}/ngen ${WORKDIR}/ngen ENV BOOST_ROOT=${WORKDIR}/boost @@ -804,7 +795,7 @@ USER ${USER} ENV VIRTUAL_ENV=/dmod/venv RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then python3 -m venv ${VIRTUAL_ENV} ; fi ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install numpy${NUMPY_CONSTRAINT} ; fi +RUN if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install numpy ; fi RUN cd ${BOOST_ROOT} && tar -xf boost_tarball.blob --strip 1 && rm boost_tarball.blob ENV BOOST_ROOT=${WORKDIR}/boost diff --git a/example.env b/example.env index c0cf82129..16b2bc94e 100644 --- a/example.env +++ b/example.env @@ -192,21 +192,6 @@ PYCHARM_REMOTE_DEBUG_SERVER_PORT_GUI=55875 ## Path string for Pycharm remote debug package egg #PYCHARM_REMOTE_DEBUG_EGG_PATH= -################################################################################ -################################################################################ -## Numpy Settings ## -################################################################################ -################################################################################ - -## Version of Numpy to select when installing (e.g., within Docker images) -## Note that this probably needs to stay in sync with what is in a few other places -## that don't read directly from this file: -## - requirement.txt -## - python/gui/dependencies.txt -# Note: mind the consecutive equal signs; the value being set here is '==1.26.4' -NUMPY_INSTALL_CONSTRAINT===1.26.4 - - ################################################################################ ################################################################################ ## Docker Settings ## diff --git a/python/gui/dependencies.txt b/python/gui/dependencies.txt index fb4a267a3..82c66ba24 100644 --- a/python/gui/dependencies.txt +++ b/python/gui/dependencies.txt @@ -17,4 +17,4 @@ channels channels-redis djangorestframework psycopg2-binary # TODO: get source package in future. Note that psycopg2 cannot be used on Mac; psycopg2-binary must be used -numpy==1.26.4 +numpy diff --git a/requirements.txt b/requirements.txt index eac573be2..67aa677e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ python-dotenv djangorestframework git+https://github.com/noaa-owp/hypy@master#egg=hypy&subdirectory=python hydrotools.nwis-client -numpy==1.26.4 +numpy scikit-learn minio uri From 74f7759b4e3c4bee8b87a2ae324aeed53fb6d0a9 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 18 Jul 2024 10:21:12 -0400 Subject: [PATCH 5/5] Use pip constraints file in ngen Dockerfile. Adding constraints file to ngen (and related) image build to control numpy and pip versions as needed for compatibility, using this instead of doing directly within the Dockerfile itself in pip commands. --- docker/main/ngen/Dockerfile | 6 +++++- docker/main/ngen/constraints.txt | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 docker/main/ngen/constraints.txt diff --git a/docker/main/ngen/Dockerfile b/docker/main/ngen/Dockerfile index bd7a59457..f126661a9 100644 --- a/docker/main/ngen/Dockerfile +++ b/docker/main/ngen/Dockerfile @@ -290,12 +290,16 @@ FROM rocky-base as rocky-ngen-packaged-deps ARG ROCKY_NGEN_DEPS_REQUIRED +# Set up pip constraints file for this and descendent stages +COPY constraints.txt ${WORKDIR}/constraints.txt +ENV PIP_CONSTRAINT=${WORKDIR}/constraints.txt + # TODO: later, go back and change all pip3/python3 to just pip/python (but leave for now to limit scope) # Note that this includes numpy, which is needed for Python BMI support, regardless of BMI module USER root RUN dnf update -y && dnf install -y ${ROCKY_NGEN_DEPS_REQUIRED} && dnf clean -y all \ && ln -s $(which python3) $(which python3 | sed 's/python3/python/') \ - && pip install --no-cache-dir "pip>=23.0,<23.1" wheel packaging \ + && pip install --no-cache-dir pip wheel packaging \ && if [ "${NGEN_WITH_PYTHON}" == "ON" ]; then pip install --no-cache-dir numpy; fi USER ${USER} diff --git a/docker/main/ngen/constraints.txt b/docker/main/ngen/constraints.txt new file mode 100644 index 000000000..6a53b3f2b --- /dev/null +++ b/docker/main/ngen/constraints.txt @@ -0,0 +1,5 @@ +### Constrain pip +pip>=23.0,<23.1 + +### Constrain numpy to < 2.0.0 due to current ngen compatibility issue +numpy~=1.0 \ No newline at end of file