From 1c4b43b1fd178bf64a393b9296f55fd56c205dc1 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 7 Sep 2023 10:58:20 -0400 Subject: [PATCH 1/5] Pass certain version clauses into py-deps build. Passing version clauses for certain dependency packages into build from .env configuration file via compose config. --- docker/py-sources/docker-build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/py-sources/docker-build.yml b/docker/py-sources/docker-build.yml index bf7c7e3ad..3e90f6de0 100644 --- a/docker/py-sources/docker-build.yml +++ b/docker/py-sources/docker-build.yml @@ -6,6 +6,14 @@ services: build: context: ../../ dockerfile: ./docker/py-sources/py-deps.Dockerfile + args: + NUMPY_VERSION_CLAUSE: ${DOCKER_BUILD_NUMPY_VERSION_CLAUSE:->=1.18.0} + CRYPTOGRAPHY_VERSION_CLAUSE: ${DOCKER_BUILD_CRYPTOGRAPHY_VERSION_CLAUSE:-} + SHAPELY_VERSION_CLAUSE: ${DOCKER_BUILD_SHAPELY_VERSION_CLAUSE:->=2.0.0} + PANDAS_VERSION_CLAUSE: ${DOCKER_BUILD_PANDAS_VERSION_CLAUSE:-} + SKLEARN_VERSION_CLAUSE: ${DOCKER_BUILD_SKLEARN_VERSION_CLAUSE:-} + # This env var flags whether the above should override a version clause present already within requirements.txt + OVERRIDE_PROJ_VERSION_CLAUSES: ${DOCKER_BUILD_OVERRIDE_PROJ_VERSION_CLAUSES:-} py-sources: image: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)}/dmod-py-sources build: From d2c87c1d6a8f49aaeca3a67f3fa073d09da5e322 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 7 Sep 2023 10:59:30 -0400 Subject: [PATCH 2/5] Fix version clause usage in py-deps image build. Fixing version clause overrides for numpy, pandas, etc. --- docker/py-sources/py-deps.Dockerfile | 60 +++++++++++++++++++++------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/docker/py-sources/py-deps.Dockerfile b/docker/py-sources/py-deps.Dockerfile index da6d7e35b..0781260be 100644 --- a/docker/py-sources/py-deps.Dockerfile +++ b/docker/py-sources/py-deps.Dockerfile @@ -1,4 +1,11 @@ ARG REQUIRE="gcc g++ musl-dev gdal-dev libffi-dev openssl-dev rust cargo git proj proj-dev proj-util openblas openblas-dev lapack lapack-dev geos-dev" +ARG NUMPY_VERSION_CLAUSE +ARG CRYPTOGRAPHY_VERSION_CLAUSE +ARG SHAPELY_VERSION_CLAUSE +ARG PANDAS_VERSION_CLAUSE +ARG SKLEARN_VERSION_CLAUSE +ARG OVERRIDE_PROJ_VERSION_CLAUSES + ################################################################################################################ ################################################################################################################ ##### Create foundational level build stage with initial structure @@ -8,7 +15,7 @@ ARG REQUIRE RUN apk update && apk upgrade && apk add --no-cache ${REQUIRE} # Along with setup and wheel to build, install all project pip dependencies for package building later RUN mkdir /DIST && mkdir /dmod && pip install --upgrade pip -RUN for d in numpy pandas crypt scikit-learn; do for b in DIST; do mkdir -p /${b}/${d}; done; done +RUN for d in numpy pandas cryptography scikit-learn shapely; do for b in DIST; do mkdir -p /${b}/${d}; done; done ################################################################################################################ ################################################################################################################ @@ -16,34 +23,35 @@ RUN for d in numpy pandas crypt scikit-learn; do for b in DIST; do mkdir -p /${b ##### them in the cache. This has the additional benefit of parallelizing these build steps. FROM foundation as build_numpy_dep RUN pip install --upgrade pip -ARG NUMPY_VERSION=">=1.18.0" -RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary numpy${NUMPY_VERSION} +ARG NUMPY_VERSION_CLAUSE +RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary numpy${NUMPY_VERSION_CLAUSE} ############################################################ FROM foundation as build_cryptograph_dep RUN pip install --upgrade pip -ARG CRYPTOGRAPHY_VERSION="" -RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary cryptography${CRYPTOGRAPHY_VERSION} +ARG CRYPTOGRAPHY_VERSION_CLAUSE +RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary cryptography${CRYPTOGRAPHY_VERSION_CLAUSE} ############################################################ -FROM foundation as build_shapely_dep +# This one also requires numpy +FROM build_numpy_dep as build_shapely_dep RUN pip install --upgrade pip -ARG SHAPELY_VERSION="" -RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary shapely${SHAPELY_VERSION} +ARG SHAPELY_VERSION_CLAUSE +RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary shapely${SHAPELY_VERSION_CLAUSE} ############################################################ # This one also requires numpy FROM build_numpy_dep as build_pandas_dep RUN pip install --upgrade pip -ARG PANDAS_VERSION="" -RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary pandas${PANDAS_VERSION} +ARG PANDAS_VERSION_CLAUSE +RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary pandas${PANDAS_VERSION_CLAUSE} ############################################################ # This one requires numpy as well FROM build_numpy_dep as build_sklearn_dep RUN pip install --upgrade pip -ARG SKLEARN_VERSION="" -RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary scikit-learn${SKLEARN_VERSION} +ARG SKLEARN_VERSION_CLAUSE +RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary scikit-learn${SKLEARN_VERSION_CLAUSE} ################################################################################################################ @@ -53,6 +61,12 @@ RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary scikit-learn$ ###### in its /DIST/ directory. FROM foundation as basis ARG REQUIRE +ARG NUMPY_VERSION_CLAUSE +ARG CRYPTOGRAPHY_VERSION_CLAUSE +ARG SHAPELY_VERSION_CLAUSE +ARG PANDAS_VERSION_CLAUSE +ARG SKLEARN_VERSION_CLAUSE +ARG OVERRIDE_PROJ_VERSION_CLAUSES # Copy what we built so far in those other (hopefully cached) stages COPY --from=build_pandas_dep /DIST/ /DIST/ @@ -77,8 +91,26 @@ RUN apk update && apk upgrade && apk add --no-cache ${REQUIRE} \ # Copy project requirements file, which should have everything needed to build any package within project COPY ./requirements.txt /dmod/requirements.txt -# Along with setup and wheel to build, install any remaining (see above) project pip dependencies for package building later -RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary -r /dmod/requirements.txt + +# Modify project requirements file to account for things for which we have version clauses set within the .env +# Then, after requirements.txt is modified as needed, install any remaining (see above) project pip dependencies for +# package building later +RUN for pair in numpy:${NUMPY_VERSION_CLAUSE} \ + pandas:${PANDAS_VERSION_CLAUSE} \ + cryptography:${CRYPTOGRAPHY_VERSION_CLAUSE} \ + scikit-learn:${SKLEARN_VERSION_CLAUSE} \ + shapely:${SHAPELY_VERSION_CLAUSE} ; do \ + pack=$(echo "${pair}" | cut -d : -f 1 -) ; \ + ver=$(echo "${pair}" | cut -d : -f 2 -) ; \ + if [ -n "${ver:-}" ]; then \ + if [ -n "${OVERRIDE_PROJ_VERSION_CLAUSES:-}" ]; then \ + cat /dmod/requirements.txt | sed "s/^${pack}.*/${pack}${ver}/" > /dmod/req_up.txt ; \ + else \ + cat /dmod/requirements.txt | sed "s/^${pack}$/${pack}${ver}/" > /dmod/req_up.txt ; \ + mv /dmod/req_up.txt /dmod/requirements.txt ; \ + fi ; \ + done \ + && pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary -r /dmod/requirements.txt ################################################################################################################ \ No newline at end of file From f029a9b490a0d06c1f2ca7085398aa6467d915c6 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 7 Sep 2023 14:06:07 -0400 Subject: [PATCH 3/5] Fix missing fi in py-deps.Dockerfile. --- docker/py-sources/py-deps.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/py-sources/py-deps.Dockerfile b/docker/py-sources/py-deps.Dockerfile index 0781260be..5382a3b8a 100644 --- a/docker/py-sources/py-deps.Dockerfile +++ b/docker/py-sources/py-deps.Dockerfile @@ -107,6 +107,7 @@ RUN for pair in numpy:${NUMPY_VERSION_CLAUSE} \ cat /dmod/requirements.txt | sed "s/^${pack}.*/${pack}${ver}/" > /dmod/req_up.txt ; \ else \ cat /dmod/requirements.txt | sed "s/^${pack}$/${pack}${ver}/" > /dmod/req_up.txt ; \ + fi ; \ mv /dmod/req_up.txt /dmod/requirements.txt ; \ fi ; \ done \ From 798bba2e729098936bc09dfa00ed0769376d1a77 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 7 Sep 2023 14:08:15 -0400 Subject: [PATCH 4/5] Update py-deps for version clause matching. Updating with additional steps to (optionally) ensure that an explicit version clause, if provided on both sides, matches between .env and requirements.txt. --- docker/py-sources/py-deps.Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/py-sources/py-deps.Dockerfile b/docker/py-sources/py-deps.Dockerfile index 5382a3b8a..66278a75f 100644 --- a/docker/py-sources/py-deps.Dockerfile +++ b/docker/py-sources/py-deps.Dockerfile @@ -67,6 +67,7 @@ ARG SHAPELY_VERSION_CLAUSE ARG PANDAS_VERSION_CLAUSE ARG SKLEARN_VERSION_CLAUSE ARG OVERRIDE_PROJ_VERSION_CLAUSES +ARG MATCH_EXPLICIT_PROJ_VERSION_CLAUSES # Copy what we built so far in those other (hopefully cached) stages COPY --from=build_pandas_dep /DIST/ /DIST/ @@ -105,6 +106,11 @@ RUN for pair in numpy:${NUMPY_VERSION_CLAUSE} \ if [ -n "${ver:-}" ]; then \ if [ -n "${OVERRIDE_PROJ_VERSION_CLAUSES:-}" ]; then \ cat /dmod/requirements.txt | sed "s/^${pack}.*/${pack}${ver}/" > /dmod/req_up.txt ; \ + elif [ -n "${MATCH_EXPLICIT_PROJ_VERSION_CLAUSES:-}" ]; then \ + proj_ver=$(cat /dmod/requirements.txt | grep -e "^${pack}" | sed "s/${pack}//") ; \ + if [ "${ver}" != "${proj_ver}" ]; then \ + echo "Error: configured version ${ver} of package ${pack} does not match ${proj_ver} from project requirements" 2> ; \ + fi ; \ else \ cat /dmod/requirements.txt | sed "s/^${pack}$/${pack}${ver}/" > /dmod/req_up.txt ; \ fi ; \ From 3e3405e700fb1b36957646705faec18f310f16f0 Mon Sep 17 00:00:00 2001 From: Robert Bartel Date: Thu, 7 Sep 2023 14:09:07 -0400 Subject: [PATCH 5/5] Update py-deps stack for ver clause match. --- docker/py-sources/docker-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/py-sources/docker-build.yml b/docker/py-sources/docker-build.yml index 3e90f6de0..0eccdbf3b 100644 --- a/docker/py-sources/docker-build.yml +++ b/docker/py-sources/docker-build.yml @@ -14,6 +14,9 @@ services: SKLEARN_VERSION_CLAUSE: ${DOCKER_BUILD_SKLEARN_VERSION_CLAUSE:-} # This env var flags whether the above should override a version clause present already within requirements.txt OVERRIDE_PROJ_VERSION_CLAUSES: ${DOCKER_BUILD_OVERRIDE_PROJ_VERSION_CLAUSES:-} + # Likewise, this one, if set, won't override, but will kill the build if there is something explicit in + # requirements.txt that doesn't match an .env-supplied clause + MATCH_EXPLICIT_PROJ_VERSION_CLAUSES: ${DOCKER_BUILD_MATCH_EXPLICIT_PROJ_VERSION_CLAUSES:-} py-sources: image: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)}/dmod-py-sources build: