Fix support for dependency version clauses for certain Python dependencies in Docker images

docker/py-sources/docker-build.yml

@@ -6,6 +6,17 @@ services:
     build:
       context: ../../
       dockerfile: ./docker/py-sources/py-deps.Dockerfile
+      args:
+        NUMPY_VERSION_CLAUSE: ${DOCKER_BUILD_NUMPY_VERSION_CLAUSE:->=1.18.0}
+        CRYPTOGRAPHY_VERSION_CLAUSE: ${DOCKER_BUILD_CRYPTOGRAPHY_VERSION_CLAUSE:-}
+        SHAPELY_VERSION_CLAUSE: ${DOCKER_BUILD_SHAPELY_VERSION_CLAUSE:->=2.0.0}
+        PANDAS_VERSION_CLAUSE: ${DOCKER_BUILD_PANDAS_VERSION_CLAUSE:-}
+        SKLEARN_VERSION_CLAUSE: ${DOCKER_BUILD_SKLEARN_VERSION_CLAUSE:-}
+        # This env var flags whether the above should override a version clause present already within requirements.txt
+        OVERRIDE_PROJ_VERSION_CLAUSES: ${DOCKER_BUILD_OVERRIDE_PROJ_VERSION_CLAUSES:-}
+        # Likewise, this one, if set, won't override, but will kill the build if there is something explicit in
+        # requirements.txt that doesn't match an .env-supplied clause
+        MATCH_EXPLICIT_PROJ_VERSION_CLAUSES: ${DOCKER_BUILD_MATCH_EXPLICIT_PROJ_VERSION_CLAUSES:-}
   py-sources:
     image: ${DOCKER_INTERNAL_REGISTRY:?Missing DOCKER_INTERNAL_REGISTRY value (see 'Private Docker Registry ' section in example.env)}/dmod-py-sources
     build:

docker/py-sources/py-deps.Dockerfile

@@ -1,4 +1,11 @@
 ARG REQUIRE="gcc g++ musl-dev gdal-dev libffi-dev openssl-dev rust cargo git proj proj-dev proj-util openblas openblas-dev lapack lapack-dev geos-dev"
+ARG NUMPY_VERSION_CLAUSE
+ARG CRYPTOGRAPHY_VERSION_CLAUSE
+ARG SHAPELY_VERSION_CLAUSE
+ARG PANDAS_VERSION_CLAUSE
+ARG SKLEARN_VERSION_CLAUSE
+ARG OVERRIDE_PROJ_VERSION_CLAUSES
+
 ################################################################################################################
 ################################################################################################################
 ##### Create foundational level build stage with initial structure
@@ -8,42 +15,43 @@ ARG REQUIRE
 RUN apk update && apk upgrade && apk add --no-cache ${REQUIRE}
 # Along with setup and wheel to build, install all project pip dependencies for package building later
 RUN mkdir /DIST && mkdir /dmod && pip install --upgrade pip
-RUN for d in numpy pandas crypt scikit-learn; do for b in DIST; do mkdir -p /${b}/${d}; done; done
+RUN for d in numpy pandas cryptography scikit-learn shapely; do for b in DIST; do mkdir -p /${b}/${d}; done; done
 
 ################################################################################################################
 ################################################################################################################
 ##### Create individual, semi-independent stages for building some of the longer-to-build packages to isolate
 ##### them in the cache.  This has the additional benefit of parallelizing these build steps.
 FROM foundation as build_numpy_dep
 RUN pip install --upgrade pip
-ARG NUMPY_VERSION=">=1.18.0"
-RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary numpy${NUMPY_VERSION}
+ARG NUMPY_VERSION_CLAUSE
+RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary numpy${NUMPY_VERSION_CLAUSE}
 
 ############################################################
 FROM foundation as build_cryptograph_dep
 RUN pip install --upgrade pip
-ARG CRYPTOGRAPHY_VERSION=""
-RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary cryptography${CRYPTOGRAPHY_VERSION}
+ARG CRYPTOGRAPHY_VERSION_CLAUSE
+RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary cryptography${CRYPTOGRAPHY_VERSION_CLAUSE}
 
 ############################################################
-FROM foundation as build_shapely_dep
+# This one also requires numpy
+FROM build_numpy_dep as build_shapely_dep
 RUN pip install --upgrade pip
-ARG SHAPELY_VERSION=""
-RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary shapely${SHAPELY_VERSION}
+ARG SHAPELY_VERSION_CLAUSE
+RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary shapely${SHAPELY_VERSION_CLAUSE}
 
 ############################################################
 # This one also requires numpy
 FROM build_numpy_dep as build_pandas_dep
 RUN pip install --upgrade pip
-ARG PANDAS_VERSION=""
-RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary pandas${PANDAS_VERSION}
+ARG PANDAS_VERSION_CLAUSE
+RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary pandas${PANDAS_VERSION_CLAUSE}
 
 ############################################################
 # This one requires numpy as well
 FROM build_numpy_dep as build_sklearn_dep
 RUN pip install --upgrade pip
-ARG SKLEARN_VERSION=""
-RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary scikit-learn${SKLEARN_VERSION}
+ARG SKLEARN_VERSION_CLAUSE
+RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary scikit-learn${SKLEARN_VERSION_CLAUSE}
 
 ################################################################################################################
 
@@ -53,6 +61,13 @@ RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary scikit-learn$
 ###### in its /DIST/ directory.
 FROM foundation as basis
 ARG REQUIRE
+ARG NUMPY_VERSION_CLAUSE
+ARG CRYPTOGRAPHY_VERSION_CLAUSE
+ARG SHAPELY_VERSION_CLAUSE
+ARG PANDAS_VERSION_CLAUSE
+ARG SKLEARN_VERSION_CLAUSE
+ARG OVERRIDE_PROJ_VERSION_CLAUSES
+ARG MATCH_EXPLICIT_PROJ_VERSION_CLAUSES
 
 # Copy what we built so far in those other (hopefully cached) stages
 COPY --from=build_pandas_dep /DIST/ /DIST/
@@ -77,8 +92,32 @@ RUN apk update && apk upgrade && apk add --no-cache ${REQUIRE} \
 
 # Copy project requirements file, which should have everything needed to build any package within project
 COPY ./requirements.txt /dmod/requirements.txt
-# Along with setup and wheel to build, install any remaining (see above) project pip dependencies for package building later
-RUN pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary -r /dmod/requirements.txt
+
+# Modify project requirements file to account for things for which we have version clauses set within the .env
+# Then, after requirements.txt is modified as needed, install any remaining (see above) project pip dependencies for
+# package building later
+RUN for pair in numpy:${NUMPY_VERSION_CLAUSE} \
+                pandas:${PANDAS_VERSION_CLAUSE} \
+                cryptography:${CRYPTOGRAPHY_VERSION_CLAUSE}  \
+                scikit-learn:${SKLEARN_VERSION_CLAUSE}  \
+                shapely:${SHAPELY_VERSION_CLAUSE} ; do \
+        pack=$(echo "${pair}" | cut -d : -f 1 -) ; \
+        ver=$(echo "${pair}" | cut -d : -f 2 -) ; \
+        if [ -n "${ver:-}" ]; then \
+            if [ -n "${OVERRIDE_PROJ_VERSION_CLAUSES:-}" ]; then \
+                cat /dmod/requirements.txt | sed "s/^${pack}.*/${pack}${ver}/" > /dmod/req_up.txt ; \
+            elif [ -n "${MATCH_EXPLICIT_PROJ_VERSION_CLAUSES:-}" ]; then \
+                proj_ver=$(cat /dmod/requirements.txt | grep -e "^${pack}" | sed "s/${pack}//") ; \
+                if [ "${ver}" != "${proj_ver}" ]; then \
+                    echo "Error: configured version ${ver} of package ${pack} does not match ${proj_ver} from project requirements" 2> ; \
+                fi ; \
+            else \
+                cat /dmod/requirements.txt | sed "s/^${pack}$/${pack}${ver}/" > /dmod/req_up.txt ; \
+            fi ; \
+            mv /dmod/req_up.txt /dmod/requirements.txt ; \
+        fi ; \
+    done \
+    && pip wheel --cache-dir /CACHE --wheel-dir /DIST --prefer-binary -r /dmod/requirements.txt
 
 
 ################################################################################################################