Bump base image Ubuntu to 24.04, Bump gdal to 3.9 (#3995)

- Rebuilt `wps-api-base`to use `osgeo/gdal:ubuntu-small-3.9.2` which comes with Ubuntu 24.04, python 3.12.3 and gdal 3.9.2 - Updates tippecanoe to 2.62.5 - Installing gdal for builds requires a few more flags otherwise it won't recognize the `numpy` installation - Images already have GID 1000 and UID 1000 so a new user with ID 1010 is created and added to existing GID 1000 for mount permissions - Uses `cutlineWkt` instead of `cutlineSQL` from `gdal` to avoid issues with gdal postgres search paths - A bunch of fixes to C-Haines code that started failing tests with new poetry updates - Documentation and setup script updates to reference the new python version - Removes `shapely` from `tileserv` project since it doesn't build with python 3.12.3 and we don't use it anymore
bcgov · Oct 15, 2024 · 1f3e00b · 1f3e00b
1 parent 72d3bbb
commit 1f3e00b
Show file tree

Hide file tree

Showing 37 changed files with 2,659 additions and 3,356 deletions.
diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   cleanup-non-db:
     name: Delete non-DB resources associated with pr
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash
@@ -40,7 +40,7 @@ jobs:
   cleanup-db:
     name: Delete DB resources associated with PR
     needs: cleanup-non-db
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash

diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml
@@ -8,7 +8,7 @@ on:
 jobs:
   prepare-dev-database:
     name: Prepare Dev Database
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash
@@ -18,6 +18,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: Deploy PostGIS instance
         shell: bash
         run: |
@@ -29,7 +34,7 @@ jobs:
     # it here increases the odds that it get's started 1st, so api and web are slightly more likely
     # to finish building at the same time.
     name: Build Web Image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash
@@ -39,6 +44,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: Build wps-web Image
         shell: bash
         run: |
@@ -47,7 +57,7 @@ jobs:
 
   build-api-image:
     name: Build API Image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash
@@ -57,6 +67,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: Build wps-api Image
         shell: bash
         run: |
@@ -83,7 +98,7 @@ jobs:
 
   configure-nats-server-name:
     name: Configure nats server name
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash
@@ -93,6 +108,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: Configure
         shell: bash
         run: |
@@ -110,7 +130,7 @@ jobs:
         deploy-dev-queue,
         configure-nats-server-name,
       ]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Set Variables
         shell: bash
@@ -120,6 +140,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: Deploy API to Dev
         shell: bash
         run: |
@@ -219,7 +244,7 @@ jobs:
   deploy-dev-queue:
     name: Deploy Message Queue to Dev
     if: github.triggering_actor != 'renovate'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     # We need
     # - the image to be built before we can deploy.
     # - we need the tileserv database up so we can write to it
@@ -233,6 +258,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: NATS Message Queue
         shell: bash
         run: |
@@ -242,7 +272,7 @@ jobs:
   scan-dev:
     name: ZAP Baseline Scan Dev
     needs: [deploy-dev]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       # f.y.i.: ZAP Scan must be able to log an issue or it will fail.
@@ -257,7 +287,7 @@ jobs:
   run-schemathesis:
     name: Schemathesis Fuzzing
     if: github.triggering_actor != 'renovate'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [deploy-dev]
 
     steps:
@@ -274,7 +304,7 @@ jobs:
   deploy-c-haines:
     name: Deploy c-haines cronjob
     if: github.triggering_actor != 'renovate'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     # We need
     # - the image to be built before we can deploy.
     # - the database to be there (so we can write to it).
@@ -289,6 +319,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install OpenShift CLI tools
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14"
+
       - name: C-Haines Cronjob
         shell: bash
         run: |

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -7,10 +7,10 @@ on:
 jobs:
   lint-api:
     name: Python - Lint
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python-version: [3.10.4]
+        python-version: [3.12.3]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -19,7 +19,7 @@ jobs:
         # cffdrs requires libudunits2-dev
         run: |
           sudo apt-get update
-          sudo apt-get -y install libgdal-dev libudunits2-dev
+          sudo apt-get -y install libudunits2-dev r-base-dev libgdal-dev libproj-dev libgeos-dev libsqlite3-dev libtirpc-dev
       - name: Setup Python ${{ matrix.python-version }} (api)
         uses: actions/setup-python@v5
         with:
@@ -31,7 +31,7 @@ jobs:
           cache-name: cache-poetry-installer
         with:
           path: "~/poetry_installer"
-          key: "poetry-installer-1.6.1"
+          key: "poetry-installer-1.8.3"
       - name: Download poetry installer
         if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
         run: |
@@ -41,7 +41,7 @@ jobs:
       - name: Install poetry (api)
         run: |
           cd ~/poetry_installer
-          python install-poetry.py --version 1.6.1
+          python install-poetry.py --version 1.8.3
           poetry config virtualenvs.create true
           poetry config virtualenvs.in-project false
       - name: Cache poetry
@@ -50,22 +50,23 @@ jobs:
           cache-name: cache-poetry
         with:
           path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-poetry-1.6.1-cache-${{ hashFiles('**/poetry.lock') }}
+          key: ${{ runner.os }}-poetry-1.8.3-cache-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
-            ${{ runner.os }}-poetry-1.6.1-cache-
+            ${{ runner.os }}-poetry-1.8.3-cache-
       - name: Cache .venv
         id: cache-venv
         uses: actions/cache@v4
         with:
           path: ~/work/wps/wps/api/.venv
-          key: ${{ runner.os }}-venv-poetry-1.6.1-${{ hashFiles('**/poetry.lock') }}
+          key: ${{ runner.os }}-venv-poetry-1.8.3-${{ hashFiles('**/poetry.lock') }}
       - name: Install python dependencies using poetry (api)
         if: steps.cache-venv.outputs.cache-hit != 'true'
         working-directory: ./api
         run: |
           poetry run python -m pip install --upgrade pip
           poetry install
-          poetry run python -m pip install gdal==$(gdal-config --version)
+          poetry run python -m pip install -U setuptools wheel
+          poetry run python -m pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==$(gdal-config --version)
       - name: Lint (api)
         # We used to be able to do linting before installing gdal, but it's not possible anymore.
         # We can however place it ahead of the R installs.
@@ -75,11 +76,11 @@ jobs:
 
   test-api:
     name: Python - Test with coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        # Match versions for python + R to Ubuntu 22.04 LTS at the time of writing.
-        python-version: [3.10.4]
+        # Match versions for python to Ubuntu 24.04 LTS at the time of writing. Keep R at same version.
+        python-version: [3.12.3]
         R: ["4.1.2"]
     steps:
       - name: Checkout repo
@@ -93,7 +94,7 @@ jobs:
         # The api uses wkhtmltopdf to generate pdf's.
         run: |
           sudo apt-get update
-          sudo apt-get -y install libgdal-dev wkhtmltopdf libudunits2-dev
+          sudo apt-get -y install wkhtmltopdf libudunits2-dev r-base-dev build-essential libgdal-dev libproj-dev libgeos-dev libsqlite3-dev libtirpc-dev
       - name: Setup Python ${{ matrix.python-version }} (api)
         uses: actions/setup-python@v5
         with:
@@ -105,7 +106,7 @@ jobs:
           cache-name: cache-poetry-installer
         with:
           path: "~/poetry_installer"
-          key: "poetry-installer-1.6.1"
+          key: "poetry-installer-1.8.3"
       - name: Download poetry installer
         if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
         run: |
@@ -115,7 +116,7 @@ jobs:
       - name: Install poetry (api)
         run: |
           cd ~/poetry_installer
-          python install-poetry.py --version 1.6.1
+          python install-poetry.py --version 1.8.3
           poetry config virtualenvs.create true
           poetry config virtualenvs.in-project false
       # poetry cache folder: /home/runner/.cache/pypoetry
@@ -125,22 +126,23 @@ jobs:
           cache-name: cache-poetry
         with:
           path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-poetry-1.6.1-cache-${{ hashFiles('**/poetry.lock') }}
+          key: ${{ runner.os }}-poetry-1.8.3-cache-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
-            ${{ runner.os }}-poetry-1.6.1-cache-
+            ${{ runner.os }}-poetry-1.8.3-cache-
       - name: Cache .venv
         id: cache-venv
         uses: actions/cache@v4
         with:
           path: ~/work/wps/wps/api/.venv
-          key: ${{ runner.os }}-venv-poetry-1.6.1-${{ hashFiles('**/poetry.lock') }}
+          key: ${{ runner.os }}-venv-poetry-1.8.3-${{ hashFiles('**/poetry.lock') }}
       - name: Install python dependencies using poetry (api)
         if: steps.cache-venv.outputs.cache-hit != 'true'
         working-directory: ./api
         run: |
           poetry run python -m pip install --upgrade pip
           poetry install
-          poetry run python -m pip install gdal==$(gdal-config --version)
+          poetry run python -m pip install -U setuptools wheel
+          poetry run python -m pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==$(gdal-config --version)
       - uses: r-lib/actions/setup-r@v2
         # r-lib/actions/setup-r@v2 is supposed to install version ${{ matrix.R }}, BUT, it doesn't.
         # When asking for 4.1.2 - it's installing a new version! Or at least reporting a newer
@@ -179,7 +181,7 @@ jobs:
           path: ./api/coverage-reports/coverage-report.xml
   lint-and-test-web:
     name: Web - Lint, Test with coverage
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         node-version: [20.x]
@@ -226,7 +228,7 @@ jobs:
           path: ./web/finalCoverage
   upload-code-coverage:
     name: Coverage with CodeCov
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [lint-and-test-web, test-api]
     steps:
       # we need to checkout, so that we have codecov.yml
@@ -244,10 +246,10 @@ jobs:
     name: Backup Prune - Lint and Test
     # We have to use an older ubuntu version, because of the older version of python that the
     # backup script uses.
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python-version: [3.11.2]
+        python-version: [3.12.3]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -265,7 +267,7 @@ jobs:
           cache-name: cache-poetry-installer
         with:
           path: "~/poetry_installer"
-          key: "poetry-installer-1.7.1"
+          key: "poetry-installer-1.8.3"
       - name: Download poetry installer
         if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
         run: |
@@ -275,7 +277,7 @@ jobs:
       - name: Install poetry (api)
         run: |
           cd ~/poetry_installer
-          python install-poetry.py --version 1.7.1
+          python install-poetry.py --version 1.8.3
           poetry config virtualenvs.create true
           poetry config virtualenvs.in-project false
       # poetry cache folder: /home/runner/.cache/pypoetry
@@ -294,7 +296,7 @@ jobs:
           poetry install
       - name: Lint (s3-backup)
         working-directory: ./openshift/s3-backup/docker
-        run: poetry run pylint *.py
+        run: poetry run ruff check *.py
       - name: Unit tests (s3-backup)
         working-directory: ./openshift/s3-backup/docker
         run: |