CI: Cache GSHHG and DCW data as artifacts to simplify workflows (#8481)

GenericMappingTools · May 31, 2024 · 58b5bea · 58b5bea
1 parent 5e53ea3
commit 58b5bea
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 83 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -71,7 +71,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
-      - name: Setup conda
+      - name: Setup conda (macOS)
         uses: conda-incubator/setup-miniconda@v3
         if: runner.os == 'macOS'
 
@@ -84,16 +84,10 @@ jobs:
         env:
           EXCLUDE_OPTIONAL: ${{ matrix.EXCLUDE_OPTIONAL }}
 
-      - name: Cache GSHHG and DCW data
-        uses: actions/cache@v4
-        id: cache-coastline
-        with:
-          path: ${{ env.COASTLINEDIR }}
-          key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
-
-      - name: Download coastlines
-        run: bash ci/download-coastlines.sh
-        if: steps.cache-coastline.outputs.cache-hit != 'true'
+      - name: Download GSHHG and DCW data
+        run: gh run download -n coastline-cache -D coastline
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Configure GMT
         run: |
@@ -121,20 +115,18 @@ jobs:
           cmake --build .
         if: runner.os == 'Windows'
 
-      - name: Download cached GMT remote data from GitHub Artifacts
-        run: |
-          gh run download -n gmt-cache -D ~/.gmt/static/
-          ls -lRh ~/.gmt/static/
-        env:
-          GH_TOKEN: ${{ github.token }}
-
       - name: Install GMT
         run: |
           cd build
           cmake --build . --target install
           # Add GMT PATH to bin
           echo "${INSTALLDIR}/bin" >> $GITHUB_PATH
 
+      - name: Download cached GMT remote data from GitHub Artifacts
+        run: gh run download -n gmt-cache -D ~/.gmt/static/
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Check a few simple commands
         run: bash ci/simple-gmt-tests.sh
 

diff --git a/.github/workflows/ci-caches.yml b/.github/workflows/ci-caches.yml
@@ -2,7 +2,8 @@
 # Create CI caches for:
 #
 # 1. GMT remote data for building documentation and running tests
-# 2. vcpkg libraries on Windows
+# 2. GMT GSHHG and DCW datasets
+# 3. vcpkg libraries on Windows
 #
 name: GMT CI Caches
 
@@ -105,8 +106,28 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: gmt-cache
-          path: |
-            ~/.gmt/static
+          path: ~/.gmt/static
+
+  coastline_cache:
+    name: Cache GSHHG and DCW datasets
+    runs-on: ubuntu-latest
+    env:
+      COASTLINEDIR: ${{ github.workspace }}/coastline
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.4
+
+      - name: Download GSHHG and DCW datasets
+        run: bash ci/download-coastlines.sh
+        env:
+          COASTLINEDIR: ${{ env.COASTLINEDIR }}
+
+      - name: Upload artifacts to GitHub
+        uses: actions/upload-artifact@v4
+        with:
+          name: coastline-cache
+          path: ${{ env.COASTLINEDIR }}
 
   vcpkg_cache:
     name: Cache vcpkg libraries

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -67,9 +67,6 @@ jobs:
           - fedora:rawhide  # rolling release with latest versions
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4.1.4
-
       - name: Install GMT dependencies
         run: |
           os=$(cat /etc/os-release | grep "^ID=" | awk -F= '{print $2}')
@@ -83,24 +80,29 @@ jobs:
                  ghostscript curl git
             apt reinstall -y ca-certificates
             update-ca-certificates
+            # Install gh from binary deb package
+            curl -LO https://github.com/cli/cli/releases/download/v2.49.0/gh_2.49.0_linux_amd64.deb
+            apt install ./gh_2.49.0_linux_amd64.deb
           elif [[ "$os" = "fedora" ]]; then
             dnf install -y \
                 cmake ninja-build \
                 libcurl-devel netcdf-devel gdal-devel gdal \
                 fftw3-devel pcre-devel lapack-devel openblas-devel glib2-devel \
-                ghostscript openssl
+                ghostscript openssl gh
           fi
 
-      - name: Cache GSHHG and DCW data
-        uses: actions/cache@v4
-        id: cache-coastline
-        with:
-          path: ${{ env.COASTLINEDIR }}
-          key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
+      - name: Checkout
+        uses: actions/checkout@v4.1.4
+
+      # Fix the git error:
+      # "failed to run git: fatal: detected dubious ownership in repository at '/__w/gmt/gmt'"
+      - name: Fixup git
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
-      - name: Download coastlines
-        run: bash ci/download-coastlines.sh
-        if: steps.cache-coastline.outputs.cache-hit != 'true'
+      - name: Download GSHHG and DCW data
+        run: gh run download -n coastline-cache -D coastline
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Configure GMT
         run: bash ci/config-gmt-unix.sh
@@ -116,18 +118,9 @@ jobs:
           echo "${INSTALLDIR}/bin" >> $GITHUB_PATH
 
       - name: Download cached GMT remote data from GitHub Artifacts
-        uses: dawidd6/action-download-artifact@v3.1.4
-        with:
-          workflow: ci-caches.yml
-          name: gmt-cache
-          path: gmt-cache
-
-      # Move downloaded files to ~/.gmt directory and list them
-      - name: Move and list downloaded remote files
-        run: |
-          mkdir -p ~/.gmt/static/
-          mv gmt-cache/* ~/.gmt/static/
-          ls -lRh ~/.gmt/static/
+        run: gh run download -n gmt-cache -D ~/.gmt/static/
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Check a few simple commands
         run: bash ci/simple-gmt-tests.sh
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -65,7 +65,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
 
-      - name: Setup conda
+      - name: Setup conda (macOS)
         uses: conda-incubator/setup-miniconda@v3
         if: runner.os == 'macOS'
 
@@ -76,19 +76,10 @@ jobs:
           os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
           bash ci/install-dependencies-${os}.sh
 
-      - name: Cache GSHHG and DCW data
-        uses: actions/cache@v4
-        id: cache-coastline
-        with:
-          path: ${{ env.COASTLINEDIR }}
-          key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
-
-      - name: Download coastlines
-        run: bash ci/download-coastlines.sh
-        if: steps.cache-coastline.outputs.cache-hit != 'true'
-
-      - name: Pull baseline image data from dvc remote
-        run: dvc pull --no-run-cache
+      - name: Download GSHHG and DCW data
+        run: gh run download -n coastline-cache -D coastline
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Configure GMT
         run: |
@@ -116,12 +107,8 @@ jobs:
           cmake --build .
         if: runner.os == 'Windows'
 
-      - name: Download cached GMT remote data from GitHub Artifacts
-        run: |
-          gh run download -n gmt-cache -D ~/.gmt/static/
-          ls -lRh ~/.gmt/static/
-        env:
-          GH_TOKEN: ${{ github.token }}
+      - name: Pull baseline image data from dvc remote
+        run: dvc pull --no-run-cache
 
       - name: Build documentation
         run: |

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -90,16 +90,10 @@ jobs:
           New-ItemProperty -Path "HKLM:\Software\GPL Ghostscript\10.02" -Name GS_LIB -PropertyType String -Value "C:\Miniconda\Library\bin;C:\Miniconda\Library\lib;C\Miniconda\Library\Font;C:\Miniconda\Library\fonts"
         if: runner.os == 'Windows'
 
-      - name: Cache GSHHG and DCW data
-        uses: actions/cache@v4
-        id: cache-coastline
-        with:
-          path: ${{ env.COASTLINEDIR }}
-          key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
-
-      - name: Download coastlines
-        run: bash ci/download-coastlines.sh
-        if: steps.cache-coastline.outputs.cache-hit != 'true'
+      - name: Download GSHHG and DCW data
+        run: gh run download -n coastline-cache -D coastline
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Configure GMT
         run: |
@@ -127,13 +121,6 @@ jobs:
           cmake --build .
         if: runner.os == 'Windows'
 
-      - name: Download cached GMT remote data from GitHub Artifacts
-        run: |
-          gh run download -n gmt-cache -D ~/.gmt/static/
-          ls -lRh ~/.gmt/static/
-        env:
-          GH_TOKEN: ${{ github.token }}
-
       - name: Pull baseline image data from dvc remote
         run: dvc pull --no-run-cache
 
@@ -144,6 +131,11 @@ jobs:
           # Add GMT PATH to bin
           echo "${INSTALLDIR}/bin" >> $GITHUB_PATH
 
+      - name: Download cached GMT remote data from GitHub Artifacts
+        run: gh run download -n gmt-cache -D ~/.gmt/static/
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Check a few simple commands
         run: bash ci/simple-gmt-tests.sh