Merge pull request #6518 from pulibrary/6501-cog

Update GDAL command to use built COG output format

.circleci/Dockerfile

@@ -1,4 +1,4 @@
-FROM cimg/ruby:3.1.0-browsers
+FROM cimg/ruby:3.1.6-browsers
 
 RUN sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
   && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
   build:
     working_directory: ~/figgy
     docker:
-      - image: pulibrary/ci-figgy:1.6
+      - image: ghcr.io/pulibrary/figgy:ci
         environment:
           RAILS_ENV: test
           FIGGY_DB_HOST: localhost
@@ -60,7 +60,7 @@ jobs:
   test:
     working_directory: ~/figgy
     docker:
-      - image: pulibrary/ci-figgy:1.6
+      - image: ghcr.io/pulibrary/figgy:ci
         environment:
           RAILS_ENV: test
           FIGGY_DB_HOST: localhost
@@ -137,7 +137,7 @@ jobs:
   coverage_report:
     working_directory: ~/figgy
     docker:
-      - image: pulibrary/ci-figgy:1.6
+      - image: ghcr.io/pulibrary/figgy:ci
     resource_class: large
     steps:
       - attach_workspace:
@@ -154,7 +154,7 @@ jobs:
   rubocop:
     working_directory: ~/figgy
     docker:
-      - image: pulibrary/ci-figgy:1.6
+      - image: ghcr.io/pulibrary/figgy:ci
         environment:
           RAILS_ENV: test
           FIGGY_DB_HOST: localhost

.github/workflows/build-ci-docker.yml

@@ -0,0 +1,69 @@
+name: Create and publish a Docker CI image
+
+on:
+  push:
+    paths:
+      - .circleci/Dockerfile
+      - bin/**
+    branches:
+      - main
+  pull_request:
+    paths:
+      - .circleci/Dockerfile
+      - bin/**
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+        env:
+          DOCKER_METADATA_PR_HEAD_SHA: true
+
+      - name: Build and push Docker image
+        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'push') && github.ref == 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: "${{ github.workspace }}"
+          file: "${{ github.workspace }}/.circleci/Dockerfile"
+          push: true
+          tags: ghcr.io/pulibrary/figgy:ci
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Build and push Docker image
+        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && github.ref != 'refs/heads/main'
+        uses: docker/build-push-action@v6
+        with:
+          context: "${{ github.workspace }}"
+          file: "${{ github.workspace }}/.circleci/Dockerfile"
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/prune-containers.yml

@@ -0,0 +1,19 @@
+name: Prune containers
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * SUN"  # every sunday at midnight
+
+jobs:
+  clean:
+    runs-on: ubuntu-latest
+    name: Delete old images
+    steps:
+      - uses: snok/container-retention-policy@v3.0.0
+        with:
+          account: pulibrary
+          image-names: figgy
+          token: ${{ secrets.GITHUB_TOKEN }}
+          image-tags: "!ci !main"
+          cut-off: 2w

app/derivative_services/geo_derivatives/processors/gdal.rb

@@ -26,7 +26,7 @@ def self.translate(in_path, out_path, _options)
         # @param options [Hash] creation options
         def self.warp(in_path, out_path, options)
           execute "gdalwarp -q -t_srs #{options[:output_srid]} "\
-                  "\"#{in_path}\" #{out_path} -co TILED=YES -co COMPRESS=NONE"
+                  "\"#{in_path}\" #{out_path} -co COMPRESS=NONE"
         end
 
         # Executes a gdal_translate command. Used to compress
@@ -46,13 +46,12 @@ def self.compress(in_path, out_path, options)
         # @param out_path [String] processor output file path
         # @param options [Hash] creation options
         def self.cloud_optimized_geotiff(in_path, out_path, _options)
-          system("gdaladdo -q -r average \"#{in_path}\" 2 4 8 16 32")
-          execute("gdal_translate -q -expand rgb \"#{in_path}\" #{out_path} -ot Byte -co TILED=YES "\
-                    "-a_nodata 256 -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES")
+          execute("gdal_translate -q -expand rgb \"#{in_path}\" #{out_path} -ot Byte -of COG "\
+                    "-a_nodata 256 -co COMPRESS=LZW -co TILING_SCHEME=GoogleMapsCompatible")
         rescue StandardError
           # Try without expanding rgb
-          execute("gdal_translate -q \"#{in_path}\" #{out_path} -ot Byte -co TILED=YES "\
-                    "-a_nodata 256 -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES")
+          execute("gdal_translate -q \"#{in_path}\" #{out_path} -ot Byte -of COG "\
+                    "-a_nodata 256 -co COMPRESS=LZW -co TILING_SCHEME=GoogleMapsCompatible")
         end
 
         # Executes a gdal_rasterize command. Used to rasterize vector

spec/derivative_services/geo_derivatives/processors/base_geo_processor_spec.rb

@@ -73,7 +73,7 @@ def source_path; end
 
     describe "#warp" do
       it "executes a reproject command" do
-        command = "gdalwarp -q -t_srs EPSG:4326 \"files/geo.tif\" output/geo.png -co TILED=YES -co COMPRESS=NONE"
+        command = "gdalwarp -q -t_srs EPSG:4326 \"files/geo.tif\" output/geo.png -co COMPRESS=NONE"
         processor.class.warp(file_name, output_file, options)
         expect(processor.class).to have_received(:execute).with command
       end