chore(palletjack): upgrade to cloudrun job and python 3.11

.github/actions/palletjack/deploy/action.yml

@@ -1,75 +1,126 @@
 name: Deploy Palletjack Skid to GCF
 description: Deploy Palletjack Skid to Google Cloud Function and Cloud Scheduler
 inputs:
-  schedule:
-    description: Cloud Scheduler cron schedule
+  project_id:
+    description: 'The GCP project ID'
     required: true
   identity_provider:
-    description: gcp federation identity provider
+    description: 'The identity provider for the workload identity'
     required: true
   service_account_email:
-    description: gcp service account email
+    description: 'The service account email'
     required: true
-  project_id:
-    description: gcp project id
+  pause_schedule_job:
+    description: 'Pause the scheduler job'
+    required: false
+    default: 'no'
+  cloudsql_instance:
+    description: 'The Cloud SQL instance'
+    required: true
+  github_token:
+    description: 'The GitHub token'
     required: true
 
 runs:
   using: composite
   steps:
+    - name: Set globals
+      id: globals
+      shell: bash
+      run: |
+        echo "TOPIC_NAME=palletjack-topic" >> "${GITHUB_OUTPUT}"
+        echo "CRON=0 6,12,18,0 * * *" >> "${GITHUB_OUTPUT}"
+        echo "JOB_NAME=palletjack" >> "${GITHUB_OUTPUT}"
+        echo "JOB_DESCRIPTION=Trigger palletjack cloud run job" >> "${GITHUB_OUTPUT}"
+
     - name: 🗝️ Authenticate to Google Cloud
       id: auth
       uses: google-github-actions/auth@v2
       with:
-        create_credentials_file: true
         token_format: access_token
         workload_identity_provider: ${{ inputs.identity_provider }}
         service_account: ${{ inputs.service_account_email }}
 
-    - name: 🚀 Deploy to Cloud Function
-      id: deploy
-      uses: google-github-actions/deploy-cloud-functions@v2
+    - name: 🐳 Set up Docker Buildx
+      id: builder
+      uses: docker/setup-buildx-action@v3
+
+    - name: 🗝️ Authenticate Docker to Google Cloud
+      uses: docker/login-action@v3
       with:
-        name: roadkill-skid
-        runtime: python39
-        entry_point: main
-        source_dir: src/palletjack/src/roadkill
-        service_account_email: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com
-        event_trigger_type: providers/cloud.pubsub/eventTypes/topic.publish
-        event_trigger_resource: projects/${{ inputs.project_id }}/topics/palletjack-topic
-        deploy_timeout: 600
-        memory_mb: 4096
-        timeout: 540 # this is the max allowed
-        secret_volumes: |
-          /secrets/app/secrets.json=${{inputs.project_id}}/skid-secrets
-        env_vars: PROJECT_ID=${{ inputs.project_id }}
+        registry: us-central1-docker.pkg.dev
+        username: oauth2accesstoken
+        password: ${{ steps.auth.outputs.access_token }}
 
-    - name: 📥 Create PubSub topic
-      shell: bash
-      run: |
-        if [ ! "$(gcloud pubsub topics list | grep palletjack-topic)" ]; then
-          gcloud pubsub topics create palletjack-topic --quiet
-        fi
+    - name: 🏷️ Extract tags from GitHub
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        github-token: ${{ inputs.github_token }}
+        images: us-central1-docker.pkg.dev/${{ inputs.project_id }}/images/job
+        tags: |
+          type=ref,suffix=-{{sha}},event=branch
+          type=ref,prefix=pr-,suffix=-{{sha}},event=pr
+          type=semver,pattern={{version}}
+          latest
+
+    - name: 📦 Build and push image
+      uses: docker/build-push-action@v6
+      with:
+        builder: ${{ steps.builder.outputs.name }}
+        tags: ${{ steps.meta.outputs.tags }}
+        context: '{{defaultContext}}:src/palletjack'
+        file: ./Dockerfile
+        push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        provenance: false
+
+    - name: 🚀 Deploy Main Cloud Run Job
+      id: deploy
+      uses: google-github-actions/deploy-cloudrun@v2
+      with:
+        job: ${{ steps.globals.outputs.JOB_NAME }}
+        image: us-central1-docker.pkg.dev/${{ inputs.project_id }}/images/job:latest
+        timeout: 20m
+        secrets: |
+          /secrets/app/secrets.json=skid-secrets:latest
+        secrets_update_strategy: overwrite
+        env_vars: |-
+          PROJECT_ID=${{ inputs.project_id }}
+        flags: |
+          --memory=4G
+          --service-account=palletjack-sa@${{ inputs.project_id }}.iam.gserviceaccount.com
+          --max-retries=1
+          --task-timeout=60m
+          --parallelism=1
+          --set-cloudsql-instances=${{ inputs.cloudsql_instance }}
 
-    - name: 🕰️ Create Cloud Scheduler
+    - name: 🕰️ Create Main Cloud Scheduler
       shell: bash
       run: |
-        if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep palletjack)" ]; then
-          gcloud scheduler jobs create pubsub palletjack \
-            --description="Trigger palletjack twice per day" \
-            --schedule="${{ inputs.schedule }}" \
+        if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.JOB_NAME }})" ]; then
+          gcloud scheduler jobs create http "${{ steps.globals.outputs.JOB_NAME }}" \
+            --description="${{ steps.globals.outputs.JOB_DESCRIPTION }}" \
+            --schedule="${{ steps.globals.outputs.CRON }}" \
             --time-zone=America/Denver \
             --location=us-central1 \
-            --topic=palletjack-topic \
-            --message-body='{"run": "now"}' \
+            --uri="https://us-central1-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${{ inputs.project_id }}/jobs/${{ steps.globals.outputs.JOB_NAME }}:run" \
+            --oauth-service-account-email=scheduler-sa@${{ inputs.project_id }}.iam.gserviceaccount.com \
             --quiet
         else
-          gcloud scheduler jobs update pubsub palletjack \
-            --description="Trigger palletjack twice per day" \
-            --schedule="${{ inputs.schedule }}" \
+          gcloud scheduler jobs update http "${{ steps.globals.outputs.JOB_NAME }}" \
+            --description="${{ steps.globals.outputs.JOB_DESCRIPTION }}" \
+            --schedule="${{ steps.globals.outputs.CRON }}" \
             --time-zone=America/Denver \
             --location=us-central1 \
-            --topic=palletjack-topic \
-            --message-body='{"run": "now"}' \
+            --uri="https://us-central1-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${{ inputs.project_id }}/jobs/${{ steps.globals.outputs.JOB_NAME }}:run" \
+            --oauth-service-account-email=scheduler-sa@${{ inputs.project_id }}.iam.gserviceaccount.com \
             --quiet
         fi
+
+    - name: 🙅 Pause Scheduler Job
+      shell: bash
+      if: inputs.pause_schedule_job != 'no'
+      run: |
+        gcloud scheduler jobs pause "${{ steps.globals.outputs.JOB_NAME }}" --location=us-central1 --quiet

.github/workflows/palletjack.yml

@@ -1,4 +1,4 @@
-name: Palletjack - Deploy to GCP Function
+name: Palletjack - Deploy to Cloud Run Job
 
 on:
   push:
@@ -34,15 +34,18 @@ jobs:
       - name: ⬇️ Set up code
         uses: actions/checkout@v4
 
-      - name: Install Dependencies
+      - name: 📥 Download extra dependencies
         run: |
-          sudo apt install -y libkrb5-dev
-          pip install -r src/roadkill/requirements.txt
-          pip install -r src/roadkill/requirements.dev.txt
+          sudo apt-get update
+          sudo apt-get install -y libkrb5-dev
 
-      - name: Test with pytest
+      - name: 🏗 Install module
+        run: pip install .[tests]
+
+      - name: 🧪 Run tests
         run: pytest
 
+
   deploy-dev:
     name: Deploy Staging to GCF
     runs-on: ubuntu-latest
@@ -58,10 +61,12 @@ jobs:
       - name: 🚀 Deploy
         uses: ./.github/actions/palletjack/deploy
         with:
+          project_id: ${{ secrets.PROJECT_ID }}
           identity_provider: ${{ secrets.IDENTITY_PROVIDER }}
           service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }}
-          project_id: ${{ secrets.PROJECT_ID }}
-          schedule: 0 6 * * *
+          pause_schedule_job: 'yes'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          cloudsql_instance: ${{ secrets.CLOUDSQL_INSTANCE }}
 
   deploy-prod:
     name: Deploy Production to GCF
@@ -78,7 +83,8 @@ jobs:
       - name: 🚀 Deploy
         uses: ./.github/actions/palletjack/deploy
         with:
+          project_id: ${{ secrets.PROJECT_ID }}
           identity_provider: ${{ secrets.IDENTITY_PROVIDER }}
           service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }}
-          project_id: ${{ secrets.PROJECT_ID }}
-          schedule: 0 6,12,18,0 * * *
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          cloudsql_instance: ${{ secrets.CLOUDSQL_INSTANCE }}

.vscode/settings.json

@@ -4,6 +4,7 @@
     "api",
     "apos",
     "autolinker",
+    "buildx",
     "Buildx",
     "cloudrun",
     "cloudsql",
@@ -12,6 +13,7 @@
     "dataframes",
     "datetimepicker",
     "DCLT",
+    "dotenv",
     "drivername",
     "Dsplay",
     "dtypes",
@@ -39,6 +41,7 @@
     "iloc",
     "immer",
     "inblow",
+    "instafail",
     "isin",
     "isna",
     "jwks",
@@ -89,6 +92,7 @@
     "tanstack",
     "tappable",
     "UDWR",
+    "ugrc",
     "unsubmitted",
     "Unsubmitted",
     "upsert",

src/palletjack/.gitignore

@@ -2,3 +2,5 @@ roadkill.egg-info
 __pycache__
 secrets.json
 secrets.*.json
+.eggs
+*.egg-info

src/palletjack/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-slim
+
+# Allow statements and log messages to immediately appear in the Knative logs
+ENV PYTHONUNBUFFERED=True
+
+USER root
+RUN useradd -s /bin/bash dummy
+
+# Set the locale
+RUN apt-get update && apt-get install -y locales && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && apt-get install -y gcc && apt-get install -y libkrb5-dev && pip install requests-kerberos
+
+COPY . /app
+WORKDIR /app
+RUN pip install .
+
+USER dummy
+ENTRYPOINT ["process"]

src/palletjack/README.md

@@ -9,5 +9,5 @@ This [palletjack](https://github.com/agrc/palletjack) skid is used to push updat
 
 ## Development
 
-1. `pip install -r src/roadkill/requirements.txt`
-1. `pip install -r src/roadkill/requirements.dev.txt`
+1. `conda create --name roadkill-skid python=3.11`
+1. `pip install -e ".[tests]"`

src/palletjack/setup.cfg

@@ -5,6 +5,6 @@ multi_line_output=5
 testpaths = src
 norecursedirs = .env data maps
 show_capture = True
-minversion = 3.9
+minversion = 3.11
 console_output_style = count
 addopts = --instafail --isort

src/palletjack/setup.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+setup.py
+A module that installs the backup process as a module
+"""
+
+from setuptools import find_packages, setup
+
+setup(
+    name="palletjack",
+    version="1.0.0",
+    license="MIT",
+    description="Roadkill skid",
+    author="UGRC Developers",
+    author_email="ugrc-developers@utah.gov",
+    url="https://github.com/agrc/roadkill-mobile",
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    include_package_data=True,
+    zip_safe=True,
+    classifiers=[
+        # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Topic :: Utilities",
+    ],
+    keywords=["gis"],
+    install_requires=[
+        "ugrc-palletjack==5.*",
+        "requests<2.32",
+        "agrc-supervisor==3.0.*",
+    ],
+    extras_require={
+        "tests": [
+            "pylint-quotes~=0.2",
+            "pylint~=2.11",
+            "pytest-instafail~=0.4",
+            "pytest-isort~=2.0",
+            "pytest-pylint~=0.18",
+            "pytest~=6.0",
+            "black~=22.12",
+            "pandas==2.*",
+        ]
+    },
+    setup_requires=[
+        "pytest-runner",
+    ],
+    entry_points={
+        "console_scripts": [
+            "process = roadkill.main:process",
+        ]
+    },
+)

src/palletjack/src/roadkill/.python-version

@@ -0,0 +1 @@
+3.11