chore(palletjack): switch to docker

.github/actions/palletjack/deploy/action.yml

@@ -14,6 +14,9 @@ inputs:
     description: 'Pause the scheduler job'
     required: false
     default: 'no'
+  github_token:
+    description: 'The GitHub token'
+    required: true
 
 runs:
   using: composite
@@ -35,12 +38,47 @@ runs:
         workload_identity_provider: ${{ inputs.identity_provider }}
         service_account: ${{ inputs.service_account_email }}
 
+    - name: 🐳 Set up Docker Buildx
+      id: builder
+      uses: docker/setup-buildx-action@v3
+
+    - name: 🗝️ Authenticate Docker to Google Cloud
+      uses: docker/login-action@v3
+      with:
+        registry: us-central1-docker.pkg.dev
+        username: oauth2accesstoken
+        password: ${{ steps.auth.outputs.access_token }}
+
+    - name: 🏷️ Extract tags from GitHub
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        github-token: ${{ inputs.github_token }}
+        images: us-central1-docker.pkg.dev/${{ inputs.project_id }}/images/job
+        tags: |
+          type=ref,suffix=-{{sha}},event=branch
+          type=ref,prefix=pr-,suffix=-{{sha}},event=pr
+          type=semver,pattern={{version}}
+          latest
+
+    - name: 📦 Build and push image
+      uses: docker/build-push-action@v6
+      with:
+        builder: ${{ steps.builder.outputs.name }}
+        tags: ${{ steps.meta.outputs.tags }}
+        context: '{{defaultContext}}:jobs'
+        file: ./Dockerfile
+        push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        provenance: false
+
     - name: 🚀 Deploy Main Cloud Run Job
       id: deploy
       uses: google-github-actions/deploy-cloudrun@v2
       with:
         job: ${{ steps.globals.outputs.JOB_NAME }}
-        source: src/palletjack/src/roadkill
+        image: us-central1-docker.pkg.dev/${{ inputs.project_id }}/images/job:latest
         timeout: 20m
         secrets: |
           /secrets/app/secrets.json=skid-secrets:latest

.github/workflows/palletjack.yml

@@ -65,6 +65,7 @@ jobs:
           identity_provider: ${{ secrets.IDENTITY_PROVIDER }}
           service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }}
           pause_schedule_job: 'yes'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
   deploy-prod:
     name: Deploy Production to GCF
@@ -84,3 +85,4 @@ jobs:
           project_id: ${{ secrets.PROJECT_ID }}
           identity_provider: ${{ secrets.IDENTITY_PROVIDER }}
           service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.vscode/settings.json

@@ -4,6 +4,7 @@
     "api",
     "apos",
     "autolinker",
+    "buildx",
     "Buildx",
     "cloudrun",
     "cloudsql",
@@ -12,6 +13,7 @@
     "dataframes",
     "datetimepicker",
     "DCLT",
+    "dotenv",
     "drivername",
     "Dsplay",
     "dtypes",
@@ -39,6 +41,7 @@
     "iloc",
     "immer",
     "inblow",
+    "instafail",
     "isin",
     "isna",
     "jwks",
@@ -89,6 +92,7 @@
     "tanstack",
     "tappable",
     "UDWR",
+    "ugrc",
     "unsubmitted",
     "Unsubmitted",
     "upsert",

src/palletjack/.gitignore

@@ -2,3 +2,5 @@ roadkill.egg-info
 __pycache__
 secrets.json
 secrets.*.json
+.eggs
+*.egg-info

src/palletjack/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.11-slim
+
+# Allow statements and log messages to immediately appear in the Knative logs
+ENV PYTHONUNBUFFERED=True
+
+USER root
+RUN useradd -s /bin/bash dummy
+
+# Set the locale
+RUN apt-get update && apt-get install -y locales && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && apt-get install -y gcc && apt-get install -y libkrb5-dev && pip install requests-kerberos
+
+COPY . /app
+WORKDIR /app
+RUN pip install .
+
+USER dummy
+ENTRYPOINT ["process"]

src/palletjack/README.md

@@ -9,5 +9,5 @@ This [palletjack](https://github.com/agrc/palletjack) skid is used to push updat
 
 ## Development
 
-1. `pip install -r src/roadkill/requirements.txt`
-1. `pip install -r src/roadkill/requirements.dev.txt`
+1. `conda create --name roadkill-skid python=3.11`
+1. `pip install -e ".[tests]"`

src/palletjack/setup.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+"""
+setup.py
+A module that installs the backup process as a module
+"""
+
+from setuptools import find_packages, setup
+
+setup(
+    name="palletjack",
+    version="1.0.0",
+    license="MIT",
+    description="Roadkill skid",
+    author="UGRC Developers",
+    author_email="ugrc-developers@utah.gov",
+    url="https://github.com/agrc/roadkill-mobile",
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    include_package_data=True,
+    zip_safe=True,
+    classifiers=[
+        # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Topic :: Utilities",
+    ],
+    keywords=["gis"],
+    install_requires=[
+        "ugrc-palletjack==5.*",
+        "agrc-supervisor==3.*",
+    ],
+    extras_require={
+        "tests": [
+            "pylint-quotes~=0.2",
+            "pylint~=2.11",
+            "pytest-instafail~=0.4",
+            "pytest-isort~=2.0",
+            "pytest-pylint~=0.18",
+            "pytest~=6.0",
+            "black~=22.12",
+            "pandas==2.*",
+        ]
+    },
+    setup_requires=[
+        "pytest-runner",
+    ],
+    entry_points={
+        "console_scripts": [
+            "process = roadkill.main:process",
+        ]
+    },
+)

src/palletjack/src/roadkill/main.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # * coding: utf8 *
 """
-Run the roadkill skid script as a cloud function.
+Run the roadkill skid
 """
 import datetime
 import json
@@ -283,36 +283,6 @@ def process():
         _remove_log_file_handlers(log_name, loggers)
 
 
-def main(event, context):  # pylint: disable=unused-argument
-    """Entry point for Google Cloud Function triggered by pub/sub event
-
-    Args:
-         event (dict):  The dictionary with data specific to this type of
-                        event. The `@type` field maps to
-                         `type.googleapis.com/google.pubsub.v1.PubsubMessage`.
-                        The `data` field maps to the PubsubMessage data
-                        in a base64-encoded string. The `attributes` field maps
-                        to the PubsubMessage attributes if any is present.
-         context (google.cloud.functions.Context): Metadata of triggering event
-                        including `event_id` which maps to the PubsubMessage
-                        messageId, `timestamp` which maps to the PubsubMessage
-                        publishTime, `event_type` which maps to
-                        `google.pubsub.topic.publish`, and `resource` which is
-                        a dictionary that describes the service API endpoint
-                        pubsub.googleapis.com, the triggering topic's name, and
-                        the triggering event type
-                        `type.googleapis.com/google.pubsub.v1.PubsubMessage`.
-    Returns:
-        None. The output is written to Cloud Logging.
-    """
-
-    #: This function must be called 'main' to act as the Google Cloud Function entry point. It must accept the two
-    #: arguments listed, but doesn't have to do anything with them (I haven't used them in anything yet).
-
-    #: Call process() and any other functions you want to be run as part of the skid here.
-    process()
-
-
 #: Putting this here means you can call the file via `python main.py` and it will run. Useful for pre-GCF testing.
 if __name__ == "__main__":
     process()