From a04268c250c0501aa345f19ca3d78dbde47c5e1c Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:16:46 -0600 Subject: [PATCH 01/24] ci: refactor actions Co-authored-by: Scott Davis --- .github/actions/deploy/action.yml | 115 +++++++++++++ .github/workflows/pull_request.yml | 39 +++++ .github/workflows/push.yml | 252 +++-------------------------- .github/workflows/release.yml | 53 ++++++ src/wmrc/version.py | 2 +- 5 files changed, 234 insertions(+), 227 deletions(-) create mode 100644 .github/actions/deploy/action.yml create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml new file mode 100644 index 0000000..e9614c0 --- /dev/null +++ b/.github/actions/deploy/action.yml @@ -0,0 +1,115 @@ +name: Deploy to GCP +description: Deploy to GCP +inputs: + project_id: + description: "The GCP project ID" + required: true + identity_provider: + description: "The identity provider for the workload identity" + required: true + service_account_email: + description: "The service account email" + required: true + storage_bucket: + description: "The GCP storage bucket" + required: true + +runs: + using: composite + steps: + - name: Set globals + id: globals + shell: bash + run: | + echo "MAIN_SCHEDULE_NAME=wmrc_main" >> "${GITHUB_OUTPUT}" + echo "MAIN_SCHEDULE_CRON=0 22 * * 6 >> "${GITHUB_OUTPUT}" + echo "MAIN_SCHEDULE_DESCRIPTION=Trigger the wmrc-skid bot every saturday evening at 10pm" >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_NAME=validator" >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_DESCRIPTION=Trigger the wmrc validation bot every 1st of April, May, and June at 8am" >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_CRON=0 8 1 4-6 * >> "${GITHUB_OUTPUT}" + + - name: โฌ‡๏ธ Set up code + uses: actions/checkout@v4 + with: + show-progress: false + + - name: ๐Ÿ—๏ธ Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2 + with: + create_credentials_file: true + token_format: access_token + workload_identity_provider: ${{ inputs.identity_provider }} + service_account: ${{ inputs.service_account_email }} + + - name: ๐Ÿš€ Deploy Main Cloud Function + id: deploy + uses: google-github-actions/deploy-cloud-functions@v3 + with: + name: wmrc-skid + runtime: python311 + entry_point: subscribe + source_dir: src/wmrc + service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com + event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished + event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ env.SCHEDULE_NAME }}-topic + memory: 1024M + service_timeout: 9m + environment_variables: STORAGE_BUCKET=${{ inputs.storage_bucket}} + secrets: | + /secrets/app/secrets.json=${{ inputs.project_id }}/skid-secrets + max_instance_count: 1 + event_trigger_retry: false + + - name: ๐Ÿ“ฅ Create Main PubSub topic + shell: bash + run: | + if [ ! "$(gcloud pubsub topics list | grep ${{ steps.globals.outputs.SCHEDULE_NAME }}-topic)" ]; then + gcloud pubsub topics create ${{ steps.globals.outputs.SCHEDULE_NAME }}-topic --quiet + fi + + - name: ๐Ÿ•ฐ๏ธ Create Main Cloud Scheduler + shell: bash + run: | + if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }})" ]; then + gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ + --description="${{ steps.globals.outputs.MAIN_CHEDULE_DESCRIPTION" }}" + --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic="${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}"-topic \ + --message-body='facility updates' \ + --quiet + else + gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ + --description="${{ steps.globals.outputs.MAIN_CHEDULE_DESCRIPTION" }}" + --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic="${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}"-topic \ + --message-body='facility updates' \ + --quiet + fi + + - name: ๐Ÿ•ฐ๏ธ Create Validator Cloud Scheduler + shell: bash + run: | + if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }})" ]; then + gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }} \ + --description="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_DESCRIPTION }}" \ + --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic=${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}-topic \ + --message-body='validate' \ + --quiet + else + gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }} \ + --description="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_DESCRIPTION }}" \ + --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ + --time-zone=America/Denver \ + --location=us-central1 \ + --topic=${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}-topic \ + --message-body='validate' \ + --quiet + fi diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..d2324e9 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,39 @@ +name: Pull Request Events + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + test-unit: + name: Unit Tests + runs-on: ubuntu-latest + + steps: + - name: โฌ‡๏ธ Set up code + uses: actions/checkout@v4 + with: + show-progress: false + + - name: ๐Ÿ Set up Python + uses: actions/setup-python@v5 + with: + cache: pip + cache-dependency-path: setup.py + + - name: ๐Ÿ“ฅ Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libkrb5-dev + + - name: ๐Ÿ— Install module + run: pip install .[tests] + + - name: ๐Ÿงถ Lint + run: ruff check --output-format=github . + + - name: ๐Ÿงช Run pytest + run: pytest diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 141c9bd..85c6d37 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -2,62 +2,38 @@ name: Push Events on: push: - branches: - - main - - dev - pull_request: - branches: - - main - - dev - -env: - CLOUD_FUNCTION_MEMORY: 1024M - CLOUD_FUNCTION_RUN_TIMEOUT: 9m - SCHEDULE_NAME: saturday-evening - SCHEDULE_CRON: 0 22 * * 6 - SCHEDULE_DESCRIPTION: Trigger the wmrc-skid bot every saturday evening at 10pm - VALIDATOR_SCHEDULE_NAME: wmrc-validator - VALIDATOR_SCHEDULE_CRON: 0 8 1 4-6 * - VALIDATOR_SCHEDULE_DESCRIPTION: Trigger the wmrc validation bot every 1st of April, May, and June at 8am concurrency: - group: "${{ github.head_ref || github.ref }}" + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - test: - name: Setup and Test + release-please: + name: Create release + if: github.ref_name == 'main' runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - show-progress: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - cache: pip - cache-dependency-path: setup.py - - - name: Install libkrb5 for Kerberos on Linux - run: | - sudo apt-get update - sudo apt install -y libkrb5-dev - - - name: Install module - run: pip install .[tests] - - - name: Test with pytest - run: pytest + - name: ๐Ÿš€ Create Release + id: release-please + uses: agrc/release-composite-action@v1 + with: + release-type: python + prerelease: ${{ github.ref_name == 'dev' }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + github-app-id: ${{ secrets.UGRC_RELEASE_BOT_APP_ID }} + github-app-key: ${{ secrets.UGRC_RELEASE_BOT_APP_KEY }} + github-app-name: ${{ secrets.UGRC_RELEASE_BOT_NAME }} + github-app-email: ${{ secrets.UGRC_RELEASE_BOT_EMAIL }} + extra-files: src/wmrc/version.py deploy-dev: name: Deploy to GCF - dev - needs: test runs-on: ubuntu-latest - if: github.ref == 'refs/heads/dev' + if: github.ref_name == 'dev' environment: name: dev permissions: @@ -70,187 +46,11 @@ jobs: with: show-progress: false - - name: ๐Ÿ—๏ธ Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v2 - with: - create_credentials_file: true - token_format: access_token - workload_identity_provider: ${{ secrets.IDENTITY_PROVIDER }} - service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} - - - name: ๐Ÿš€ Deploy Main Cloud Function - id: deploy - uses: google-github-actions/deploy-cloud-functions@v3 + - name: Deploy + uses: ./.github/actions/deploy timeout-minutes: 15 with: - name: wmrc-skid - runtime: python311 - entry_point: subscribe - source_dir: src/wmrc - service_account: cloud-function-sa@${{ secrets.PROJECT_ID }}.iam.gserviceaccount.com - event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished - event_trigger_pubsub_topic: projects/${{ secrets.PROJECT_ID }}/topics/${{ env.SCHEDULE_NAME }}-topic - memory: ${{ env.CLOUD_FUNCTION_MEMORY }} - service_timeout: ${{ env.CLOUD_FUNCTION_RUN_TIMEOUT }} - environment_variables: STORAGE_BUCKET=${{secrets.STORAGE_BUCKET}} - secrets: | - /secrets/app/secrets.json=${{secrets.PROJECT_ID}}/skid-secrets - max_instance_count: 1 - event_trigger_retry: false - - - name: ๐Ÿ“ฅ Create Main PubSub topic - run: | - if [ ! "$(gcloud pubsub topics list | grep $SCHEDULE_NAME-topic)" ]; then - gcloud pubsub topics create $SCHEDULE_NAME-topic --quiet - fi - - - name: ๐Ÿ•ฐ๏ธ Create Main Cloud Scheduler - run: | - if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep $SCHEDULE_NAME)" ]; then - gcloud scheduler jobs create pubsub $SCHEDULE_NAME \ - --description="$SCHEDULE_DESCRIPTION" \ - --schedule="$SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='facility updates' \ - --quiet - else - gcloud scheduler jobs update pubsub $SCHEDULE_NAME \ - --description="$SCHEDULE_DESCRIPTION" \ - --schedule="$SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='facility updates' \ - --quiet - fi - - - name: ๐Ÿ•ฐ๏ธ Create Validator Cloud Scheduler - run: | - if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep $VALIDATOR_SCHEDULE_NAME)" ]; then - gcloud scheduler jobs create pubsub $VALIDATOR_SCHEDULE_NAME \ - --description="$VALIDATOR_SCHEDULE_DESCRIPTION" \ - --schedule="$VALIDATOR_SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='validate' \ - --quiet - else - gcloud scheduler jobs update pubsub $VALIDATOR_SCHEDULE_NAME \ - --description="$VALIDATOR_SCHEDULE_DESCRIPTION" \ - --schedule="$VALIDATOR_SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='validate' \ - --quiet - fi - - deploy-prod: - name: Deploy to GCF - prod - needs: test - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - environment: - name: prod - permissions: - id-token: write - contents: read - - steps: - - name: โฌ‡๏ธ Set up code - uses: actions/checkout@v4 - with: - show-progress: false - - - name: ๐Ÿ—๏ธ Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v2 - with: - create_credentials_file: true - token_format: access_token - workload_identity_provider: ${{ secrets.IDENTITY_PROVIDER }} - service_account: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} - - - name: ๐Ÿš€ Deploy Main Cloud Function - id: deploy - uses: google-github-actions/deploy-cloud-functions@v3 - timeout-minutes: 15 - with: - name: wmrc-skid - runtime: python311 - entry_point: subscribe - source_dir: src/wmrc - service_account: cloud-function-sa@${{ secrets.PROJECT_ID }}.iam.gserviceaccount.com - event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished - event_trigger_pubsub_topic: projects/${{ secrets.PROJECT_ID }}/topics/${{ env.SCHEDULE_NAME }}-topic - memory: ${{ env.CLOUD_FUNCTION_MEMORY }} - service_timeout: ${{ env.CLOUD_FUNCTION_RUN_TIMEOUT }} - environment_variables: STORAGE_BUCKET=${{secrets.STORAGE_BUCKET}} - secrets: | - /secrets/app/secrets.json=${{secrets.PROJECT_ID}}/skid-secrets - max_instance_count: 1 - event_trigger_retry: false - - - name: ๐Ÿ“ฅ Create Main PubSub topic - run: | - if [ ! "$(gcloud pubsub topics list | grep $SCHEDULE_NAME-topic)" ]; then - gcloud pubsub topics create $SCHEDULE_NAME-topic --quiet - fi - - - name: ๐Ÿ•ฐ๏ธ Create Main Cloud Scheduler - run: | - if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep $SCHEDULE_NAME)" ]; then - gcloud scheduler jobs create pubsub $SCHEDULE_NAME \ - --description="$SCHEDULE_DESCRIPTION" \ - --schedule="$SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='facility updates' \ - --quiet - else - gcloud scheduler jobs update pubsub $SCHEDULE_NAME \ - --description="$SCHEDULE_DESCRIPTION" \ - --schedule="$SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='facility updates' \ - --quiet - fi - - - name: ๐Ÿ•ฐ๏ธ Create Validator Cloud Scheduler - run: | - if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep $VALIDATOR_SCHEDULE_NAME)" ]; then - gcloud scheduler jobs create pubsub $VALIDATOR_SCHEDULE_NAME \ - --description="$VALIDATOR_SCHEDULE_DESCRIPTION" \ - --schedule="$VALIDATOR_SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='validate' \ - --quiet - else - gcloud scheduler jobs update pubsub $VALIDATOR_SCHEDULE_NAME \ - --description="$VALIDATOR_SCHEDULE_DESCRIPTION" \ - --schedule="$VALIDATOR_SCHEDULE_CRON" \ - --time-zone=America/Denver \ - --location=us-central1 \ - --topic=$SCHEDULE_NAME-topic \ - --message-body='validate' \ - --quiet - fi - - - name: ๐Ÿ”” Create deployment notification - uses: agrc/service-now-worknote-action@v1 - with: - repo-token: ${{ github.token }} - username: ${{ secrets.SN_USERNAME }} - password: ${{ secrets.SN_PASSWORD }} - instance-name: ${{ secrets.SN_INSTANCE }} - table-name: ${{ secrets.SN_TABLE }} - system-id: ${{ secrets.SN_SYS_ID }} + project_id: ${{ secrets.PROJECT_ID }} + identity_provider: ${{ secrets.IDENTITY_PROVIDER }} + service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + storage_bucket: ${{ secrets.STORAGE_BUCKET }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..959b25d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,53 @@ +name: Release Events +on: + release: + types: [published] + +jobs: + deploy-prod: + name: Deploy to GCF - prod + runs-on: ubuntu-latest + environment: + name: prod + permissions: + id-token: write + contents: read + + steps: + - name: โฌ‡๏ธ Set up code + uses: actions/checkout@v4 + with: + show-progress: false + + - name: Deploy + uses: ./.github/actions/deploy + timeout-minutes: 15 + with: + project_id: ${{ secrets.PROJECT_ID }} + identity_provider: ${{ secrets.IDENTITY_PROVIDER }} + service_account_email: ${{ secrets.SERVICE_ACCOUNT_EMAIL }} + storage_bucket: ${{ secrets.STORAGE_BUCKET }} + + - name: ๐Ÿ”” Create deployment notification + uses: agrc/service-now-worknote-action@v1 + with: + repo-token: ${{ github.token }} + username: ${{ secrets.SN_USERNAME }} + password: ${{ secrets.SN_PASSWORD }} + instance-name: ${{ secrets.SN_INSTANCE }} + table-name: ${{ secrets.SN_TABLE }} + system-id: ${{ secrets.SN_SYS_ID }} + + notify: + name: Notifications + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + + steps: + - name: Release Notifier + uses: agrc/release-issue-notifications-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/wmrc/version.py b/src/wmrc/version.py index 7d01ed7..26a62ab 100644 --- a/src/wmrc/version.py +++ b/src/wmrc/version.py @@ -1,6 +1,6 @@ """A single source of truth for the version in a programatically-accessible variable. -This must only include a single line: __version__ = 'x.y.z' +This must only include a single line setting the dunder version variable. """ __version__ = "1.2.0" From 655b79cecd5c6a32087d1614f3919dcd355e30e6 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:19:32 -0600 Subject: [PATCH 02/24] ci: weird name liveshare bug --- .github/actions/deploy/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index e9614c0..1813748 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -73,7 +73,7 @@ runs: run: | if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }})" ]; then gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ - --description="${{ steps.globals.outputs.MAIN_CHEDULE_DESCRIPTION" }}" + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION" }}" --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ @@ -82,7 +82,7 @@ runs: --quiet else gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ - --description="${{ steps.globals.outputs.MAIN_CHEDULE_DESCRIPTION" }}" + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION" }}" --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ From 54a027993deea530d2e59a2564a03eb01fef8dce Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:27:46 -0600 Subject: [PATCH 03/24] ci: fix topic name --- .github/actions/deploy/action.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 1813748..7031868 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -27,6 +27,7 @@ runs: echo "VALIDATOR_SCHEDULE_NAME=validator" >> "${GITHUB_OUTPUT}" echo "VALIDATOR_SCHEDULE_DESCRIPTION=Trigger the wmrc validation bot every 1st of April, May, and June at 8am" >> "${GITHUB_OUTPUT}" echo "VALIDATOR_SCHEDULE_CRON=0 8 1 4-6 * >> "${GITHUB_OUTPUT}" + echo "TOPIC_NAME=wmrc-topic" >> "${GITHUB_OUTPUT}" - name: โฌ‡๏ธ Set up code uses: actions/checkout@v4 @@ -52,10 +53,10 @@ runs: source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished - event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ env.SCHEDULE_NAME }}-topic + event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }}-topic memory: 1024M service_timeout: 9m - environment_variables: STORAGE_BUCKET=${{ inputs.storage_bucket}} + environment_variables: STORAGE_BUCKET=${{ inputs.storage_bucket }} secrets: | /secrets/app/secrets.json=${{ inputs.project_id }}/skid-secrets max_instance_count: 1 @@ -64,8 +65,8 @@ runs: - name: ๐Ÿ“ฅ Create Main PubSub topic shell: bash run: | - if [ ! "$(gcloud pubsub topics list | grep ${{ steps.globals.outputs.SCHEDULE_NAME }}-topic)" ]; then - gcloud pubsub topics create ${{ steps.globals.outputs.SCHEDULE_NAME }}-topic --quiet + if [ ! "$(gcloud pubsub topics list | grep ${{ steps.globals.outputs.TOPIC_NAME }})" ]; then + gcloud pubsub topics create ${{ steps.globals.outputs.TOPIC_NAME }}--quiet fi - name: ๐Ÿ•ฐ๏ธ Create Main Cloud Scheduler @@ -77,7 +78,7 @@ runs: --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic="${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}"-topic \ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ --message-body='facility updates' \ --quiet else @@ -86,7 +87,7 @@ runs: --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic="${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}"-topic \ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ --message-body='facility updates' \ --quiet fi @@ -100,7 +101,7 @@ runs: --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic=${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}-topic \ + --topic=${{ steps.globals.outputs.TOPIC_NAME }}\ --message-body='validate' \ --quiet else @@ -109,7 +110,7 @@ runs: --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic=${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}-topic \ + --topic=${{ steps.globals.outputs.TOPIC_NAME }}\ --message-body='validate' \ --quiet fi From 35aa45a9ab4501dd58b095828289a2cfba5eaae5 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:28:58 -0600 Subject: [PATCH 04/24] ci: one more _topic --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 7031868..a6d9126 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -53,7 +53,7 @@ runs: source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished - event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }}-topic + event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M service_timeout: 9m environment_variables: STORAGE_BUCKET=${{ inputs.storage_bucket }} From cc45c7b5439532bf853abbbe50b678eaa4b80c1c Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:34:30 -0600 Subject: [PATCH 05/24] ci: read the actual error message Co-authored-by: Scott Davis --- .github/actions/deploy/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index a6d9126..44047b8 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -74,7 +74,7 @@ runs: run: | if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }})" ]; then gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ - --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION" }}" + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ @@ -83,7 +83,7 @@ runs: --quiet else gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ - --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION" }}" + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ @@ -101,16 +101,16 @@ runs: --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic=${{ steps.globals.outputs.TOPIC_NAME }}\ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ --message-body='validate' \ --quiet else - gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }} \ + gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}" \ --description="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_DESCRIPTION }}" \ --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic=${{ steps.globals.outputs.TOPIC_NAME }}\ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ --message-body='validate' \ --quiet fi From 4ca48551533752e1ae338f24ed5d422a4c163573 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:39:22 -0600 Subject: [PATCH 06/24] ci: topic should be before deploy --- .github/actions/deploy/action.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 44047b8..21067bf 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -43,6 +43,13 @@ runs: workload_identity_provider: ${{ inputs.identity_provider }} service_account: ${{ inputs.service_account_email }} + - name: ๐Ÿ“ฅ Create Main PubSub topic + shell: bash + run: | + if [ ! "$(gcloud pubsub topics list | grep ${{ steps.globals.outputs.TOPIC_NAME }})" ]; then + gcloud pubsub topics create ${{ steps.globals.outputs.TOPIC_NAME }} --quiet + fi + - name: ๐Ÿš€ Deploy Main Cloud Function id: deploy uses: google-github-actions/deploy-cloud-functions@v3 @@ -62,13 +69,6 @@ runs: max_instance_count: 1 event_trigger_retry: false - - name: ๐Ÿ“ฅ Create Main PubSub topic - shell: bash - run: | - if [ ! "$(gcloud pubsub topics list | grep ${{ steps.globals.outputs.TOPIC_NAME }})" ]; then - gcloud pubsub topics create ${{ steps.globals.outputs.TOPIC_NAME }}--quiet - fi - - name: ๐Ÿ•ฐ๏ธ Create Main Cloud Scheduler shell: bash run: | From 7740ccd3db65afa5f1461d12e652c7dd552e560a Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:45:07 -0600 Subject: [PATCH 07/24] ci: line continuations --- .github/actions/deploy/action.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 21067bf..13a7a18 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -74,20 +74,20 @@ runs: run: | if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }})" ]; then gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ - --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" \ --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ --message-body='facility updates' \ --quiet else gcloud scheduler jobs update pubsub "${{ steps.globals.outputs.MAIN_SCHEDULE_NAME }}" \ - --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" + --description="${{ steps.globals.outputs.MAIN_SCHEDULE_DESCRIPTION }}" \ --schedule="${{ steps.globals.outputs.MAIN_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ --message-body='facility updates' \ --quiet fi @@ -96,12 +96,12 @@ runs: shell: bash run: | if [ ! "$(gcloud scheduler jobs list --location=us-central1 | grep ${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }})" ]; then - gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }} \ + gcloud scheduler jobs create pubsub "${{ steps.globals.outputs.VALIDATOR_SCHEDULE_NAME }}" \ --description="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_DESCRIPTION }}" \ --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ --message-body='validate' \ --quiet else @@ -110,7 +110,7 @@ runs: --schedule="${{ steps.globals.outputs.VALIDATOR_SCHEDULE_CRON }}" \ --time-zone=America/Denver \ --location=us-central1 \ - --topic="${{ steps.globals.outputs.TOPIC_NAME }}"\ + --topic="${{ steps.globals.outputs.TOPIC_NAME }}" \ --message-body='validate' \ --quiet fi From 598f8fa2385419905dc7a7745f67a0967f91e06d Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 12:48:15 -0600 Subject: [PATCH 08/24] ci: missing close quote --- .github/actions/deploy/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 13a7a18..76a6f2f 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -22,11 +22,11 @@ runs: shell: bash run: | echo "MAIN_SCHEDULE_NAME=wmrc_main" >> "${GITHUB_OUTPUT}" - echo "MAIN_SCHEDULE_CRON=0 22 * * 6 >> "${GITHUB_OUTPUT}" + echo "MAIN_SCHEDULE_CRON=0 22 * * 6" >> "${GITHUB_OUTPUT}" echo "MAIN_SCHEDULE_DESCRIPTION=Trigger the wmrc-skid bot every saturday evening at 10pm" >> "${GITHUB_OUTPUT}" echo "VALIDATOR_SCHEDULE_NAME=validator" >> "${GITHUB_OUTPUT}" echo "VALIDATOR_SCHEDULE_DESCRIPTION=Trigger the wmrc validation bot every 1st of April, May, and June at 8am" >> "${GITHUB_OUTPUT}" - echo "VALIDATOR_SCHEDULE_CRON=0 8 1 4-6 * >> "${GITHUB_OUTPUT}" + echo "VALIDATOR_SCHEDULE_CRON=0 8 1 4-6 *" >> "${GITHUB_OUTPUT}" echo "TOPIC_NAME=wmrc-topic" >> "${GITHUB_OUTPUT}" - name: โฌ‡๏ธ Set up code From 98d787c86f709768f37929791e856b4a109047f5 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Fri, 16 Aug 2024 14:46:35 -0600 Subject: [PATCH 09/24] ci: code already checked out by calling action --- .github/actions/deploy/action.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 76a6f2f..8559073 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -29,11 +29,6 @@ runs: echo "VALIDATOR_SCHEDULE_CRON=0 8 1 4-6 *" >> "${GITHUB_OUTPUT}" echo "TOPIC_NAME=wmrc-topic" >> "${GITHUB_OUTPUT}" - - name: โฌ‡๏ธ Set up code - uses: actions/checkout@v4 - with: - show-progress: false - - name: ๐Ÿ—๏ธ Authenticate to Google Cloud id: auth uses: google-github-actions/auth@v2 From 9d94ec75e8880af79da620abbf7cfca69647cc30 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 08:53:56 -0600 Subject: [PATCH 10/24] ci: update service_account --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 8559073..7b3010c 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -53,7 +53,7 @@ runs: runtime: python311 entry_point: subscribe source_dir: src/wmrc - service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com + service_account: ${{ inputs.service_account_email }} event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M From d0c1e439b195dcbf655c40cc00971b8252259147 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 11:12:39 -0600 Subject: [PATCH 11/24] ci: revert service account change --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 7b3010c..8559073 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -53,7 +53,7 @@ runs: runtime: python311 entry_point: subscribe source_dir: src/wmrc - service_account: ${{ inputs.service_account_email }} + service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M From b27f99f13a30acd201efd2063de4792b401da55e Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 11:30:49 -0600 Subject: [PATCH 12/24] chore: add facility classification to validation --- src/wmrc/main.py | 12 +++++++++--- src/wmrc/validate.py | 3 ++- tests/test_validate.py | 12 +++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/wmrc/main.py b/src/wmrc/main.py index 2a06466..aa9d81a 100644 --- a/src/wmrc/main.py +++ b/src/wmrc/main.py @@ -453,9 +453,13 @@ def run_validation(): slice_b = all_changes.columns.slice_indexer("msw_recycling_rate_pct_change", "msw_recycling_rate_diff") index_c = all_changes.columns.get_loc("msw_recycling_rate_diff") + 1 new_index = all_changes.columns[slice_b].append([all_changes.columns[:index_a], all_changes.columns[index_c:]]) + reordered = all_changes.reindex(columns=new_index) + classifications = [val[-1] if isinstance(val, tuple) else None for val in reordered.index] + reordered.insert(0, "classification", classifications) + reordered.index = [val[:-1] if isinstance(val, tuple) else val for val in reordered.index] wmrc_skid.skid_logger.debug("Writing report to csv...") - all_changes.reindex(columns=new_index).to_csv(report_path) + reordered.to_csv(report_path) end = datetime.now() @@ -510,5 +514,7 @@ def subscribe(cloud_event: CloudEvent) -> None: #: Putting this here means you can call the file via `python main.py` and it will run. Useful for pre-GCF testing. if __name__ == "__main__": - wmrc_skid = Skid() - wmrc_skid.process() + # wmrc_skid = Skid() + # wmrc_skid.process() + + run_validation() diff --git a/src/wmrc/validate.py b/src/wmrc/validate.py index 8054c86..ae67926 100644 --- a/src/wmrc/validate.py +++ b/src/wmrc/validate.py @@ -71,6 +71,7 @@ def facility_year_over_year( "facility_id": "id", "Calendar_Year__c": "data_year", "Municipal_Solid_Waste__c": "percent_msw", + "Classifications__c": "classification", } column_renaming.update({col: col.rstrip("__c") for col in all_facility_records.columns if "_County__c" in col}) all_facility_records_renamed = all_facility_records.rename(columns=column_renaming) @@ -86,7 +87,7 @@ def facility_year_over_year( right_index=True, ) .reset_index() - .set_index(["data_year", "id", "name"]) + .set_index(["data_year", "id", "name", "classification"]) ) return _year_over_year_changes(facility_summary_by_year, current_year) diff --git a/tests/test_validate.py b/tests/test_validate.py index 25f830c..6be350f 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -3,7 +3,6 @@ import numpy as np import pandas as pd import pytest - from wmrc import validate @@ -87,6 +86,7 @@ def test_facility_year_over_year_happy_path(self, input_df, expected_output): { "facility_id": ["SW01", "SW03", "SW01", "SW03"], "Calendar_Year__c": [2022, 2022, 2023, 2023], + "Classifications__c": ["Class 1", "Recycling", "Class 1", "Recycling"], "Municipal_Solid_Waste__c": [10, 50, 100, 100], "Cache_County__c": [80, 50, 40, 100], } @@ -109,6 +109,11 @@ def test_facility_year_over_year_happy_path(self, input_df, expected_output): ) expected_output = pd.concat([expected_output, new_output_columns], axis=1) + #: Add the classification level to the index + expected_output.index = pd.MultiIndex.from_tuples( + [("SW01", "foo", "Class 1"), ("SW03", "baz", "Recycling")], names=["id", "name", "classification"] + ) + pd.testing.assert_frame_equal(expected_output, output) def test_facility_year_over_year_switches_record_year_to_int(self, input_df, expected_output): @@ -124,6 +129,7 @@ def test_facility_year_over_year_switches_record_year_to_int(self, input_df, exp { "facility_id": ["SW01", "SW03", "SW01", "SW03"], "Calendar_Year__c": ["2022", "2022", "2023", "2023"], + "Classifications__c": ["Class 1", "Recycling", "Class 1", "Recycling"], "Municipal_Solid_Waste__c": [10, 50, 100, 100], "Cache_County__c": [80, 50, 40, 100], } @@ -145,6 +151,10 @@ def test_facility_year_over_year_switches_record_year_to_int(self, input_df, exp index=pd.MultiIndex.from_tuples([("SW01", "foo"), ("SW03", "baz")], names=["id", "name"]), ) expected_output = pd.concat([expected_output, new_output_columns], axis=1) + #: Add the classification level to the index + expected_output.index = pd.MultiIndex.from_tuples( + [("SW01", "foo", "Class 1"), ("SW03", "baz", "Recycling")], names=["id", "name", "classification"] + ) pd.testing.assert_frame_equal(expected_output, output) From f99fb407357cfc41fa76bae10d9e1c817907729a Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 13:02:48 -0600 Subject: [PATCH 13/24] ci: proper trigger type? --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 8559073..5182406 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -54,7 +54,7 @@ runs: entry_point: subscribe source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com - event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished + event_trigger_type: google.cloud.pubsub.publish event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M service_timeout: 9m From 8992ac2060435ca1c5313440b63205f7ae91da6f Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 13:05:23 -0600 Subject: [PATCH 14/24] ci: user error --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 5182406..06bcd28 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -54,7 +54,7 @@ runs: entry_point: subscribe source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com - event_trigger_type: google.cloud.pubsub.publish + event_trigger_type: google.pubsub.topic.publish event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M service_timeout: 9m From 1c408928964f147c9d36caf5be985966300303ad Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 13:15:50 -0600 Subject: [PATCH 15/24] ci: pub/sub is default? --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 06bcd28..fcd105a 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -54,7 +54,7 @@ runs: entry_point: subscribe source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com - event_trigger_type: google.pubsub.topic.publish + # event_trigger_type: google.pubsub.topic.publish event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M service_timeout: 9m From 7e2dd890655df7e2ec82eacd482072390ee7c783 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 13:54:25 -0600 Subject: [PATCH 16/24] ci: nope, default seems to be http --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index fcd105a..06bcd28 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -54,7 +54,7 @@ runs: entry_point: subscribe source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com - # event_trigger_type: google.pubsub.topic.publish + event_trigger_type: google.pubsub.topic.publish event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M service_timeout: 9m From b660a4855d7f3cc422b09d1378a6dfcbbdff98e1 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Mon, 19 Aug 2024 14:11:42 -0600 Subject: [PATCH 17/24] ci: starting from scratch, copying scott's --- .github/actions/deploy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml index 06bcd28..8559073 100644 --- a/.github/actions/deploy/action.yml +++ b/.github/actions/deploy/action.yml @@ -54,7 +54,7 @@ runs: entry_point: subscribe source_dir: src/wmrc service_account: cloud-function-sa@${{ inputs.project_id }}.iam.gserviceaccount.com - event_trigger_type: google.pubsub.topic.publish + event_trigger_type: google.cloud.pubsub.topic.v1.messagePublished event_trigger_pubsub_topic: projects/${{ inputs.project_id }}/topics/${{ steps.globals.outputs.TOPIC_NAME }} memory: 1024M service_timeout: 9m From e12e0fbedb9a9496a137ebedd7f71b050144fde3 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Tue, 20 Aug 2024 17:44:05 -0600 Subject: [PATCH 18/24] feat: calc total diverted tons for county and state --- src/wmrc/main.py | 6 +++--- src/wmrc/yearly.py | 36 ++++++++++++++---------------------- tests/test_yearly.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/wmrc/main.py b/src/wmrc/main.py index aa9d81a..0767b6f 100644 --- a/src/wmrc/main.py +++ b/src/wmrc/main.py @@ -514,7 +514,7 @@ def subscribe(cloud_event: CloudEvent) -> None: #: Putting this here means you can call the file via `python main.py` and it will run. Useful for pre-GCF testing. if __name__ == "__main__": - # wmrc_skid = Skid() - # wmrc_skid.process() + wmrc_skid = Skid() + wmrc_skid.process() - run_validation() + # run_validation() diff --git a/src/wmrc/yearly.py b/src/wmrc/yearly.py index 012519b..c3d8b1d 100644 --- a/src/wmrc/yearly.py +++ b/src/wmrc/yearly.py @@ -57,18 +57,14 @@ def county_summaries(year_df: pd.DataFrame, county_fields: list[str]) -> pd.Data counties_df["county_wide_msw_composted"] = composted_df.sum() counties_df["county_wide_msw_digested"] = digested_df.sum() counties_df["county_wide_msw_landfilled"] = landfilled_df.sum() + counties_df["county_wide_msw_diverted_total"] = ( + counties_df["county_wide_msw_recycled"] + + counties_df["county_wide_msw_composted"] + + counties_df["county_wide_msw_digested"] + ) counties_df["county_wide_msw_recycling_rate"] = ( - ( - counties_df["county_wide_msw_recycled"] - + counties_df["county_wide_msw_composted"] - + counties_df["county_wide_msw_digested"] - ) - / ( - counties_df["county_wide_msw_recycled"] - + counties_df["county_wide_msw_composted"] - + counties_df["county_wide_msw_digested"] - + counties_df["county_wide_msw_landfilled"] - ) + counties_df["county_wide_msw_diverted_total"] + / (counties_df["county_wide_msw_diverted_total"] + counties_df["county_wide_msw_landfilled"]) * 100 ) @@ -194,18 +190,14 @@ def statewide_metrics(county_year_df: pd.DataFrame) -> pd.DataFrame: statewide_series["statewide_msw_composted"] = in_state_only["county_wide_msw_composted"].sum() statewide_series["statewide_msw_digested"] = in_state_only["county_wide_msw_digested"].sum() statewide_series["statewide_msw_landfilled"] = in_state_only["county_wide_msw_landfilled"].sum() + statewide_series["statewide_msw_diverted_total"] = ( + statewide_series["statewide_msw_recycled"] + + statewide_series["statewide_msw_composted"] + + statewide_series["statewide_msw_digested"] + ) statewide_series["statewide_msw_recycling_rate"] = ( - ( - statewide_series["statewide_msw_recycled"] - + statewide_series["statewide_msw_composted"] - + statewide_series["statewide_msw_digested"] - ) - / ( - statewide_series["statewide_msw_recycled"] - + statewide_series["statewide_msw_composted"] - + statewide_series["statewide_msw_digested"] - + statewide_series["statewide_msw_landfilled"] - ) + statewide_series["statewide_msw_diverted_total"] + / (statewide_series["statewide_msw_diverted_total"] + statewide_series["statewide_msw_landfilled"]) * 100 ) diff --git a/tests/test_yearly.py b/tests/test_yearly.py index 3963955..ff26506 100644 --- a/tests/test_yearly.py +++ b/tests/test_yearly.py @@ -1,10 +1,38 @@ import pandas as pd - from wmrc import yearly class TestYearlyMetrics: + def test_county_wide_metrics_happy_path(self): + facility_year_df = pd.DataFrame( + { + "Municipal_Solid_Waste__c": [50, 50], + "Combined_Total_of_Material_Recycled__c": [10, 20], + "Total_Materials_sent_to_composting__c": [0, 50], + "Total_Material_managed_by_ADC__c": [10, 0], + "Municipal_Waste_In_State_in_Tons__c": [80, 30], + "Cache_County__c": [50, 50], + "Utah_County__c": [50, 50], + } + ) + + expected_output = pd.DataFrame( + { + "county_wide_msw_recycled": [7.5, 7.5], + "county_wide_msw_composted": [12.5, 12.5], + "county_wide_msw_digested": [2.5, 2.5], + "county_wide_msw_landfilled": [55.0, 55.0], + "county_wide_msw_diverted_total": [22.5, 22.5], + "county_wide_msw_recycling_rate": [22.5 / (22.5 + 55.0) * 100, 22.5 / (22.5 + 55.0) * 100], + }, + index=["Cache_County__c", "Utah_County__c"], + ) + + output = yearly.county_summaries(facility_year_df, ["Cache_County__c", "Utah_County__c"]) + + pd.testing.assert_frame_equal(output, expected_output) + def test_statewide_metrics_happy_path(self): input_df = pd.DataFrame( { @@ -12,6 +40,7 @@ def test_statewide_metrics_happy_path(self): "county_wide_msw_composted": [0, 0, 50], "county_wide_msw_digested": [10, 0, 0], "county_wide_msw_landfilled": [80, 90, 30], + "county_wide_msw_diverted_total": [20, 10, 70], }, index=["foo", "bar", "baz"], ) @@ -22,6 +51,7 @@ def test_statewide_metrics_happy_path(self): "statewide_msw_composted": 50, "statewide_msw_digested": 10, "statewide_msw_landfilled": 200, + "statewide_msw_diverted_total": 100, "statewide_msw_recycling_rate": 100 / 300 * 100, } ) From 3ea9dec66a31c0a659abab0d52cc3e934a510017 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Wed, 4 Sep 2024 17:23:10 -0600 Subject: [PATCH 19/24] feat: add recycling totals for recycling facilities --- src/wmrc/config.py | 4 ++-- src/wmrc/main.py | 3 ++- src/wmrc/yearly.py | 6 +++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/wmrc/config.py b/src/wmrc/config.py index 859c6c2..c8ff4ea 100644 --- a/src/wmrc/config.py +++ b/src/wmrc/config.py @@ -33,8 +33,8 @@ COUNTIES_ITEMID = "90431cac2f9f49f4bcf1505419583753" -# FACILITIES_LAYER_ITEMID = "d6736e9525274eadb7461b2fd08c426d" #: Beta version -FACILITIES_LAYER_ITEMID = "056bbc52ff3240f6b69666750a61aeff" #: Live version +FACILITIES_LAYER_ITEMID = "d6736e9525274eadb7461b2fd08c426d" #: Beta version +# FACILITIES_LAYER_ITEMID = "056bbc52ff3240f6b69666750a61aeff" #: Live version JOIN_COLUMN = "id_" COUNTY_LAYER_ITEMID = "ea80c3c34bb147fba462ea100179bf09" diff --git a/src/wmrc/main.py b/src/wmrc/main.py index 0767b6f..e3a0497 100644 --- a/src/wmrc/main.py +++ b/src/wmrc/main.py @@ -265,7 +265,7 @@ def _update_facilities(self, gis: arcgis.gis.GIS, facility_summary_df: pd.DataFr #: Merge facility summaries and google sheet on id_ google_and_sf_data = with_counties_df.merge( - facility_summary_df[["id_", "tons_of_material_diverted_from_"]], + facility_summary_df[["id_", "tons_of_material_diverted_from_", "tons_recycled_at_recycle_fac"]], on="id_", how="left", ) @@ -307,6 +307,7 @@ def _update_facilities(self, gis: arcgis.gis.GIS, facility_summary_df: pd.DataFr "longitude", "tons_of_material_diverted_from_", "gallons_of_used_oil_collected_f", + "tons_recycled_at_recycle_fac", ], ) diff --git a/src/wmrc/yearly.py b/src/wmrc/yearly.py index c3d8b1d..3c1805f 100644 --- a/src/wmrc/yearly.py +++ b/src/wmrc/yearly.py @@ -118,13 +118,17 @@ def facility_tons_diverted_from_landfills(year_df: pd.DataFrame) -> pd.DataFrame + sum_df["Total_WT_for_combustion_in_Tons__c"] ) + #: Include recycling facility recycling totals + sum_df["tons_recycled_at_recycle_fac"] = sum_df["Combined_Total_of_Material_Recycled__c"] + #: Extract just the number part of the facility id, strip leading zeros sum_df["id_"] = sum_df["facility_id"].astype(str).str[3:].str.lstrip("0") #: Replace 0s with NaN for AGOL/Arcade logic (want to identify missing data as such, not as 0s) sum_df["tons_of_material_diverted_from_"] = sum_df["tons_of_material_diverted_from_"].replace(0, np.nan) + sum_df["tons_recycled_at_recycle_fac"] = sum_df["tons_recycled_at_recycle_fac"].replace(0, np.nan) - return sum_df[["Facility_Name__c", "id_", "tons_of_material_diverted_from_"]] + return sum_df[["Facility_Name__c", "id_", "tons_of_material_diverted_from_", "tons_recycled_at_recycle_fac"]] def rates_per_material(year_df: pd.DataFrame, classification: str, fields: list[str], total_field: str) -> pd.DataFrame: From 5ba3011aea2c04a3c2f560257e8d0e97f50469e8 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Wed, 4 Sep 2024 17:32:36 -0600 Subject: [PATCH 20/24] tests: add columns for total diverted --- src/wmrc/config.py | 4 ++-- tests/test_validate.py | 4 ++++ tests/test_yearly.py | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/wmrc/config.py b/src/wmrc/config.py index c8ff4ea..859c6c2 100644 --- a/src/wmrc/config.py +++ b/src/wmrc/config.py @@ -33,8 +33,8 @@ COUNTIES_ITEMID = "90431cac2f9f49f4bcf1505419583753" -FACILITIES_LAYER_ITEMID = "d6736e9525274eadb7461b2fd08c426d" #: Beta version -# FACILITIES_LAYER_ITEMID = "056bbc52ff3240f6b69666750a61aeff" #: Live version +# FACILITIES_LAYER_ITEMID = "d6736e9525274eadb7461b2fd08c426d" #: Beta version +FACILITIES_LAYER_ITEMID = "056bbc52ff3240f6b69666750a61aeff" #: Live version JOIN_COLUMN = "id_" COUNTY_LAYER_ITEMID = "ea80c3c34bb147fba462ea100179bf09" diff --git a/tests/test_validate.py b/tests/test_validate.py index 6be350f..9e20913 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -212,6 +212,10 @@ def test_state_year_over_year_happy_path(self): "statewide_msw_landfilled_2023": [100.0], "statewide_msw_landfilled_2022": [50.0], "statewide_msw_landfilled_diff": [50.0], + "statewide_msw_diverted_total_pct_change": [(64 - 53) / 53 * 100], + "statewide_msw_diverted_total_2023": [64.0], + "statewide_msw_diverted_total_2022": [53.0], + "statewide_msw_diverted_total_diff": [11.0], "statewide_msw_recycling_rate_pct_change": [recycling_rate_diff / recycling_rate_2022 * 100], "statewide_msw_recycling_rate_2023": [recycling_rate_2023], "statewide_msw_recycling_rate_2022": [recycling_rate_2022], diff --git a/tests/test_yearly.py b/tests/test_yearly.py index ff26506..d7a72dd 100644 --- a/tests/test_yearly.py +++ b/tests/test_yearly.py @@ -1,4 +1,5 @@ import pandas as pd + from wmrc import yearly @@ -77,6 +78,7 @@ def test_statewide_metrics_removes_out_of_state_values(self): "statewide_msw_composted": 50, "statewide_msw_digested": 10, "statewide_msw_landfilled": 200, + "statewide_msw_diverted_total": 100, "statewide_msw_recycling_rate": 100 / 300 * 100, } ) From 2f065765f6b4ac8531ade1dcf31145b1cba0285b Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Wed, 11 Sep 2024 14:02:35 -0600 Subject: [PATCH 21/24] chore: rename BFS materials --- src/wmrc/summarize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wmrc/summarize.py b/src/wmrc/summarize.py index f6a8200..559b0d7 100644 --- a/src/wmrc/summarize.py +++ b/src/wmrc/summarize.py @@ -194,6 +194,9 @@ def materials_composted(records: helpers.SalesForceRecords) -> pd.DataFrame: .rename(columns={"Calendar_Year__c": "year_"}) ) materials_composted["year_"] = materials_composted["year_"].apply(helpers.convert_to_int) + materials_composted["material"] = materials_composted["material"].replace( + {"BFS": "Biosolids, Food Processing Residuals, and Sewage Sludge"} + ) return materials_composted From 23defe04c59bb5cd70d440b60e7a91e3f1d0d1f4 Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Tue, 24 Sep 2024 15:47:26 -0600 Subject: [PATCH 22/24] ci: set python version in tests --- .github/workflows/pull_request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d2324e9..0cc51bb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,6 +21,7 @@ jobs: - name: ๐Ÿ Set up Python uses: actions/setup-python@v5 with: + python-version: 3.11 cache: pip cache-dependency-path: setup.py From 85eb7d5b0c6b9e105910ac817900ab35323fc9aa Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Wed, 25 Sep 2024 12:52:55 -0600 Subject: [PATCH 23/24] ci: run on dev, main --- .github/workflows/push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 85c6d37..dc8c856 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -2,6 +2,9 @@ name: Push Events on: push: + branches: + - main + - dev concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 3b0d872bf766a1712b30ea745b69f04099d6567e Mon Sep 17 00:00:00 2001 From: Jake Adams Date: Wed, 25 Sep 2024 13:36:37 -0600 Subject: [PATCH 24/24] ci: report test coverage --- .github/workflows/pull_request.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0cc51bb..acd01be 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -38,3 +38,9 @@ jobs: - name: ๐Ÿงช Run pytest run: pytest + + - name: โฌ†๏ธ Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./cov.xml