Peskas Timor Data Pipeline #2098
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Peskas Timor Data Pipeline | |
| on: | |
| push: | |
| schedule: | |
| - cron: '0 0 */2 * *' | |
| env: | |
| KOBO_PASSWORD: ${{ secrets.KOBO_PASSWORD }} | |
| KOBO_USERNAME: ${{ secrets.KOBO_USERNAME }} | |
| KOBO_PESKAS1: ${{ secrets.KOBO_PESKAS1 }} | |
| KOBO_PESKAS2: ${{ secrets.KOBO_PESKAS2 }} | |
| KOBO_PESKAS3: ${{ secrets.KOBO_PESKAS3 }} | |
| GCP_SA_KEY: ${{ secrets.PESKAS_DATAINGESTION_GCS_KEY }} | |
| AIRTABLE_KEY: ${{ secrets.AIRTABLE_KEY }} | |
| PDS_TOKEN: ${{ secrets.PESKAS_PDS_TOKEN }} | |
| PDS_SECRET: ${{ secrets.PESKAS_PDS_SECRET }} | |
| VALID_SHEET_ID: ${{ secrets.VALID_SHEET_ID }} | |
| GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} | |
| jobs: | |
| build-container: | |
| name: Build R container | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Build image with cache | |
| id: build-docker | |
| uses: whoan/docker-build-with-cache-action@v5 | |
| with: | |
| username: $GITHUB_ACTOR | |
| password: "${{ secrets.GITHUB_TOKEN }}" | |
| registry: docker.pkg.github.com | |
| image_name: r-runner-peskas-timor | |
| dockerfile: Dockerfile.prod | |
| ingest-preprocess-metadata-tables: | |
| name: Ingest metadata tables | |
| needs: build-container | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call ingest_metadata_tables() | |
| run: Rscript -e 'peskas.timor.data.pipeline::ingest_metadata_tables()' | |
| - name: Call preprocess_metadata_tables() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_metadata_tables()' | |
| - name: Call ingest_rfish_table() | |
| continue-on-error: true | |
| run: Rscript -e 'peskas.timor.data.pipeline::ingest_rfish_table()' | |
| ingest-preprocess-landings-v1-v3: | |
| name: Ingest and preprocess legacy and ongoing landings | |
| needs: build-container | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call ingest_landings() | |
| run: Rscript -e 'peskas.timor.data.pipeline::ingest_landings_v1v3()' | |
| - name: Call preprocess_legacy_landings() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_legacy_landings()' | |
| - name: Call preprocess_updated_landings() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_updated_landings()' | |
| ingest-preprocess-v2-landings-step1: | |
| name: "Ingest-process landings 1/2" | |
| needs: build-container | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call ingest_landings_v2() | |
| run: Rscript -e 'peskas.timor.data.pipeline::ingest_landings_v2()' | |
| - name: Call preprocess_landings_step_1() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_landings_step_1()' | |
| ingest-preprocess-v2-landings-step2: | |
| name: "Ingest-process landings 2/2" | |
| needs: ingest-preprocess-v2-landings-step1 | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call preprocess_landings_step_2() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_landings_step_2()' | |
| merge-landings: | |
| name: Merge landings | |
| needs: [ingest-preprocess-v2-landings-step2, ingest-preprocess-landings-v1-v3, ingest-preprocess-metadata-tables] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call merge_landings() | |
| run: Rscript -e 'peskas.timor.data.pipeline::merge_landings()' | |
| - name: Call calculate_weights() | |
| run: Rscript -e 'peskas.timor.data.pipeline::calculate_weights()' | |
| validate-landings: | |
| name: Validate landings | |
| needs: [merge-landings] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Validate landings | |
| run: Rscript -e 'peskas.timor.data.pipeline::validate_landings()' | |
| - name: Test validated landings | |
| run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_validated_landings.R", package = "peskas.timor.data.pipeline"))' | |
| ingest-pds-data: | |
| name: Ingest pds data | |
| needs: build-container | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call ingest_pds_trips() | |
| run: Rscript -e 'peskas.timor.data.pipeline::ingest_pds_trips()' | |
| - name: Call ingest_pds_tracks() | |
| run: Rscript -e 'peskas.timor.data.pipeline::ingest_pds_tracks()' | |
| preprocess-pds-data: | |
| name: Preprocess pds data | |
| needs: [ingest-pds-data] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call preprocess_pds_trips() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_pds_trips()' | |
| - name: Call preprocess_pds_tracks() | |
| run: Rscript -e 'peskas.timor.data.pipeline::preprocess_pds_tracks()' | |
| validate-pds-data: | |
| name: Validate pds data | |
| needs: preprocess-pds-data | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call validate_pds_trips() | |
| run: Rscript -e 'peskas.timor.data.pipeline::validate_pds_trips()' | |
| - name: Test validated PDS trips | |
| run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_validated_pds_trips.R", package = "peskas.timor.data.pipeline"))' | |
| merge-trips: | |
| name: Merge trips | |
| needs: [validate-landings, validate-pds-data] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call merge_trips() | |
| run: Rscript -e 'peskas.timor.data.pipeline::merge_trips()' | |
| - name: Test merged_trips | |
| run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_merged_trips.R", package = "peskas.timor.data.pipeline"))' | |
| model-indicators: | |
| name: Estimate fishery indicators | |
| needs: [merge-trips] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call estimate_fishery_indicators() | |
| run: Rscript -e 'peskas.timor.data.pipeline::estimate_fishery_indicators()' | |
| export-trips: | |
| name: Export trips | |
| needs: [merge-trips, model-indicators] | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call format_public_data() | |
| run: Rscript -e 'peskas.timor.data.pipeline::format_public_data()' | |
| - name: Test public data | |
| run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_public_data.R", package = "peskas.timor.data.pipeline"))' | |
| - name: Generate and upload enumerators report | |
| run: Rscript -e 'source(system.file("report/generate_enumerators_report.R", package = "peskas.timor.data.pipeline"))' |