Skip to content

Add a fetcher for AWS S3 Argo GDAC data #1158

Add a fetcher for AWS S3 Argo GDAC data

Add a fetcher for AWS S3 Argo GDAC data #1158

Workflow file for this run

name: CI tests
# CI tests ran every time new code is pushed to the repo
on:
push:
branches:
- master
pull_request:
branches-ignore:
- 'dependabot*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
id-token: write
contents: read
jobs:
detect-ci-skip:
name: detect ci skip
runs-on: ubuntu-latest
if: |
github.repository == 'euroargodev/argopy'
&& (github.event_name == 'push' || github.event_name == 'pull_request')
outputs:
skipped: ${{ steps.detect-skip.outputs.trigger-found }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- uses: xarray-contrib/ci-trigger@v1.2
id: detect-skip
with:
keyword: "[skip-ci]"
core-pinned:
# CI tests for environments with core requirements in pinned versions
# All commits must pass this
name: Core - Pinned - Py${{matrix.python-version}} - ${{ matrix.os }}
needs: detect-ci-skip
runs-on: ${{ matrix.os }}
if: needs.detect-ci-skip.outputs.skipped == 'false'
defaults:
run:
shell: bash -l {0}
continue-on-error: ${{ matrix.experimental }}
# timeout-minutes: 45
strategy:
max-parallel: 12
fail-fast: false
matrix:
python-version: ["3.10", "3.11"]
os: ["ubuntu-latest", "windows-latest", "macos-latest"]
experimental: [false]
steps:
- name: Energy Estimation - Initialize
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
with:
task: start-measurement
company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }}
project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }}
machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS }}
continue-on-error: true
- uses: actions/checkout@v4
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4.0.2
with:
aws-region: us-west-1
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01
- name: Set environment variables
run: |
echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-pinned.yml" >> $GITHUB_ENV
echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
echo "LOG_FILE=argopy-tests-Core-Pinned-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV
- name: Setup Micromamba ${{ matrix.python-version }}
uses: mamba-org/setup-micromamba@v2
with:
micromamba-version: '1.5.10-0'
environment-name: argopy-tests
environment-file: ${{ env.CONDA_ENV_FILE }}
init-shell: bash
cache-environment: true
cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
create-args: >-
python=${{matrix.python-version}}
- name: Install argopy
run: |
python -m pip install --no-deps -e .
- name: Version info
run: |
micromamba info
micromamba list
- name: Energy Estimation - Measure Tests Setup
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
env:
ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }}
with:
task: get-measurement
label: 'setup core-pinned'
continue-on-error: true
- name: Lint with flake8
run: |
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: |
pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing --log-file=${{env.LOG_FILE}} --junitxml=junit.xml -o junit_family=legacy
- name: Energy Estimation - Measure Tests Exec
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
env:
ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }}
with:
task: get-measurement
label: 'pytest core-pinned'
continue-on-error: true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4.6.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./cov.xml
flags: unittests, core, pinned, py${{matrix.python-version}}
name: codecov-github
fail_ci_if_error: false
env_vars: RUNNER_OS,PYTHON_VERSION
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Save tests log as artifact
uses: actions/upload-artifact@v4
with:
name: Argopy-Tests-Core-${{matrix.python-version}}-${{matrix.os}}
path: ${{env.LOG_FILE}}
retention-days: 90
- name: Energy Estimation - Show Results
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
env:
ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }}
with:
task: display-results
send-data: true
continue-on-error: true
all-pinned:
# CI tests for environments with all requirements in pinned versions
# All commits must pass this
name: All - Pinned - Py${{matrix.python-version}} - ${{ matrix.os }}
needs: detect-ci-skip
runs-on: ${{ matrix.os }}
if: needs.detect-ci-skip.outputs.skipped == 'false'
defaults:
run:
shell: bash -l {0}
continue-on-error: ${{ matrix.experimental }}
# timeout-minutes: 45
strategy:
max-parallel: 12
fail-fast: false
matrix:
python-version: ["3.10", "3.11"]
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
experimental: [false]
steps:
- name: Energy Estimation - Initialize
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
with:
task: start-measurement
company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }}
project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }}
machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS }}
continue-on-error: true
- uses: actions/checkout@v4
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4.0.2
with:
aws-region: us-west-1
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01
- name: Set environment variables
run: |
echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-pinned.yml" >> $GITHUB_ENV
echo "PYTHON_VERSION=${{matrix.python-version}}" >> $GITHUB_ENV
echo "LOG_FILE=argopy-tests-All-Pinned-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV
- name: Setup Micromamba ${{ matrix.python-version }}
uses: mamba-org/setup-micromamba@v2
with:
micromamba-version: '1.5.10-0'
environment-name: argopy-tests
environment-file: ${{ env.CONDA_ENV_FILE }}
init-shell: bash
cache-environment: true
cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
create-args: >-
python=${{matrix.python-version}}
- name: Install argopy
run: |
python -m pip install --no-deps -e .
- name: Version info
run: |
micromamba info
micromamba list
- name: Energy Estimation - Measure Tests Setup
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
env:
ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }}
with:
task: get-measurement
label: 'setup all-pinned'
continue-on-error: true
- name: Lint with flake8
run: |
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: |
pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing --log-file=${{env.LOG_FILE}} --junitxml=junit.xml -o junit_family=legacy
- name: Energy Estimation - Measure Tests Exec
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
env:
ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }}
with:
task: get-measurement
label: 'pytest all-pinned'
continue-on-error: true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4.6.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./cov.xml
flags: unittests, all, pinned, py${{matrix.python-version}}
name: codecov-github
fail_ci_if_error: false
env_vars: RUNNER_OS,PYTHON_VERSION
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload tests log to GA
uses: actions/upload-artifact@v4
with:
name: Argopy-Tests-All-${{matrix.python-version}}-${{matrix.os}}
path: ${{env.LOG_FILE}}
retention-days: 90
- name: Energy Estimation - Show Results
if: ${{matrix.os == 'ubuntu-latest'}}
uses: green-coding-solutions/eco-ci-energy-estimation@v4.0-rc3
env:
ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }}
with:
task: display-results
send-data: true
continue-on-error: true