[CR][ENG-5681] Great Big Python Upgrade

.github/actions/gen-report/action.yml

@@ -0,0 +1,15 @@
+name: 'Generate Report'
+description: 'Generate test report from junit xml file'
+runs:
+ using: "composite"
+ steps:
+ - name: Generate Report
+ uses: dorny/test-reporter@v1
+ if: success() || failure() # run this step even if previous step failed
+ with:
+ name: ${{github.job}} REPORT # Name of the check run which will be created
+ path: report.xml # Path to test results
+ reporter: java-junit # Format of test results
+ max-annotations: 49
+ list-tests: failed
+ list-suites: failed

.github/actions/start-build/action.yml

@@ -3,6 +3,27 @@ description: 'Last composite action before tests are run'
 runs:
  using: "composite"
  steps:
+ - id: cache-objects
+ uses: actions/cache@v2
+ with:
+ path: ~/.cache
+ key: reqs_${{ hashFiles('poetry.lock') }}
+ restore-keys: reqs_
+ - uses: ./.github/actions/build-es
+ with:
+ ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
+ - uses: ./.github/actions/build-es6
+ with:
+ ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
+ - name: Set up Python 3.12
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ - name: Install lxml
+ shell: bash
+ run: |
+ sudo apt update
+ sudo apt-get install libxml2-dev libxslt-dev python-dev
  - name: Copy Settings
  shell: bash
  run: |
@@ -13,19 +34,15 @@ runs:
  - name: PIP install
  shell: bash
  run: |
- pip install --upgrade pip
- pip install invoke==0.13.0
- pip install flake8==2.4.0 --force-reinstall --upgrade
- pip install wheel
- invoke wheelhouse --dev --addons
+ # pinned pip to 21.0a and setuptools<58.0.0 as current dependencies do not build on newer versions of pip and setuptools
+ pip install poetry==1.8.0
+ poetry install --no-root --without release
  - name: Other installs
  shell: bash
  run: |
- pip install psycopg2==2.7.3 --no-binary psycopg2
- invoke travis_addon_settings
- invoke requirements --dev --addons
+ # bumped psycopg to match requirements.txt, as otherwise build would fail
+ poetry run python3 -m invoke travis-addon-settings
  pip uninstall uritemplate.py --yes
- pip install uritemplate.py==0.3.0
  # use yarn add --exact to match versions in yarn.lock w/o installing all deps
  yarn add --exact bower@^1.8.8
  yarn add --exact @centerforopenscience/list-of-licenses@^1.1.0

.github/workflows/test-build.yml

@@ -33,10 +33,11 @@ jobs:
  addons:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -49,37 +50,21 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_addons -n 1
+ run: poetry run python3 -m invoke test-travis-addons -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  website:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -92,37 +77,21 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_website -n 1
+ run: poetry run python3 -m invoke test-travis-website -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  api1_and_js:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -135,40 +104,23 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: NVM & yarn install
- run: |
- invoke assets --dev
+ run: poetry run python3 -m invoke assets --dev
  - name: Run test
- run: invoke test_travis_api1_and_js -n 1
+ run: poetry run python3 -m invoke test-travis-api1-and-js -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  api2:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -181,32 +133,17 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_api2 -n 1
+ run: poetry run python3 -m invoke test-travis-api2 -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  api3_and_osf:
  runs-on: ubuntu-20.04
+ permissions:
+ checks: write
  needs: build-cache
  services:
  postgres:
@@ -224,27 +161,9 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_api3_and_osf -n 1
-
+ run: poetry run python3 -m invoke test-travis-api3-and-osf -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report

.pre-commit-config.yaml

@@ -1,21 +1,24 @@
 default_language_version:
- python: python3.6
+ python: python3.12
 repos:
 - repo: https://github.com/asottile/add-trailing-comma
- rev: v0.7.0
+ rev: v3.1.0
  hooks:
  - id: add-trailing-comma
  # TODO: Remove this line. For now, we only format the api/ directory
  files: ^api/
 - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v1.4.0
+ rev: v4.6.0
  hooks:
  - id: double-quote-string-fixer
  - id: trailing-whitespace
  exclude: "^(website/static/vendor/|osf/metadata/definitions/.*\\.xsd)"
- - id: flake8
- additional_dependencies: ["flake8==3.6.0", "flake8-mutable==1.2.0"]
+- repo: https://github.com/pycqa/flake8
+ rev: '7.0.0' # pick a git hash / tag to point to
+ hooks:
+ - id: flake8
+ additional_dependencies: ["flake8-mutable==1.2.0"]
 - repo: https://github.com/pre-commit/mirrors-jshint
- rev: v2.9.6
+ rev: v2.13.6
  hooks:
  - id: jshint

.travis.yml

@@ -3,7 +3,7 @@
 language: python
 
 python:
- - "3.6"
+ - "3.10"
 
 dist: trusty
 
@@ -132,7 +132,7 @@ install:
 
  - travis_retry pip install --upgrade pip
  - travis_retry pip install invoke==0.13.0
- - travis_retry pip install flake8==2.4.0 --force-reinstall --upgrade
+ - travis_retry pip install flake8==7.0.0 --force-reinstall --upgrade
  - travis_retry invoke wheelhouse --dev --addons
 
  - |
@@ -145,7 +145,8 @@ install:
  fi
 
  - travis_retry invoke travis_addon_settings
- - travis_retry pip install psycopg2==2.7.3 --no-binary psycopg2
+ # bumped psycopg to match requirements.txt, as otherwise build would fail
+ - travis_retry pip install psycopg2==2.9.9 --no-binary psycopg2
  - travis_retry invoke requirements --dev --addons
  # Hack to fix package conflict between uritemplate and uritemplate.py (dependency of github3.py)
  - pip uninstall uritemplate.py --yes