Merge branch 'develop' of https://github.com/CenterForOpenScience/osf.io

 into remove-quickfiles-code

* 'develop' of https://github.com/CenterForOpenScience/osf.io: (166 commits)
  update dataverse dep revision to get changes
  Update CHANGELOG, bump version
  [CR][ENG-5681] Great Big Python Upgrade (CenterForOpenScience#10648)
  Revert "[ENG-3685] Add permissions for withdrawn registration files (CenterForOpenScience#10650)" (CenterForOpenScience#10666)
  Check Registration READ perms on the Registration - Do not record download metrics for renders
  Fix signature
  Allow DOI metadata updates to be queued
  [ENG-3685] Add permissions for withdrawn registration files (CenterForOpenScience#10650)
  Update CHANGELOG, bump version
  [ENG-5030] Preprints Phase 2 - BE (CenterForOpenScience#10617)
  Update CHANGELOG, bump version
  Ensure Assumed-HAM users do not get autobanned
  [ENG-5762] Get GV set up in osf docker configs (CenterForOpenScience#10643)
  [ENG-5718] Use `make_auth` to avoid assumptions about `auth.user` (CenterForOpenScience#10647)
  [ENG-5699] Framework for getting Addon Info from GV (CenterForOpenScience#10641)
  [ENG-5178] Allow unauthenticated users to see public files (CenterForOpenScience#10645)
  Fix RelationshipPostMakesNoChanges exception in project creation (CenterForOpenScience#10644)
  [ENG - 5008] Support Unicode and special characters in file names during archiving (CenterForOpenScience#10627)
  Set Default Resource Type for Registrations to "Study Registration" (CenterForOpenScience#10636)
  Configurable GV Mock + HMAC Auth (CenterForOpenScience#10623)
  ...

# Conflicts:
#	addons/base/views.py
#	api/caching/tasks.py
#	api_tests/files/views/test_file_detail.py
#	api_tests/wb/views/test_wb_hooks.py
#	osf/management/commands/data_storage_usage.py
#	osf/management/commands/reindex_quickfiles.py
#	osf/management/commands/transfer_quickfiles_to_projects.py
#	osf/models/__init__.py
#	osf/models/private_link.py
#	osf/models/quickfiles.py
#	osf/models/user.py
#	scripts/fix_merged_user_quickfiles.py
#	tests/test_views.py
#	website/mails/mails.py
#	website/search/elastic_search.py
#	website/settings/defaults.py

.bowerrc

@@ -1,3 +1,4 @@
 {
- "directory": "website/static/vendor/bower_components"
+ "directory": "website/static/vendor/bower_components",
+ "registry": "https://bower.herokuapp.com"
 }

.docker-compose.env

@@ -9,6 +9,7 @@ ELASTIC_URI=192.168.168.167:9200
 ELASTIC6_URI=192.168.168.167:9201
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
+REDIS_HOST=redis://192.168.168.167:6379
 WATERBUTLER_URL=http://localhost:7777
 WATERBUTLER_INTERNAL_URL=http://192.168.168.167:7777
 CAS_SERVER_URL=http://192.168.168.167:8080

.docker-compose.gv.env

@@ -0,0 +1,11 @@
+ALLOWED_HOSTS="localhost,192.168.168.167"
+CORS_ALLOWED_ORIGINS="http://localhost:5000,http://192.168.168.167:5000"
+OSFDB_HOST=192.168.168.167
+POSTGRES_HOST="$OSFDB_HOST"
+OSF_BASE_URL="http://192.168.168.167:5000"
+OSF_API_BASE_URL="http://192.168.168.167:8000"
+POSTGRES_USER="postgres"
+POSTGRES_DB="gravyvalet"
+SECRET_KEY="secret"
+PYTHONUNBUFFERED=0 # This when set to 0 will allow print statements to be visible in the Docker logs
+

.github/actions/gen-report/action.yml

@@ -0,0 +1,15 @@
+name: 'Generate Report'
+description: 'Generate test report from junit xml file'
+runs:
+ using: "composite"
+ steps:
+ - name: Generate Report
+ uses: dorny/test-reporter@v1
+ if: success() || failure() # run this step even if previous step failed
+ with:
+ name: ${{github.job}} REPORT # Name of the check run which will be created
+ path: report.xml # Path to test results
+ reporter: java-junit # Format of test results
+ max-annotations: 49
+ list-tests: failed
+ list-suites: failed

.github/actions/start-build/action.yml

@@ -3,6 +3,27 @@ description: 'Last composite action before tests are run'
 runs:
  using: "composite"
  steps:
+ - id: cache-objects
+ uses: actions/cache@v2
+ with:
+ path: ~/.cache
+ key: reqs_${{ hashFiles('poetry.lock') }}
+ restore-keys: reqs_
+ - uses: ./.github/actions/build-es
+ with:
+ ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
+ - uses: ./.github/actions/build-es6
+ with:
+ ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
+ - name: Set up Python 3.12
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ - name: Install lxml
+ shell: bash
+ run: |
+ sudo apt update
+ sudo apt-get install libxml2-dev libxslt-dev python-dev
  - name: Copy Settings
  shell: bash
  run: |
@@ -13,19 +34,15 @@ runs:
  - name: PIP install
  shell: bash
  run: |
- pip install --upgrade pip
- pip install invoke==0.13.0
- pip install flake8==2.4.0 --force-reinstall --upgrade
- pip install wheel
- invoke wheelhouse --dev --addons
+ # pinned pip to 21.0a and setuptools<58.0.0 as current dependencies do not build on newer versions of pip and setuptools
+ pip install poetry==1.8.0
+ poetry install --no-root --without release
  - name: Other installs
  shell: bash
  run: |
- pip install psycopg2==2.7.3 --no-binary psycopg2
- invoke travis_addon_settings
- invoke requirements --dev --addons
+ # bumped psycopg to match requirements.txt, as otherwise build would fail
+ poetry run python3 -m invoke travis-addon-settings
  pip uninstall uritemplate.py --yes
- pip install uritemplate.py==0.3.0
  # use yarn add --exact to match versions in yarn.lock w/o installing all deps
  yarn add --exact bower@^1.8.8
  yarn add --exact @centerforopenscience/list-of-licenses@^1.1.0

.github/workflows/test-build.yml

@@ -33,10 +33,11 @@ jobs:
  addons:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -49,37 +50,21 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_addons -n 1
+ run: poetry run python3 -m invoke test-travis-addons -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  website:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -92,37 +77,21 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_website -n 1
+ run: poetry run python3 -m invoke test-travis-website -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  api1_and_js:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -135,40 +104,23 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: NVM & yarn install
- run: |
- invoke assets --dev
+ run: poetry run python3 -m invoke assets --dev
  - name: Run test
- run: invoke test_travis_api1_and_js -n 1
+ run: poetry run python3 -m invoke test-travis-api1-and-js -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  api2:
  runs-on: ubuntu-20.04
  needs: build-cache
+ permissions:
+ checks: write
  services:
  postgres:
  image: postgres
-
  env:
  POSTGRES_PASSWORD: postgres
  options: >-
@@ -181,32 +133,17 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_api2 -n 1
+ run: poetry run python3 -m invoke test-travis-api2 -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report
 
  api3_and_osf:
  runs-on: ubuntu-20.04
+ permissions:
+ checks: write
  needs: build-cache
  services:
  postgres:
@@ -224,27 +161,9 @@ jobs:
  - 5432:5432
  steps:
  - uses: actions/checkout@v2
- - id: cache-objects
- uses: actions/cache@v2
- with:
- path: ~/.cache
- key: reqs_${{ hashFiles('**/requirements.txt') }}
- restore-keys: reqs_
- - uses: ./.github/actions/build-es
- with:
- ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
- - uses: ./.github/actions/build-es6
- with:
- ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
- - name: Set up Python 3.6
- uses: actions/setup-python@v2
- with:
- python-version: 3.6
- - name: Install lxml
- run: |
- sudo apt update
- sudo apt-get install libxml2-dev libxslt-dev python-dev
  - uses: ./.github/actions/start-build
  - name: Run tests
- run: invoke test_travis_api3_and_osf -n 1
-
+ run: poetry run python3 -m invoke test-travis-api3-and-osf -n 1 --junit
+ - name: Upload report
+ if: (github.event_name != 'pull_request') && (success() || failure()) # run this step even if previous step failed
+ uses: ./.github/actions/gen-report

.gitignore

@@ -203,4 +203,3 @@ ssl/
 
 # pyenv
 .python-version
-

.pre-commit-config.yaml

@@ -1,21 +1,24 @@
 default_language_version:
- python: python3.6
+ python: python3.12
 repos:
 - repo: https://github.com/asottile/add-trailing-comma
- rev: v0.7.0
+ rev: v3.1.0
  hooks:
  - id: add-trailing-comma
  # TODO: Remove this line. For now, we only format the api/ directory
  files: ^api/
 - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v1.4.0
+ rev: v4.6.0
  hooks:
  - id: double-quote-string-fixer
  - id: trailing-whitespace
- exclude: website/static/vendor/*
- - id: flake8
- additional_dependencies: ["flake8==3.6.0", "flake8-mutable==1.2.0"]
+ exclude: "^(website/static/vendor/|osf/metadata/definitions/.*\\.xsd)"
+- repo: https://github.com/pycqa/flake8
+ rev: '7.0.0' # pick a git hash / tag to point to
+ hooks:
+ - id: flake8
+ additional_dependencies: ["flake8-mutable==1.2.0"]
 - repo: https://github.com/pre-commit/mirrors-jshint
- rev: v2.9.6
+ rev: v2.13.6
  hooks:
  - id: jshint

.travis.yml

@@ -3,7 +3,7 @@
 language: python
 
 python:
- - "3.6"
+ - "3.10"
 
 dist: trusty
 
@@ -132,7 +132,7 @@ install:
 
  - travis_retry pip install --upgrade pip
  - travis_retry pip install invoke==0.13.0
- - travis_retry pip install flake8==2.4.0 --force-reinstall --upgrade
+ - travis_retry pip install flake8==7.0.0 --force-reinstall --upgrade
  - travis_retry invoke wheelhouse --dev --addons
 
  - |
@@ -145,7 +145,8 @@ install:
  fi
 
  - travis_retry invoke travis_addon_settings
- - travis_retry pip install psycopg2==2.7.3 --no-binary psycopg2
+ # bumped psycopg to match requirements.txt, as otherwise build would fail
+ - travis_retry pip install psycopg2==2.9.9 --no-binary psycopg2
  - travis_retry invoke requirements --dev --addons
  # Hack to fix package conflict between uritemplate and uritemplate.py (dependency of github3.py)
  - pip uninstall uritemplate.py --yes