[CR][ENG-5681] Great Big Python Upgrade (CenterForOpenScience#10648)

- Bump base python version from py3.6 to py3.12.
- Switch to using poetry for dependency management
- Bump most (not all) dependencies to their maximum version as of mid-March.
- Significantly update Dockerfile
- Upgrade Django to v4.2
- Generate test summary reports in CI

---------

Co-authored-by: Oleh Paduchak <opaduchak@exoft.net>
Co-authored-by: Mariia Lychko <lychko.mariia@gmail.com>
Co-authored-by: Longze Chen <cslzchen@gmail.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-alpine3.17 AS base
+FROM python:3.12-alpine3.17 as base
 
 # Creation of www-data group was removed as it is created by default in alpine 3.14 and higher
 # Alpine does not create a www-data user, so we still need to create that. 82 is the standard
@@ -30,7 +30,7 @@ ENV POETRY_NO_INTERACTION=1 \
  POETRY_VIRTUALENVS_OPTIONS_ALWAYS_COPY=1 \
  POETRY_VIRTUALENVS_CREATE=0
 
-FROM base AS build
+FROM base as build
 
 ENV POETRY_VIRTUALENVS_IN_PROJECT=1 \
  YARN_CACHE_FOLDER=/tmp/yarn-cache \
@@ -149,7 +149,7 @@ RUN for module in \
  ; done \
  && rm ./website/settings/local.py ./api/base/settings/local.py
 
-FROM base AS runtime
+FROM base as runtime
 
 WORKDIR /code
 COPY --from=build /usr/local/lib/python3.12 /usr/local/lib/python3.12

addons/dataverse/requirements.txt

@@ -1,3 +1,3 @@
 # Allow for optional timeout parameter.
 # https://github.com/IQSS/dataverse-client-python/pull/27
-git+https://github.com/CenterForOpenScience/dataverse-client-python.git@2b3827578048e6df3818f82381c7ea9a2395e526 # branch is feature/dv-client-updates
+git+https://github.com/CenterForOpenScience/dataverse-client-python.git@feature/dv-client-updates

addons/mendeley/requirements.txt

@@ -1,2 +1,2 @@
 # up-to-date with mendeley's master + add folder support and future dep updates
-git+https://github.com/CenterForOpenScience/mendeley-python-sdk.git@be8a811fa6c3b105d9f5c656cabb6b1ba855ed5b # branch is feature/osf-dep-updates
+git+https://github.com/CenterForOpenScience/mendeley-python-sdk.git@feature/osf-dep-updates

admin/base/urls.py

@@ -36,6 +36,7 @@
  re_path(r'^schema_responses/', include('admin.schema_responses.urls', namespace='schema_responses')),
  re_path(r'^registration_schemas/', include('admin.registration_schemas.urls', namespace='registration_schemas')),
  re_path(r'^cedar_metadata_templates/', include('admin.cedar.urls', namespace='cedar_metadata_templates')),
+ re_path(r'^notifications/', include('admin.notifications.urls', namespace='notifications')),
  ]),
  ),
 ]

admin/notifications/urls.py

@@ -0,0 +1,8 @@
+from django.urls import re_path
+from admin.notifications import views
+
+app_name = 'notifications'
+
+urlpatterns = [
+ re_path(r'^$', views.handle_duplicate_notifications, name='handle_duplicate_notifications'),
+]

admin/notifications/views.py

@@ -0,0 +1,32 @@
+from django.contrib.auth.decorators import user_passes_test
+from django.shortcuts import render, redirect
+from admin.base.utils import osf_staff_check
+from osf.models.notifications import NotificationSubscription
+from django.db.models import Count
+
+@user_passes_test(osf_staff_check)
+def handle_duplicate_notifications(request):
+ duplicates = NotificationSubscription.objects.values('user', 'node', 'event_name').annotate(count=Count('id')).filter(count__gt=1)
+
+ detailed_duplicates = []
+ for dup in duplicates:
+ notifications = NotificationSubscription.objects.filter(user=dup['user'], node=dup['node'], event_name=dup['event_name'])
+ for notification in notifications:
+ detailed_duplicates.append({
+ 'id': notification.id,
+ 'user': notification.user,
+ 'node': notification.node,
+ 'event_name': notification.event_name,
+ 'created': notification.created,
+ 'count': dup['count']
+ })
+
+ context = {'duplicates': detailed_duplicates}
+
+ if request.method == 'POST':
+ selected_ids = request.POST.getlist('selected_notifications')
+ NotificationSubscription.objects.filter(id__in=selected_ids).delete()
+ context['message'] = 'Selected duplicate notifications have been deleted.'
+ return redirect('notifications:handle_duplicate_notifications')
+
+ return render(request, 'handle_duplicate_notifications.html', context)

admin/templates/base.html

@@ -311,6 +311,9 @@
  {% if perms.osf.change_maintenancestate %}
  <li><a href="{% url 'maintenance:display' %}"><i class='fa fa-link'></i> <span>Maintenance Alerts</span></a></li>
  {% endif %}
+ {% if perms.osf.view_notification %}
+ <li><a href="{% url 'notifications:handle_duplicate_notifications' %}"><i class='fa fa-link'></i><span>Duplicate Notifications</span> </a></li>
+ {% endif %}
  </ul><!-- /.sidebar-menu -->
  </section>
  <!-- /.sidebar -->

admin/templates/handle_duplicate_notifications.html

@@ -0,0 +1,54 @@
+{% extends "base.html" %}
+{% load render_bundle from webpack_loader %}
+{% load static %}
+
+{% block title %}
+ <title>Duplicate Notifications</title>
+{% endblock title %}
+
+{% block content %}
+ <h2>Duplicate Notifications</h2>
+
+ {% if message %}
+ <div class="alert alert-success">
+ {{ message }}
+ </div>
+ {% endif %}
+
+ {% if duplicates %}
+ <form method="post">
+ {% csrf_token %}
+ <table class="table table-striped table-hover table-responsive">
+ <thead>
+ <tr>
+ <th>Select</th>
+ <th>User</th>
+ <th>Node</th>
+ <th>Event Name</th>
+ <th>Created</th>
+ <th>Count</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for notification in duplicates %}
+ <tr>
+ <td><input type="checkbox" name="selected_notifications" value="{{ notification.id }}"></td>
+ <td>{{ notification.user }}</td>
+ <td>{{ notification.node }}</td>
+ <td>{{ notification.event_name }}</td>
+ <td>{{ notification.created }}</td>
+ <td>{{ notification.count }}</td>
+ </tr>
+ {% empty %}
+ <tr>
+ <td colspan="6">No duplicate notifications found!</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ <button type="submit" class="btn btn-danger">Delete Selected</button>
+ </form>
+ {% else %}
+ <p>No duplicate notifications found.</p>
+ {% endif %}
+{% endblock content %}

api/users/serializers.py

@@ -19,10 +19,11 @@
  JSONAPIListField,
  ShowIfCurrentUser,
 )
-from api.base.utils import absolute_reverse, default_node_list_queryset, get_user_auth, is_deprecated, hashids
+from api.base.utils import absolute_reverse, get_user_auth, is_deprecated, hashids
+from api.base.utils import default_node_list_queryset
 from api.base.versioning import get_kebab_snake_case_field
 from api.nodes.serializers import NodeSerializer, RegionRelationshipField
-from framework.auth.views import send_confirm_email_async
+from framework.auth.views import send_confirm_email
 from osf.exceptions import ValidationValueError, ValidationError, BlockedEmailError
 from osf.models import Email, Node, OSFUser, Preprint, Registration
 from osf.models.provider import AbstractProviderGroupObjectPermission

api_tests/draft_registrations/views/test_draft_registration_list.py

@@ -30,8 +30,8 @@ def invisible_and_inactive_schema():
 class TestDraftRegistrationListTopLevelEndpoint:
 
  @pytest.fixture()
- def url_draft_registrations(self):
- return f'/{API_BASE}draft_registrations/'
+ def url_draft_registrations(self, project_public):
+ return f'/{API_BASE}draft_registrations/?'
 
  @pytest.fixture()
  def user(self):

api_tests/nodes/views/test_node_draft_registration_list.py

@@ -342,13 +342,10 @@ def test_type_is_draft_registrations(self, app, user, metaschema_open_ended, url
  assert res.status_code == 409
 
  def test_admin_can_create_draft(
- self, app, user, project_public, url_draft_registrations, payload, metaschema_open_ended
- ):
- res = app.post_json_api(
- f'{url_draft_registrations}&embed=branched_from&embed=initiator',
- payload,
- auth=user.auth
- )
+ self, app, user, project_public, url_draft_registrations,
+ payload, metaschema_open_ended):
+ url = f'{url_draft_registrations}&embed=branched_from&embed=initiator'
+ res = app.post_json_api(url, payload, auth=user.auth)
  assert res.status_code == 201
  data = res.json['data']
  assert metaschema_open_ended._id in data['relationships']['registration_schema']['links']['related']['href']

api_tests/users/views/test_user_draft_registration_list.py

@@ -44,6 +44,10 @@ def draft_registration(self, user, project_public, schema):
  branched_from=project_public
  )
 
+ @pytest.fixture()
+ def url_draft_registrations(self, project_public):
+ return f'/{API_BASE}users/me/draft_registrations/'
+
  def test_unacceptable_methods(self):
  assert only_supports_methods(UserDraftRegistrations, ['GET'])
 
@@ -133,16 +137,16 @@ def test_draft_with_deleted_registered_node_shows_up_in_draft_list(
  assert data[0]['id'] == draft_registration._id
  assert data[0]['attributes']['registration_metadata'] == {}
 
- def test_cannot_access_other_users_draft_registration(self, app, user, other_admin, draft_registration, schema):
- res = app.get(
- f'/{API_BASE}users/{user._id}/draft_registrations/',
- auth=other_admin.auth,
- expect_errors=True
- )
+ def test_cannot_access_other_users_draft_registration(
+ self, app, user, other_admin, project_public,
+ draft_registration, schema):
+ url = f'/{API_BASE}users/{user._id}/draft_registrations/'
+ res = app.get(url, auth=other_admin.auth, expect_errors=True)
  assert res.status_code == 403
 
- def test_can_access_own_draft_registrations_with_guid(self, app, user, draft_registration):
- url = '/{}users/{}/draft_registrations/'.format(API_BASE, user._id)
+ def test_can_access_own_draft_registrations_with_guid(
+ self, app, user, draft_registration):
+ url = f'/{API_BASE}users/{user._id}/draft_registrations/'
  res = app.get(url, auth=user.auth, expect_errors=True)
  assert res.status_code == 200
  assert len(res.json['data']) == 1

api_tests/users/views/test_user_settings.py

@@ -577,8 +577,8 @@ def test_updating_verified_for_merge(self, app, user_one, user_two, payload):
  assert res.json['data']['attributes']['confirmed'] is True
  assert res.json['data']['attributes']['is_merge'] is False
 
- @mock.patch('api.users.views.send_confirm_email_async')
- def test_resend_confirmation_email(self, mock_send_confirm_email_async, app, user_one, unconfirmed_url, confirmed_url):
+ @mock.patch('api.users.views.send_confirm_email')
+ def test_resend_confirmation_email(self, mock_send_confirm_email, app, user_one, unconfirmed_url, confirmed_url):
  url = f'{unconfirmed_url}?resend_confirmation=True'
  res = app.get(url, auth=user_one.auth)
  assert res.status_code == 202

docker-compose.yml

@@ -1,5 +1,7 @@
 # Reference README-docker-compose.md for instructions.
 
+version: '3.5'
+
 volumes:
  redis_data_vol:
  external: false

osf/management/commands/create_notifications.py

@@ -0,0 +1,27 @@
+from django.core.management.base import BaseCommand
+from osf.models.notifications import NotificationSubscription
+from osf.models import OSFUser, Node
+from django.utils.crypto import get_random_string
+from django.utils import timezone
+
+class Command(BaseCommand):
+ help = 'Create duplicate notifications for testing'
+
+ def handle(self, *args, **kwargs):
+ user = OSFUser.objects.first()
+ node = Node.objects.first()
+ event_name = 'file_added'
+
+ for _ in range(3):
+ unique_id = get_random_string(length=32)
+ notification = NotificationSubscription.objects.create(
+ user=user,
+ node=node,
+ event_name=event_name,
+ _id=unique_id,
+ created=timezone.now()
+ )
+ notification.email_transactional.add(user)
+ notification.save()
+
+ self.stdout.write(self.style.SUCCESS('Successfully created duplicate notifications'))