Merge branch 'release/19.28.0'

CHANGELOG

@@ -2,6 +2,13 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+19.28.0 (2019-09-24)
+===================
+- API v2: Use consistent naming for JSON API type (kebab-case)
+- API v2: Fix sorting on fields with source attribute
+- API v2: Add SparseLists for Nodes and Registrations
+- Management command to remove duplicate files and folders
+
 19.27.0 (2019-09-18)
 ===================
 - Automatically map subjects when a preprint is moved to a different

api/addons/serializers.py

@@ -1,10 +1,13 @@
 from rest_framework import serializers as ser
 from api.base.serializers import JSONAPISerializer, LinksField
 from api.base.utils import absolute_reverse
+from api.base.versioning import get_kebab_snake_case_field
 
 class NodeAddonFolderSerializer(JSONAPISerializer):
  class Meta:
- type_ = 'node_addon_folders'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'node-addon-folders')
 
  id = ser.CharField(read_only=True)
  kind = ser.CharField(default='folder', read_only=True)

api/base/filters.py

@@ -66,6 +66,51 @@ def filter_queryset(self, request, queryset, view):
  return queryset.sort(*ordering)
  return queryset
 
+ def get_serializer_source_field(self, view, request):
+ """
+ Returns a dictionary of serializer fields and source names. i.e. {'date_created': 'created'}
+
+ Logic borrowed from OrderingFilter.get_default_valid_fields with modifications to retrieve
+ source fields for serializer field names.
+
+ :param view api view
+ :
+ """
+ field_to_source_mapping = {}
+
+ if hasattr(view, 'get_serializer_class'):
+ serializer_class = view.get_serializer_class()
+ else:
+ serializer_class = getattr(view, 'serializer_class', None)
+
+ # This will not allow any serializer fields with nested related fields to be sorted on
+ for field_name, field in serializer_class(context={'request': request}).fields.items():
+ if not getattr(field, 'write_only', False) and not field.source == '*' and field_name != field.source:
+ field_to_source_mapping[field_name] = field.source.replace('.', '_')
+
+ return field_to_source_mapping
+
+ # Overrides OrderingFilter
+ def remove_invalid_fields(self, queryset, fields, view, request):
+ """
+ Returns an array of valid fields to be used for ordering.
+ Any valid source fields which are input remain in the valid fields list using the super method.
+ Serializer fields are mapped to their source fields and returned.
+ :param fields, array, input sort fields
+ :returns array of source fields for sorting.
+ """
+ valid_fields = super(OSFOrderingFilter, self).remove_invalid_fields(queryset, fields, view, request)
+ if not valid_fields:
+ for invalid_field in fields:
+ ordering_sign = '-' if invalid_field[0] == '-' else ''
+ invalid_field = invalid_field.lstrip('-')
+
+ field_source_mapping = self.get_serializer_source_field(view, request)
+ source_field = field_source_mapping.get(invalid_field, None)
+ if source_field:
+ valid_fields.append(ordering_sign + source_field)
+ return valid_fields
+
 
 class FilterMixin(object):
  """ View mixin with helper functions for filtering. """

api/base/renderers.py

@@ -23,15 +23,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
  # See JSON-API documentation on meta information: http://jsonapi.org/format/#document-meta
  data_type = type(data)
  if renderer_context is not None and data_type != str and data is not None:
- meta_dict = renderer_context.get('meta')
+ meta_dict = renderer_context.get('meta', {})
  version = getattr(renderer_context['request'], 'version', None)
- if meta_dict is not None:
- if version:
- meta_dict['version'] = renderer_context['request'].version
- data.setdefault('meta', {}).update(meta_dict)
- elif version:
- meta_dict = {'version': renderer_context['request'].version}
- data.setdefault('meta', {}).update(meta_dict)
+ warning = renderer_context['request'].META.get('warning', None)
+ if version:
+ meta_dict['version'] = version
+ if warning:
+ meta_dict['warning'] = warning
+ data.setdefault('meta', {}).update(meta_dict)
  return super(JSONAPIRenderer, self).render(data, accepted_media_type, renderer_context)
 
 

api/base/serializers.py

@@ -23,6 +23,7 @@
 from osf.models import AbstractNode, MaintenanceState, Preprint
 from website import settings
 from website.project.model import has_anonymous_link
+from api.base.versioning import KEBAB_CASE_VERSION, get_kebab_snake_case_field
 
 
 def get_meta_type(serializer_class, request):
@@ -404,8 +405,11 @@ def to_internal_value(self, data):
  type_ = get_meta_type(self.root.child, request)
  else:
  type_ = get_meta_type(self.root, request)
-
- if type_ != data:
+ kebab_case = str(type_).replace('-', '_')
+ if type_ != data and kebab_case == data:
+ type_ = kebab_case
+ self.context['request'].META.setdefault('warning', 'As of API Version {0}, all types are now Kebab-case. {0} will accept snake_case, but this will be deprecated in future versions.'.format(KEBAB_CASE_VERSION))
+ elif type_ != data:
  raise api_exceptions.Conflict(detail=('This resource has a type of "{}", but you set the json body\'s type field to "{}". You probably need to change the type field to match the resource\'s type.'.format(type_, data)))
  return super(TypeField, self).to_internal_value(data)
 
@@ -1578,7 +1582,9 @@ class AddonAccountSerializer(JSONAPISerializer):
  })
 
  class Meta:
- type_ = 'external_accounts'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'external-accounts')
 
  def get_absolute_url(self, obj):
  kwargs = self.context['request'].parser_context['kwargs']

api/base/settings/defaults.py

@@ -168,6 +168,7 @@
  '2.15',
  '2.16',
  '2.17',
+ '2.18',
  ),
  'DEFAULT_FILTER_BACKENDS': ('api.base.filters.OSFOrderingFilter',),
  'DEFAULT_PAGINATION_CLASS': 'api.base.pagination.JSONAPIPagination',

api/base/urls.py

@@ -33,6 +33,7 @@
  url(r'^status/', views.status_check, name='status_check'),
  url(r'^actions/', include('api.actions.urls', namespace='actions')),
  url(r'^addons/', include('api.addons.urls', namespace='addons')),
+ url(r'^alerts/', include('api.alerts.urls', namespace='alerts')),
  url(r'^applications/', include('api.applications.urls', namespace='applications')),
  url(r'^citations/', include('api.citations.urls', namespace='citations')),
  url(r'^collections/', include('api.collections.urls', namespace='collections')),
@@ -56,16 +57,16 @@
  url(r'^requests/', include('api.requests.urls', namespace='requests')),
  url(r'^scopes/', include('api.scopes.urls', namespace='scopes')),
  url(r'^search/', include('api.search.urls', namespace='search')),
+ url(r'^sparse/', include('api.sparse.urls', namespace='sparse')),
  url(r'^subjects/', include('api.subjects.urls', namespace='subjects')),
  url(r'^subscriptions/', include('api.subscriptions.urls', namespace='subscriptions')),
  url(r'^taxonomies/', include('api.taxonomies.urls', namespace='taxonomies')),
  url(r'^test/', include('api.test.urls', namespace='test')),
  url(r'^tokens/', include('api.tokens.urls', namespace='tokens')),
  url(r'^users/', include('api.users.urls', namespace='users')),
  url(r'^view_only_links/', include('api.view_only_links.urls', namespace='view-only-links')),
- url(r'^_waffle/', include('api.waffle.urls', namespace='waffle')),
  url(r'^wikis/', include('api.wikis.urls', namespace='wikis')),
- url(r'^alerts/', include('api.alerts.urls', namespace='alerts')),
+ url(r'^_waffle/', include('api.waffle.urls', namespace='waffle')),
  ],
  ),
  ),

api/base/versioning.py

@@ -3,12 +3,16 @@
 from rest_framework import versioning as drf_versioning
 from rest_framework.compat import unicode_http_header
 from rest_framework.utils.mediatypes import _MediaType
+from distutils.version import StrictVersion
 
 from api.base import exceptions
 from api.base import utils
 from api.base.renderers import BrowsableAPIRendererNoForms
 from api.base.settings import LATEST_VERSIONS
 
+# KEBAB_CASE_VERSION determines the API version in which kebab-case will begin being accepted.
+# Note that this version will not deprecate snake_case yet.
+KEBAB_CASE_VERSION = '2.18'
 
 def get_major_version(version):
  return int(version.split('.')[0])
@@ -28,6 +32,12 @@ def get_latest_sub_version(major_version):
  # '2' --> '2.6'
  return LATEST_VERSIONS.get(major_version, None)
 
+def get_kebab_snake_case_field(version, field):
+ if StrictVersion(version) < StrictVersion(KEBAB_CASE_VERSION):
+ return field.replace('-', '_')
+ else:
+ return field
+
 class BaseVersioning(drf_versioning.BaseVersioning):
 
  def __init__(self):

api/comments/serializers.py

@@ -17,6 +17,7 @@
  AnonymizedRegexField,
  VersionedDateTimeField,
 )
+from api.base.versioning import get_kebab_snake_case_field
 
 
 class CommentReport(object):
@@ -223,7 +224,9 @@ class CommentReportSerializer(JSONAPISerializer):
  links = LinksField({'self': 'get_absolute_url'})
 
  class Meta:
- type_ = 'comment_reports'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'comment-reports')
 
  def get_absolute_url(self, obj):
  return absolute_reverse(

api/files/serializers.py

@@ -35,6 +35,7 @@
 from api.base.utils import absolute_reverse, get_user_auth
 from api.base.exceptions import Conflict, InvalidModelValueError
 from api.base.schemas.utils import from_json
+from api.base.versioning import get_kebab_snake_case_field
 
 class CheckoutField(ser.HyperlinkedRelatedField):
 
@@ -423,7 +424,9 @@ def get_name(self, obj):
  return obj.get_basefilenode_version(file).version_name
 
  class Meta:
- type_ = 'file_versions'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'file-versions')
 
  def self_url(self, obj):
  return absolute_reverse(
@@ -514,7 +517,9 @@ def get_absolute_url(self, obj):
  return obj.absolute_api_v2_url
 
  class Meta:
- type_ = 'metadata_records'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'metadata-records')
 
 
 def get_file_download_link(obj, version=None, view_only=None):

api/nodes/serializers.py

@@ -21,6 +21,7 @@
  absolute_reverse, get_object_or_error,
  get_user_auth, is_truthy,
 )
+from api.base.versioning import get_kebab_snake_case_field
 from api.taxonomies.serializers import TaxonomizableSerializerMixin
 from django.apps import apps
 from django.conf import settings
@@ -886,7 +887,9 @@ def update(self, node, validated_data):
 
 class NodeAddonSettingsSerializerBase(JSONAPISerializer):
  class Meta:
- type_ = 'node_addons'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'node-addons')
 
  id = ser.CharField(source='config.short_name', read_only=True)
  node_has_auth = ser.BooleanField(source='has_auth', read_only=True)
@@ -1313,7 +1316,9 @@ class NodeLinksSerializer(JSONAPISerializer):
 
  )
  class Meta:
- type_ = 'node_links'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'node-links')
 
  links = LinksField({
  'self': 'get_absolute_url',
@@ -1520,7 +1525,9 @@ def create(self, validated_data):
  return draft
 
  class Meta:
- type_ = 'draft_registrations'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'draft-registrations')
 
 
 class DraftRegistrationDetailSerializer(DraftRegistrationSerializer):
@@ -1629,7 +1636,9 @@ def get_absolute_url(self, obj):
  )
 
  class Meta:
- type_ = 'view_only_links'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'view-only-links')
 
 
 class NodeViewOnlyLinkUpdateSerializer(NodeViewOnlyLinkSerializer):

api/schemas/serializers.py

@@ -1,6 +1,6 @@
 from rest_framework import serializers as ser
 from api.base.serializers import (JSONAPISerializer, IDField, TypeField, LinksField)
-
+from api.base.versioning import get_kebab_snake_case_field
 
 class SchemaSerializer(JSONAPISerializer):
 
@@ -28,13 +28,17 @@ class RegistrationSchemaSerializer(SchemaSerializer):
  filterable_fields = ['active']
 
  class Meta:
- type_ = 'registration_schemas'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'registration-schemas')
 
 
 class FileMetadataSchemaSerializer(SchemaSerializer):
 
  class Meta:
- type_ = 'file_metadata_schemas'
+ @staticmethod
+ def get_type(request):
+ return get_kebab_snake_case_field(request.version, 'file-metadata-schemas')
 
 
 class DeprecatedMetaSchemaSerializer(SchemaSerializer):