Merge branch 'release/19.28.0'

Author: brianjgeiger

CHANGELOG

@@ -2,6 +2,13 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+19.28.0 (2019-09-24)
+===================
+- API v2: Use consistent naming for JSON API type (kebab-case)
+- API v2: Fix sorting on fields with source attribute
+- API v2: Add SparseLists for Nodes and Registrations
+- Management command to remove duplicate files and folders
+
 19.27.0 (2019-09-18)
 ===================
 - Automatically map subjects when a preprint is moved to a different

api/addons/serializers.py

@@ -1,10 +1,13 @@
 from rest_framework import serializers as ser
 from api.base.serializers import JSONAPISerializer, LinksField
 from api.base.utils import absolute_reverse
+from api.base.versioning import get_kebab_snake_case_field
 
 class NodeAddonFolderSerializer(JSONAPISerializer):
     class Meta:
-        type_ = 'node_addon_folders'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'node-addon-folders')
 
     id = ser.CharField(read_only=True)
     kind = ser.CharField(default='folder', read_only=True)

api/base/filters.py

@@ -66,6 +66,51 @@ def filter_queryset(self, request, queryset, view):
             return queryset.sort(*ordering)
         return queryset
 
+    def get_serializer_source_field(self, view, request):
+        """
+        Returns a dictionary of serializer fields and source names. i.e. {'date_created': 'created'}
+
+        Logic borrowed from OrderingFilter.get_default_valid_fields with modifications to retrieve
+        source fields for serializer field names.
+
+        :param view api view
+        :
+        """
+        field_to_source_mapping = {}
+
+        if hasattr(view, 'get_serializer_class'):
+            serializer_class = view.get_serializer_class()
+        else:
+            serializer_class = getattr(view, 'serializer_class', None)
+
+        # This will not allow any serializer fields with nested related fields to be sorted on
+        for field_name, field in serializer_class(context={'request': request}).fields.items():
+            if not getattr(field, 'write_only', False) and not field.source == '*' and field_name != field.source:
+                field_to_source_mapping[field_name] = field.source.replace('.', '_')
+
+        return field_to_source_mapping
+
+    # Overrides OrderingFilter
+    def remove_invalid_fields(self, queryset, fields, view, request):
+        """
+        Returns an array of valid fields to be used for ordering.
+        Any valid source fields which are input remain in the valid fields list using the super method.
+        Serializer fields are mapped to their source fields and returned.
+        :param fields, array, input sort fields
+        :returns array of source fields for sorting.
+        """
+        valid_fields = super(OSFOrderingFilter, self).remove_invalid_fields(queryset, fields, view, request)
+        if not valid_fields:
+            for invalid_field in fields:
+                ordering_sign = '-' if invalid_field[0] == '-' else ''
+                invalid_field = invalid_field.lstrip('-')
+
+                field_source_mapping = self.get_serializer_source_field(view, request)
+                source_field = field_source_mapping.get(invalid_field, None)
+                if source_field:
+                    valid_fields.append(ordering_sign + source_field)
+        return valid_fields
+
 
 class FilterMixin(object):
     """ View mixin with helper functions for filtering. """

api/base/renderers.py

@@ -23,15 +23,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
         # See JSON-API documentation on meta information: http://jsonapi.org/format/#document-meta
         data_type = type(data)
         if renderer_context is not None and data_type != str and data is not None:
-            meta_dict = renderer_context.get('meta')
+            meta_dict = renderer_context.get('meta', {})
             version = getattr(renderer_context['request'], 'version', None)
-            if meta_dict is not None:
-                if version:
-                    meta_dict['version'] = renderer_context['request'].version
-                    data.setdefault('meta', {}).update(meta_dict)
-            elif version:
-                meta_dict = {'version': renderer_context['request'].version}
-                data.setdefault('meta', {}).update(meta_dict)
+            warning = renderer_context['request'].META.get('warning', None)
+            if version:
+                meta_dict['version'] = version
+            if warning:
+                meta_dict['warning'] = warning
+            data.setdefault('meta', {}).update(meta_dict)
         return super(JSONAPIRenderer, self).render(data, accepted_media_type, renderer_context)
 
 

api/base/serializers.py

@@ -23,6 +23,7 @@
 from osf.models import AbstractNode, MaintenanceState, Preprint
 from website import settings
 from website.project.model import has_anonymous_link
+from api.base.versioning import KEBAB_CASE_VERSION, get_kebab_snake_case_field
 
 
 def get_meta_type(serializer_class, request):
@@ -404,8 +405,11 @@ def to_internal_value(self, data):
             type_ = get_meta_type(self.root.child, request)
         else:
             type_ = get_meta_type(self.root, request)
-
-        if type_ != data:
+        kebab_case = str(type_).replace('-', '_')
+        if type_ != data and kebab_case == data:
+            type_ = kebab_case
+            self.context['request'].META.setdefault('warning', 'As of API Version {0}, all types are now Kebab-case. {0} will accept snake_case, but this will be deprecated in future versions.'.format(KEBAB_CASE_VERSION))
+        elif type_ != data:
             raise api_exceptions.Conflict(detail=('This resource has a type of "{}", but you set the json body\'s type field to "{}". You probably need to change the type field to match the resource\'s type.'.format(type_, data)))
         return super(TypeField, self).to_internal_value(data)
 
@@ -1578,7 +1582,9 @@ class AddonAccountSerializer(JSONAPISerializer):
     })
 
     class Meta:
-        type_ = 'external_accounts'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'external-accounts')
 
     def get_absolute_url(self, obj):
         kwargs = self.context['request'].parser_context['kwargs']

api/base/settings/defaults.py

@@ -168,6 +168,7 @@
         '2.15',
         '2.16',
         '2.17',
+        '2.18',
     ),
     'DEFAULT_FILTER_BACKENDS': ('api.base.filters.OSFOrderingFilter',),
     'DEFAULT_PAGINATION_CLASS': 'api.base.pagination.JSONAPIPagination',

api/base/urls.py

@@ -33,6 +33,7 @@
                 url(r'^status/', views.status_check, name='status_check'),
                 url(r'^actions/', include('api.actions.urls', namespace='actions')),
                 url(r'^addons/', include('api.addons.urls', namespace='addons')),
+                url(r'^alerts/', include('api.alerts.urls', namespace='alerts')),
                 url(r'^applications/', include('api.applications.urls', namespace='applications')),
                 url(r'^citations/', include('api.citations.urls', namespace='citations')),
                 url(r'^collections/', include('api.collections.urls', namespace='collections')),
@@ -56,16 +57,16 @@
                 url(r'^requests/', include('api.requests.urls', namespace='requests')),
                 url(r'^scopes/', include('api.scopes.urls', namespace='scopes')),
                 url(r'^search/', include('api.search.urls', namespace='search')),
+                url(r'^sparse/', include('api.sparse.urls', namespace='sparse')),
                 url(r'^subjects/', include('api.subjects.urls', namespace='subjects')),
                 url(r'^subscriptions/', include('api.subscriptions.urls', namespace='subscriptions')),
                 url(r'^taxonomies/', include('api.taxonomies.urls', namespace='taxonomies')),
                 url(r'^test/', include('api.test.urls', namespace='test')),
                 url(r'^tokens/', include('api.tokens.urls', namespace='tokens')),
                 url(r'^users/', include('api.users.urls', namespace='users')),
                 url(r'^view_only_links/', include('api.view_only_links.urls', namespace='view-only-links')),
-                url(r'^_waffle/', include('api.waffle.urls', namespace='waffle')),
                 url(r'^wikis/', include('api.wikis.urls', namespace='wikis')),
-                url(r'^alerts/', include('api.alerts.urls', namespace='alerts')),
+                url(r'^_waffle/', include('api.waffle.urls', namespace='waffle')),
             ],
         ),
     ),

api/base/versioning.py

@@ -3,12 +3,16 @@
 from rest_framework import versioning as drf_versioning
 from rest_framework.compat import unicode_http_header
 from rest_framework.utils.mediatypes import _MediaType
+from distutils.version import StrictVersion
 
 from api.base import exceptions
 from api.base import utils
 from api.base.renderers import BrowsableAPIRendererNoForms
 from api.base.settings import LATEST_VERSIONS
 
+# KEBAB_CASE_VERSION determines the API version in which kebab-case will begin being accepted.
+# Note that this version will not deprecate snake_case yet.
+KEBAB_CASE_VERSION = '2.18'
 
 def get_major_version(version):
     return int(version.split('.')[0])
@@ -28,6 +32,12 @@ def get_latest_sub_version(major_version):
     # '2' --> '2.6'
     return LATEST_VERSIONS.get(major_version, None)
 
+def get_kebab_snake_case_field(version, field):
+    if StrictVersion(version) < StrictVersion(KEBAB_CASE_VERSION):
+        return field.replace('-', '_')
+    else:
+        return field
+
 class BaseVersioning(drf_versioning.BaseVersioning):
 
     def __init__(self):

api/comments/serializers.py

@@ -17,6 +17,7 @@
     AnonymizedRegexField,
     VersionedDateTimeField,
 )
+from api.base.versioning import get_kebab_snake_case_field
 
 
 class CommentReport(object):
@@ -223,7 +224,9 @@ class CommentReportSerializer(JSONAPISerializer):
     links = LinksField({'self': 'get_absolute_url'})
 
     class Meta:
-        type_ = 'comment_reports'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'comment-reports')
 
     def get_absolute_url(self, obj):
         return absolute_reverse(

api/files/serializers.py

@@ -35,6 +35,7 @@
 from api.base.utils import absolute_reverse, get_user_auth
 from api.base.exceptions import Conflict, InvalidModelValueError
 from api.base.schemas.utils import from_json
+from api.base.versioning import get_kebab_snake_case_field
 
 class CheckoutField(ser.HyperlinkedRelatedField):
 
@@ -423,7 +424,9 @@ def get_name(self, obj):
         return obj.get_basefilenode_version(file).version_name
 
     class Meta:
-        type_ = 'file_versions'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'file-versions')
 
     def self_url(self, obj):
         return absolute_reverse(
@@ -514,7 +517,9 @@ def get_absolute_url(self, obj):
         return obj.absolute_api_v2_url
 
     class Meta:
-        type_ = 'metadata_records'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'metadata-records')
 
 
 def get_file_download_link(obj, version=None, view_only=None):

api/nodes/serializers.py

@@ -21,6 +21,7 @@
     absolute_reverse, get_object_or_error,
     get_user_auth, is_truthy,
 )
+from api.base.versioning import get_kebab_snake_case_field
 from api.taxonomies.serializers import TaxonomizableSerializerMixin
 from django.apps import apps
 from django.conf import settings
@@ -886,7 +887,9 @@ def update(self, node, validated_data):
 
 class NodeAddonSettingsSerializerBase(JSONAPISerializer):
     class Meta:
-        type_ = 'node_addons'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'node-addons')
 
     id = ser.CharField(source='config.short_name', read_only=True)
     node_has_auth = ser.BooleanField(source='has_auth', read_only=True)
@@ -1313,7 +1316,9 @@ class NodeLinksSerializer(JSONAPISerializer):
 
     )
     class Meta:
-        type_ = 'node_links'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'node-links')
 
     links = LinksField({
         'self': 'get_absolute_url',
@@ -1520,7 +1525,9 @@ def create(self, validated_data):
         return draft
 
     class Meta:
-        type_ = 'draft_registrations'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'draft-registrations')
 
 
 class DraftRegistrationDetailSerializer(DraftRegistrationSerializer):
@@ -1629,7 +1636,9 @@ def get_absolute_url(self, obj):
         )
 
     class Meta:
-        type_ = 'view_only_links'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'view-only-links')
 
 
 class NodeViewOnlyLinkUpdateSerializer(NodeViewOnlyLinkSerializer):

api/schemas/serializers.py

@@ -1,6 +1,6 @@
 from rest_framework import serializers as ser
 from api.base.serializers import (JSONAPISerializer, IDField, TypeField, LinksField)
-
+from api.base.versioning import get_kebab_snake_case_field
 
 class SchemaSerializer(JSONAPISerializer):
 
@@ -28,13 +28,17 @@ class RegistrationSchemaSerializer(SchemaSerializer):
     filterable_fields = ['active']
 
     class Meta:
-        type_ = 'registration_schemas'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'registration-schemas')
 
 
 class FileMetadataSchemaSerializer(SchemaSerializer):
 
     class Meta:
-        type_ = 'file_metadata_schemas'
+        @staticmethod
+        def get_type(request):
+            return get_kebab_snake_case_field(request.version, 'file-metadata-schemas')
 
 
 class DeprecatedMetaSchemaSerializer(SchemaSerializer):