[ENG-4961] Allow s3 full configuration via V2 API

addons/s3/models.py

@@ -54,6 +54,10 @@ class NodeSettings(BaseOAuthNodeSettings, BaseStorageAddon):
  encrypt_uploads = models.BooleanField(default=ENCRYPT_UPLOADS_DEFAULT)
  user_settings = models.ForeignKey(UserSettings, null=True, blank=True, on_delete=models.CASCADE)
 
+ @property
+ def has_auth(self):
+ return bool(self.user_settings and self.user_settings.external_accounts.exists() and self.external_account)
+
  @property
  def folder_path(self):
  return self.folder_name

api/nodes/serializers.py

@@ -1,6 +1,7 @@
 from django.db import connection
 from distutils.version import StrictVersion
 
+from addons.s3.utils import connect_s3
 from api.base.exceptions import (
  Conflict, EndpointNotImplementedError,
  InvalidModelValueError,
@@ -43,6 +44,7 @@
 from website.project import new_private_link
 from website.project.model import NodeUpdateError
 from osf.utils import permissions as osf_permissions
+from boto.exception import S3ResponseError
 
 
 class RegistrationProviderRelationshipField(RelationshipField):
@@ -1093,6 +1095,48 @@ def update(self, instance, validated_data):
  return instance
 
 
+class S3NodeAddonSettingsSerializer(NodeAddonSettingsSerializer):
+
+ access_token = ser.CharField(required=False, allow_null=True, write_only=True)
+ secret_token = ser.CharField(required=False, allow_null=True, write_only=True)
+
+ def update(self, instance, validated_data):
+ """
+ Overrides NodeSettings behavior to allow S3 specific behavior.
+ """
+ instance = super().update(instance, validated_data)
+
+ access_token = validated_data.get('access_token')
+ secret_token = validated_data.get('secret_token')
+ if access_token and secret_token:
+ # Validate S3 credentials before creating account
+ try:
+ user_info = connect_s3(access_token, secret_token).get_all_buckets().owner
+ except S3ResponseError:
+ raise exceptions.PermissionDenied('The S3 credentials provided are incorrect.')
+
+ account, created = ExternalAccount.objects.update_or_create(
+ provider='s3',
+ provider_id=user_info.id,
+ defaults={
+ 'display_name': user_info.display_name,
+ 'oauth_key': access_token,
+ 'oauth_secret': secret_token,
+ },
+ )
+ # If the user adds credentials via the v2 API add them to that user's user settings too.
+ instance.external_account = account
+ user = self.context['request'].user
+ instance.user_settings = user.get_or_add_addon('s3')
+ instance.save()
+
+ if not user.external_accounts.filter(id=account.id).exists():
+ user.external_accounts.add(account)
+ user.save()
+
+ return instance
+
+
 class NodeDetailSerializer(NodeSerializer):
  """
  Overrides NodeSerializer to make id required.

api/nodes/views.py

@@ -93,6 +93,7 @@
 from api.nodes.serializers import (
  NodeSerializer,
  ForwardNodeAddonSettingsSerializer,
+ S3NodeAddonSettingsSerializer,
  NodeAddonSettingsSerializer,
  NodeLinksSerializer,
  NodeForksSerializer,
@@ -1419,8 +1420,12 @@ def get_serializer_class(self):
  """
  Use NodeDetailSerializer which requires 'id'
  """
- if 'provider' in self.kwargs and self.kwargs['provider'] == 'forward':
+ provider = self.kwargs.get('provider')
+
+ if provider == 'forward':
  return ForwardNodeAddonSettingsSerializer
+ elif provider == 's3':
+ return S3NodeAddonSettingsSerializer
  else:
  return NodeAddonSettingsSerializer
 

api_tests/addons_tests/s3/test_configure_s3.py

@@ -0,0 +1,96 @@
+import mock
+import pytest
+from framework.auth.core import Auth
+from api.base.settings.defaults import API_BASE
+from osf_tests.factories import ProjectFactory, AuthUserFactory, ExternalAccountFactory
+from addons.s3.tests.factories import S3UserSettingsFactory
+
+
+def _mock_s3_client():
+ """
+ Mock client for boto.s3.connection.S3Connection
+ """
+ _mock = lambda attributes: type('MockObject', (mock.Mock,), attributes)
+ return _mock({
+ 'get_all_buckets': _mock({
+ 'owner': _mock({
+ 'display_name': 'Jalen Hurts',
+ 'id': '#1',
+ }),
+ }),
+ 'head_bucket': _mock({}),
+ 'get_bucket': _mock({
+ 'get_location': lambda *args, **kwargs: 'us-west-1',
+ }),
+ })
+
+
+@pytest.mark.django_db
+class TestS3Config:
+ """
+ This tests features added as part of the the POSE grant, these features should allow our Amazon S3 addons to be
+ fully configured via osf.io's REST API, instead of relying on the legacy FE.
+ Features added:
+
+ 1. Ability to add credentials via API tested in `test_addon_credentials_PATCH`
+ 2. Ability to configure AWS Bucket and base folders entirely via API in tested in `test_addon_folders_PATCH`
+ """
+
+ @pytest.fixture()
+ def user(self):
+ return AuthUserFactory()
+
+ @pytest.fixture()
+ def node(self, user):
+ return ProjectFactory(creator=user)
+
+ @pytest.fixture()
+ def enabled_addon(self, node, user):
+ addon = node.get_or_add_addon('s3', auth=Auth(user))
+ addon.user_settings = S3UserSettingsFactory(owner=user)
+ addon.save()
+ return addon
+
+ @pytest.fixture()
+ def node_with_authorized_addon(self, user, node, enabled_addon):
+ external_account = ExternalAccountFactory(provider='s3')
+ user.external_accounts.add(external_account)
+ enabled_addon.external_account = external_account
+ enabled_addon.save()
+ return node
+
+ @mock.patch('addons.s3.utils.S3Connection', return_value=_mock_s3_client())
+ def test_addon_credentials_PATCH(self, mock_s3, app, node, user, enabled_addon):
+ resp = app.patch_json_api(
+ f'/{API_BASE}nodes/{node._id}/addons/s3/',
+ {
+ 'data': {
+ 'attributes': {
+ 'access_token': 'test_access_key',
+ 'secret_token': 'test_secret_key'
+ }
+ },
+ },
+ auth=user.auth
+ )
+ assert resp.status_code == 200
+ assert resp.json['data']['attributes']['external_account_id']
+ assert resp.json['data']['attributes']['node_has_auth']
+
+ @mock.patch('addons.s3.utils.S3Connection', return_value=_mock_s3_client())
+ def test_addon_folders_PATCH(self, mock_s3, app, node_with_authorized_addon, user):
+ resp = app.patch_json_api(
+ f'/{API_BASE}nodes/{node_with_authorized_addon._id}/addons/s3/',
+ {
+ 'data': {
+ 'attributes': {
+ 'folder_id': 'test_folder_id',
+ 'folder_path': 'test_folder_path:/'
+ }
+ },
+ },
+ auth=user.auth
+ )
+ assert resp.status_code == 200
+ assert resp.json['data']['attributes']['folder_id'] == 'test_folder_id'
+ assert resp.json['data']['attributes']['folder_path'] == 'test_folder_id (California)'