Upload external urls using /process endpoint (#54)

Author: bkwi

filestack/uploads/external_url.py

@@ -1,36 +1,31 @@
-import base64
-from collections import OrderedDict
-
-from filestack.utils import requests
-
 from filestack import config
+from filestack.utils import requests
 
 
-def build_store_task(store_params):
-    if not store_params:
-        return 'store'
-
-    task_args = []
-    store_params = OrderedDict(sorted(store_params.items(), key=lambda t: t[0]))
-    for key, value in store_params.items():
-        if key in ('filename', 'location', 'path', 'container', 'region', 'access', 'base64decode', 'workflows'):
-            if key == 'workflows':
-                value = '[{}]'.format(','.join('"{}"'.format(item) for item in value))
-            if key == 'path':
-                value = '"{}"'.format(value)
-            task_args.append('{}:{}'.format(key, str(value).lower()))
-
-    return 'store={}'.format(','.join(task_args))
+def upload_external_url(url, apikey, store_params=None, security=None):
+    store_params = store_params or {}
 
+    # remove params that are currently not supported in external url upload
+    for item in ('mimetype', 'upload_tags'):
+        store_params.pop(item, None)
 
-def upload_external_url(url, apikey, store_params=None, security=None):
-    store_task = build_store_task(store_params or {})
-    encoded_url = 'b64://{}'.format(base64.urlsafe_b64encode(url.encode()).decode())
-    url_elements = [config.CDN_URL, apikey, store_task, encoded_url]
+    payload = {
+        'apikey': apikey,
+        'sources': [url],
+        'tasks': [{
+            'name': 'store',
+            'params': store_params
+        }]
+    }
 
     if security is not None:
-        url_elements.insert(3, security.as_url_string())
-
-    # TODO: use processing endpoint and "store" task for uploading external urls
-    response = requests.get('/'.join(url_elements))
+        payload['tasks'].append({
+            'name': 'security',
+            'params': {
+                'policy': security.policy_b64,
+                'signature': security.signature
+            }
+        })
+
+    response = requests.post('{}/process'.format(config.CDN_URL), json=payload)
     return response.json()['handle']

tests/client_test.py

@@ -29,7 +29,7 @@ def test_wrong_storage():
 
 
 def test_store_external_url(client):
-    @urlmatch(netloc=r'cdn.filestackcontent\.com', method='get', scheme='https')
+    @urlmatch(netloc=r'cdn.filestackcontent\.com', method='post', scheme='https')
     def api_store(url, request):
         return response(200, {'handle': HANDLE})
 

tests/uploads/test_upload_external_url.py

@@ -1,63 +1,48 @@
-import base64
 from unittest.mock import patch
 
 import pytest
 
 from tests.helpers import DummyHttpResponse
 from filestack import config
-from filestack.models import Security
 from filestack.uploads.external_url import upload_external_url
 
 url = 'http://image.url'
-encoded_url = 'b64://{}'.format(base64.urlsafe_b64encode(url.encode()).decode())
 apikey = 'TESTAPIKEY'
 
 
-@patch('filestack.uploads.external_url.requests.get')
-def test_upload(post_mock):
-    post_mock.return_value = DummyHttpResponse(json_dict={'handle': 'newHandle'})
-
-    handle = upload_external_url(url, apikey)
-    assert handle == 'newHandle'
-    post_mock.assert_called_once_with('{}/{}/store/{}'.format(config.CDN_URL, apikey, encoded_url))
-
-
-@pytest.mark.parametrize('store_params, expected_store_task', [
-    [{'location': 'S3'}, 'store=location:s3'],
-    [{'path': 'store/path/image.jpg'}, 'store=path:"store/path/image.jpg"'],
-    [{'base64decode': True, 'access': 'public'}, 'store=access:public,base64decode:true'],
-    [
-        {'workflows': ['uuid-1', 'uuid-2'], 'container': 'bucket-name'},
-        'store=container:bucket-name,workflows:["uuid-1","uuid-2"]'
-    ],
+@pytest.mark.parametrize('store_params, security, expected_store_tasks', [
+    (
+        {'location': 's3'},
+        None,
+        [
+            {
+                'name': 'store', 'params': {'location': 's3'}
+            }
+        ]
+    ),
+    (
+        {'path': 'new-path/', 'mimetype': 'application/json'},
+        type('SecurityMock', (), {'policy_b64': 'abc', 'signature': '123'}),
+        [
+            {
+                'name': 'store', 'params': {'path': 'new-path/'}
+            },
+            {
+                'name': 'security', 'params': {'policy': 'abc', 'signature': '123'}
+            }
+        ]
+    )
 ])
-@patch('filestack.uploads.external_url.requests.get')
-def test_upload_with_store_params(post_mock, store_params, expected_store_task):
+@patch('filestack.uploads.external_url.requests.post')
+def test_upload_with_store_params(post_mock, store_params, security, expected_store_tasks):
+    expected_payload = {
+        'apikey': 'TESTAPIKEY',
+        'sources': ['http://image.url'],
+        'tasks': expected_store_tasks
+    }
     post_mock.return_value = DummyHttpResponse(json_dict={'handle': 'newHandle'})
 
-    handle = upload_external_url(url, apikey, store_params=store_params)
+    handle = upload_external_url(url, apikey, store_params=store_params, security=security)
     assert handle == 'newHandle'
     post_args, _ = post_mock.call_args
-    req_url = post_args[0]
-    assert expected_store_task in req_url
-
-
-@patch('filestack.uploads.external_url.requests.get')
-def test_upload_with_security(post_mock):
-    post_mock.return_value = DummyHttpResponse(json_dict={'handle': 'newHandle'})
-    security = Security({'expiry': 123123123123, 'call': ['write']}, 'SECRET')
-    handle = upload_external_url(url, apikey, security=security)
-    assert handle == 'newHandle'
-    expected_url = '{}/{}/store/{}/{}'.format(
-        config.CDN_URL, apikey, security.as_url_string(), encoded_url
-    )
-    post_mock.assert_called_once_with(expected_url)
-
-
-@patch('filestack.uploads.external_url.requests.get')
-def test_upload_exception(post_mock):
-    error_message = 'Oops!'
-    post_mock.side_effect = Exception(error_message)
-
-    with pytest.raises(Exception, match=error_message):
-        upload_external_url(url, apikey)
+    post_mock.assert_called_once_with('{}/process'.format(config.CDN_URL), json=expected_payload)