Use responses to mock requests in upload tests (#64)

Author: bkwi

requirements.txt

@@ -3,5 +3,5 @@ httmock==1.2.6
 pytest==4.6.3
 pytest-cov==2.5.0
 requests==2.25.1
-responses==0.10.6
+responses==0.14.0
 trafaret==2.0.2

tests/intelligent_ingestion_test.py

@@ -1,91 +1,78 @@
-from unittest.mock import patch, call, ANY
-from collections import defaultdict
+import json
 
 import pytest
+import responses
 
 from filestack.uploads.intelligent_ingestion import upload_part
-from tests.helpers import DummyHttpResponse
 
 
-@patch('filestack.uploads.intelligent_ingestion.requests.put')
-@patch('filestack.uploads.intelligent_ingestion.requests.post')
-def test_upload_part_success(post_mock, put_mock):
-    post_mock.side_effect = [
-        DummyHttpResponse(json_dict={'url': 'http://upload.url', 'headers': {'upload': 'headers'}}),
-        DummyHttpResponse()
-    ]
-
-    put_mock.return_value = DummyHttpResponse()
+@responses.activate
+def test_upload_part_success():
+    responses.add(
+        responses.POST, 'https://fs-upload.com/multipart/upload',
+        json={'url': 'http://s3.url', 'headers': {'filestack': 'headers'}}
+    )
+    responses.add(responses.PUT, 'http://s3.url')
+    responses.add(responses.POST, 'https://fs-upload.com/multipart/commit')
 
     part = {'seek_point': 0, 'num': 1}
-    upload_part(
-        'Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 1234, 's3', defaultdict(lambda: 'fs-upload.com'), part
-    )
-    assert post_mock.call_args_list == [
-        call(
-            'https://fs-upload.com/multipart/upload',
-            json={
-                'apikey': 'Aaaaapikey', 'uri': 'fs-upload.com', 'region': 'fs-upload.com',
-                'upload_id': 'fs-upload.com', 'store': {'location': 's3'},
-                'part': 1, 'size': 5415034, 'md5': 'IuNjhgPo2wbzGFo6f7WhUA==', 'offset': 0, 'fii': True
-            },
-        ),
-        call(
-            'https://fs-upload.com/multipart/commit',
-            json={
-                'apikey': 'Aaaaapikey', 'uri': 'fs-upload.com', 'region': 'fs-upload.com',
-                'upload_id': 'fs-upload.com', 'store': {'location': 's3'}, 'part': 1, 'size': 1234
-            },
-        )
-    ]
-    put_mock.assert_called_once_with(
-        'http://upload.url',
-        data=ANY,
-        headers={'upload': 'headers'}
-    )
+    start_response = {
+        'uri': 'fs-upload.com', 'location_url': 'fs-upload.com', 'region': 'region', 'upload_id': 'abc'
+    }
+    upload_part('Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 1234, 's3', start_response, part)
+    multipart_upload_payload = json.loads(responses.calls[0].request.body.decode())
+    assert multipart_upload_payload == {
+        'apikey': 'Aaaaapikey', 'uri': 'fs-upload.com', 'region': 'region',
+        'upload_id': 'abc', 'store': {'location': 's3'},
+        'part': 1, 'size': 5415034, 'md5': 'IuNjhgPo2wbzGFo6f7WhUA==', 'offset': 0, 'fii': True
+    }
+    with open('tests/data/doom.mp4', 'rb') as f:
+        assert responses.calls[1].request.body == f.read()
+    multipart_commit_payload = json.loads(responses.calls[2].request.body.decode())
+    assert multipart_commit_payload == {
+        'apikey': 'Aaaaapikey', 'uri': 'fs-upload.com', 'region': 'region',
+        'upload_id': 'abc', 'store': {'location': 's3'}, 'part': 1, 'size': 1234
+    }
 
 
-@patch('filestack.uploads.intelligent_ingestion.requests.put')
-@patch('filestack.uploads.intelligent_ingestion.requests.post')
-def test_upload_part_with_resize(post_mock, put_mock):
-    # this mock will work fine for commit request too
-    post_mock.return_value = DummyHttpResponse(
-        ok=True, json_dict={'url': 'http://upload.url', 'headers': {'upload': 'headers'}}
+@responses.activate
+def test_upload_part_with_resize():
+    responses.add(
+        responses.POST, 'https://fs-upload.com/multipart/upload',
+        json={'url': 'https://s3.url', 'headers': {'filestack': 'headers'}}
     )
+    responses.add(responses.PUT, 'https://s3.url', status=400)
+    responses.add(responses.PUT, 'https://s3.url')  # chunks 1 & 2 of part 1
+    responses.add(responses.POST, 'https://fs-upload.com/multipart/commit')
 
-    put_mock.side_effect = [
-        DummyHttpResponse(ok=False),  # fail first attempt, should split file part
-        DummyHttpResponse(),  # part 1, chunk 1
-        DummyHttpResponse(),  # part 1, chunk 2
-    ]
-
+    start_response = {
+        'uri': 'fs-upload.com', 'location_url': 'fs-upload.com', 'region': 'region', 'upload_id': 'abc'
+    }
     part = {'seek_point': 0, 'num': 1}
-    upload_part(
-        'Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 5415034, 's3', defaultdict(lambda: 'fs-upload.com'), part
-    )
+    upload_part('Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 5415034, 's3', start_response, part)
 
-    assert post_mock.call_count == 4  # 3x upload, 1 commit
-    # 1st attempt
-    req_args, req_kwargs = post_mock.call_args_list[0]
-    assert req_kwargs['json']['size'] == 5415034
-    # 2nd attempt
-    req_args, req_kwargs = post_mock.call_args_list[1]
-    assert req_kwargs['json']['size'] == 4194304
-    # 3rd attempt
-    req_args, req_kwargs = post_mock.call_args_list[2]
-    assert req_kwargs['json']['size'] == 1220730
+    responses.assert_call_count('https://fs-upload.com/multipart/upload', 3)
+    responses.assert_call_count('https://s3.url', 3)
+    assert len(responses.calls[1].request.body) == 5415034
+    assert len(responses.calls[3].request.body) == 4194304
+    assert len(responses.calls[5].request.body) == 1220730
 
 
-@patch('filestack.uploads.intelligent_ingestion.requests.put')
-@patch('filestack.uploads.intelligent_ingestion.requests.post')
-def test_min_chunk_size_exception(post_mock, put_mock):
-    post_mock.return_value = DummyHttpResponse(
-        ok=True, json_dict={'url': 'http://upload.url', 'headers': {'upload': 'headers'}}
+@responses.activate
+def test_min_chunk_size_exception():
+    responses.reset()
+    responses.add(
+        responses.POST, 'https://fs-upload.com/multipart/upload',
+        json={'url': 'https://upload.url', 'headers': {'filestack': 'headers'}}
     )
-    put_mock.return_value = DummyHttpResponse(ok=False)
+    responses.add(responses.PUT, 'https://upload.url', status=400)
 
     part = {'seek_point': 0, 'num': 1}
+    start_response = {
+        'uri': 'fs-upload.com', 'location_url': 'fs-upload.com', 'region': 'region', 'upload_id': 'abc'
+    }
     with pytest.raises(Exception, match='Minimal chunk size failed'):
-        upload_part(
-            'Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 5415034, 's3', defaultdict(lambda: 'fs-upload.com'), part
-        )
+        upload_part('Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 5415034, 's3', start_response, part)
+
+    chunk_sizes = [len(call.request.body) for call in responses.calls if call.request.method == 'PUT']
+    assert chunk_sizes[-1] == 32768  # check size of last attempt

tests/uploads/test_upload_external_url.py

@@ -1,13 +1,13 @@
-from unittest.mock import patch
+import json
 
 import pytest
+import responses
 
-from tests.helpers import DummyHttpResponse
-from filestack import config
 from filestack.uploads.external_url import upload_external_url
 
 url = 'http://image.url'
 apikey = 'TESTAPIKEY'
+PROCESS_URL = 'https://cdn.filestackcontent.com/process'
 
 
 @pytest.mark.parametrize('default_storage, store_params, security, expected_store_tasks', [
@@ -35,18 +35,17 @@
         ]
     )
 ])
-@patch('filestack.uploads.external_url.requests.post')
-def test_upload_with_store_params(post_mock, default_storage, store_params, security, expected_store_tasks):
+@responses.activate
+def test_upload_with_store_params(default_storage, store_params, security, expected_store_tasks):
     expected_payload = {
         'apikey': 'TESTAPIKEY',
         'sources': ['http://image.url'],
         'tasks': expected_store_tasks
     }
-    post_mock.return_value = DummyHttpResponse(json_dict={'handle': 'newHandle'})
-
+    responses.add(responses.POST, PROCESS_URL, json={'handle': 'newHandle'})
     upload_response = upload_external_url(
         url, apikey, default_storage, store_params=store_params, security=security
     )
     assert upload_response['handle'] == 'newHandle'
-    post_args, _ = post_mock.call_args
-    post_mock.assert_called_once_with('{}/process'.format(config.CDN_URL), json=expected_payload)
+    req_payload = json.loads(responses.calls[0].request.body.decode())
+    assert req_payload == expected_payload