Merge pull request #47 from bkwi/refactor-multipart-complete

Use built-in retry mechanism

Author: bkwi

filestack/uploads/intelligent_ingestion.py

@@ -4,10 +4,10 @@
 import mimetypes
 import hashlib
 import logging
-import time
 import functools
-from multiprocessing.pool import ThreadPool
 import threading
+from urllib3.util.retry import Retry
+from multiprocessing.pool import ThreadPool
 
 from base64 import b64encode
 
@@ -143,17 +143,12 @@ def upload(apikey, filepath, file_obj, storage, params=None, security=None):
         payload['store']['workflows'] = params['workflows']
 
     complete_url = 'https://{}/multipart/complete'.format(start_response['location_url'])
-    for wait_time in (0, 1, 2, 3, 5):
-        time.sleep(wait_time)
-        complete_response = requests.post(complete_url, json=payload, headers=config.HEADERS)
-        log.debug('Complete response: %s. Content: %s', complete_response, complete_response.content)
-        if complete_response.status_code == 200:
-            break
-    else:
-        log.error(
-            'Did not receive a correct complete response: %s. Content %s',
-            complete_response, complete_response.content
-        )
-        raise
-
-    return complete_response.json()
+    session = requests.Session()
+    retries = Retry(total=7, backoff_factor=0.2, status_forcelist=[202], method_whitelist=frozenset(['POST']))
+    session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
+    response = session.post(complete_url, json=payload, headers=config.HEADERS)
+    if response.status_code != 200:
+        log.error('Did not receive a correct complete response: %s. Content %s', response, response.content)
+        raise Exception('Invalid complete response: {}'.format(response.content))
+
+    return response.json()

filestack/utils.py

@@ -19,7 +19,7 @@ class RequestsWrapper:
     def __getattr__(self, name):
         if name in ('get', 'post', 'put', 'delete'):
             return partial(self.handle_request, name)
-        return super().__getattribute__(name)
+        return original_requests.__getattribute__(name)
 
     def handle_request(self, name, *args, **kwargs):
         if 'headers' not in kwargs:

tests/intelligent_ingestion_test.py

@@ -90,31 +90,3 @@ def test_min_chunk_size_exception(post_mock, put_mock):
         upload_part(
             'Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 5415034, 's3', defaultdict(lambda: 'fs-upload.com'), part
         )
-
-
-@patch('filestack.uploads.intelligent_ingestion.time.sleep')
-@patch('filestack.uploads.intelligent_ingestion.requests.post')
-@patch('filestack.uploads.intelligent_ingestion.upload_part')
-def test_wait_for_complete(upload_part, post_mock, sleep_mock):
-    post_mock.side_effect = [
-        DummyHttpResponse(json_dict={
-            'uri': 'upload-uri', 'region': 'upload-region', 'upload_id': 'upload-id',
-            'location_url': 'upload-loc-url'
-        }),  # start response
-        DummyHttpResponse(status_code=202),
-        DummyHttpResponse(status_code=202),
-        DummyHttpResponse(status_code=202),
-        DummyHttpResponse(status_code=200)
-    ]
-    security = Security({'expires': 999}, 'secret')
-    upload_params = {'filename': 'new-filename.mp4', 'path': 'some/new/path'}
-    upload('AAApikeyz', 'tests/data/doom.mp4', None, 's3', upload_params, security)
-    assert post_mock.call_count == 5
-    start_resp_args, start_resp_kwargs = post_mock.call_args_list[0]
-    url = start_resp_args[0]
-    request_payload = start_resp_kwargs['json']
-    assert url == 'https://upload.filestackapi.com/multipart/start'
-    assert request_payload['store']['path'] == 'some/new/path'
-    assert request_payload['filename'] == 'new-filename.mp4'
-    assert request_payload['policy'] == 'eyJleHBpcmVzIjogOTk5fQ=='
-    assert request_payload['signature'] == 'c0b1b4d794f867287eedb34e477805aa7f5e1c9d1ec24fc55a085608b79e65fa'