Merge branch 'release/3.1.0'

Bartek Kwiecien · Bartek Kwiecien · commit 5d3d350ece0a · 2020-04-14T14:34:26.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Filestack-Python Changelog
 
+### 3.1.0 (April 14th, 2020)
+- Transformations added: auto_image, minify_js, minify_css
+- Added configurable CNAME
+- Fixed path param in store task
+
 ### 3.0.1 (July 22nd, 2019)
 - Added enhance presets
 - Set filelink security when uploading from url
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-3.0.1
+3.1.0
diff --git a/filestack/__init__.py b/filestack/__init__.py
@@ -1,4 +1,33 @@
-__version__ = '3.0.1'
+__version__ = '3.1.0'
+
+
+class CFG:
+    API_URL = 'https://www.filestackapi.com/api'
+    DEFAULT_CHUNK_SIZE = 5 * 1024 ** 2
+    DEFAULT_UPLOAD_MIMETYPE = 'application/octet-stream'
+
+    HEADERS = {
+        'User-Agent': 'filestack-python {}'.format(__version__),
+        'Filestack-Source': 'Python-{}'.format(__version__)
+    }
+
+    def __init__(self):
+        self.CNAME = ''
+
+    @property
+    def CDN_URL(self):
+        return 'https://cdn.{}'.format(self.CNAME or 'filestackcontent.com')
+
+    @property
+    def MULTIPART_START_URL(self):
+        return 'https://upload.{}/multipart/start'.format(
+            self.CNAME or 'filestackapi.com'
+        )
+
+
+config = CFG()
+
+
 from .models.client import Client
 from .models.filelink import Filelink
 from .models.security import Security
diff --git a/filestack/config.py b/filestack/config.py
diff --git a/filestack/mixins/common.py b/filestack/mixins/common.py
@@ -64,6 +64,8 @@ def store(self, filename=None, location=None, path=None, container=None,
         Returns:
             :class:`filestack.Filelink`: new Filelink object
         """
+        if path:
+            path = '"{}"'.format(path)
         instance = self._add_transform_task('store', locals())
         response = requests.post(instance.url)
         return filestack.models.Filelink(handle=response.json()['handle'])
diff --git a/filestack/mixins/imagetransformation.py b/filestack/mixins/imagetransformation.py
@@ -126,6 +126,13 @@ def pdf_info(self, colorinfo=None):
     def pdf_convert(self, pageorientation=None, pageformat=None, pages=None):
         return self._add_transform_task('pdfconvert', locals())
 
+    def minify_js(self, gzip=None, use_babel_polyfill=None, keep_fn_name=None, keep_class_name=None,
+                  mangle=None, merge_vars=None, remove_console=None, remove_undefined=None, targets=None):
+        return self._add_transform_task('minify_js', locals())
+
+    def minify_css(self, level=None, gzip=None):
+        return self._add_transform_task('minify_css', locals())
+
     def av_convert(self, *, preset=None, force=None, title=None, extname=None, filename=None,
                    width=None, height=None, upscale=None, aspect_mode=None, two_pass=None,
                    video_bitrate=None, fps=None, keyframe_interval=None, location=None,
@@ -143,6 +150,9 @@ def av_convert(self, *, preset=None, force=None, title=None, extname=None, filen
             new_transform.url, uuid, timestamp, apikey=new_transform.apikey, security=new_transform.security
         )
 
+    def auto_image(self):
+        return self._add_transform_task('auto_image', locals())
+
     def _add_transform_task(self, transformation, params):
         if isinstance(self, filestack.models.Transformation):
             instance = self
diff --git a/filestack/models/client.py b/filestack/models/client.py
@@ -25,7 +25,7 @@ class Client:
     def __init__(self, apikey, storage='S3', security=None):
         """
         Args:
-            apikey (str): The path of the file to wrap
+            apikey (str): your Filestack API key
             storage (str): default storage to be used for uploads (one of S3, `gcs`, dropbox, azure)
             security (:class:`filestack.Security`): Security object that will be used by default
                 for all API calls
@@ -109,7 +109,7 @@ def zip(self, destination_path, files, security=None):
 
     def upload_url(self, url, store_params=None, security=None):
         """
-        Uploads local file from external url
+        Uploads file from external url
 
         Args:
             url (str): file URL
diff --git a/filestack/uploads/external_url.py b/filestack/uploads/external_url.py
@@ -1,3 +1,4 @@
+import base64
 from collections import OrderedDict
 
 from filestack.utils import requests
@@ -24,7 +25,8 @@ def build_store_task(store_params):
 
 def upload_external_url(url, apikey, store_params=None, security=None):
     store_task = build_store_task(store_params or {})
-    url_elements = [config.CDN_URL, apikey, store_task, url]
+    encoded_url = 'b64://{}'.format(base64.urlsafe_b64encode(url.encode()).decode())
+    url_elements = [config.CDN_URL, apikey, store_task, encoded_url]
 
     if security is not None:
         url_elements.insert(3, security.as_url_string())
diff --git a/filestack/uploads/intelligent_ingestion.py b/filestack/uploads/intelligent_ingestion.py
@@ -4,10 +4,10 @@
 import mimetypes
 import hashlib
 import logging
-import time
 import functools
-from multiprocessing.pool import ThreadPool
 import threading
+from urllib3.util.retry import Retry
+from multiprocessing.pool import ThreadPool
 
 from base64 import b64encode
 
@@ -143,17 +143,12 @@ def upload(apikey, filepath, file_obj, storage, params=None, security=None):
         payload['store']['workflows'] = params['workflows']
 
     complete_url = 'https://{}/multipart/complete'.format(start_response['location_url'])
-    for wait_time in (0, 1, 2, 3, 5):
-        time.sleep(wait_time)
-        complete_response = requests.post(complete_url, json=payload, headers=config.HEADERS)
-        log.debug('Complete response: %s. Content: %s', complete_response, complete_response.content)
-        if complete_response.status_code == 200:
-            break
-    else:
-        log.error(
-            'Did not receive a correct complete response: %s. Content %s',
-            complete_response, complete_response.content
-        )
-        raise
-
-    return complete_response.json()
+    session = requests.Session()
+    retries = Retry(total=7, backoff_factor=0.2, status_forcelist=[202], method_whitelist=frozenset(['POST']))
+    session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
+    response = session.post(complete_url, json=payload, headers=config.HEADERS)
+    if response.status_code != 200:
+        log.error('Did not receive a correct complete response: %s. Content %s', response, response.content)
+        raise Exception('Invalid complete response: {}'.format(response.content))
+
+    return response.json()
diff --git a/filestack/uploads/multipart.py b/filestack/uploads/multipart.py
@@ -33,7 +33,6 @@ def bytes(self):
 
 
 def multipart_request(url, payload, params=None, security=None):
-
     for key in ('path', 'location', 'region', 'container', 'access'):
         if key in params:
             payload['store'][key] = params[key]
diff --git a/filestack/utils.py b/filestack/utils.py
@@ -19,7 +19,7 @@ class RequestsWrapper:
     def __getattr__(self, name):
         if name in ('get', 'post', 'put', 'delete'):
             return partial(self.handle_request, name)
-        return super().__getattribute__(name)
+        return original_requests.__getattribute__(name)
 
     def handle_request(self, name, *args, **kwargs):
         if 'headers' not in kwargs:
@@ -43,7 +43,6 @@ def return_transform_task(transformation, params):
     transform_tasks = []
 
     for key, value in params.items():
-
         if isinstance(value, list):
             value = str(value).replace("'", "").replace('"', '').replace(" ", "")
         if isinstance(value, bool):
diff --git a/tests/intelligent_ingestion_test.py b/tests/intelligent_ingestion_test.py
@@ -90,31 +90,3 @@ def test_min_chunk_size_exception(post_mock, put_mock):
         upload_part(
             'Aaaaapikey', 'file.txt', 'tests/data/doom.mp4', 5415034, 's3', defaultdict(lambda: 'fs-upload.com'), part
         )
-
-
-@patch('filestack.uploads.intelligent_ingestion.time.sleep')
-@patch('filestack.uploads.intelligent_ingestion.requests.post')
-@patch('filestack.uploads.intelligent_ingestion.upload_part')
-def test_wait_for_complete(upload_part, post_mock, sleep_mock):
-    post_mock.side_effect = [
-        DummyHttpResponse(json_dict={
-            'uri': 'upload-uri', 'region': 'upload-region', 'upload_id': 'upload-id',
-            'location_url': 'upload-loc-url'
-        }),  # start response
-        DummyHttpResponse(status_code=202),
-        DummyHttpResponse(status_code=202),
-        DummyHttpResponse(status_code=202),
-        DummyHttpResponse(status_code=200)
-    ]
-    security = Security({'expires': 999}, 'secret')
-    upload_params = {'filename': 'new-filename.mp4', 'path': 'some/new/path'}
-    upload('AAApikeyz', 'tests/data/doom.mp4', None, 's3', upload_params, security)
-    assert post_mock.call_count == 5
-    start_resp_args, start_resp_kwargs = post_mock.call_args_list[0]
-    url = start_resp_args[0]
-    request_payload = start_resp_kwargs['json']
-    assert url == 'https://upload.filestackapi.com/multipart/start'
-    assert request_payload['store']['path'] == 'some/new/path'
-    assert request_payload['filename'] == 'new-filename.mp4'
-    assert request_payload['policy'] == 'eyJleHBpcmVzIjogOTk5fQ=='
-    assert request_payload['signature'] == 'c0b1b4d794f867287eedb34e477805aa7f5e1c9d1ec24fc55a085608b79e65fa'
diff --git a/tests/multipart_test.py b/tests/multipart_test.py
@@ -8,7 +8,7 @@
 
 from tests.helpers import DummyHttpResponse
 from filestack import Client
-from filestack.config import MULTIPART_START_URL
+from filestack import config
 from filestack.uploads.multipart import upload_chunk, Chunk
 
 APIKEY = 'APIKEY'
@@ -27,7 +27,7 @@ def test_upload_filepath():
 
     # add the different HTTP responses that are called during the multipart upload
     responses.add(
-        responses.POST, MULTIPART_START_URL, status=200, content_type='application/json',
+        responses.POST, config.MULTIPART_START_URL, status=200, content_type='application/json',
         json={'region': 'us-east-1', 'upload_id': 'someuuid', 'uri': 'someuri', 'location_url': 'fs-uploads.com'}
     )
     responses.add(
diff --git a/tests/transformation_test.py b/tests/transformation_test.py
@@ -259,6 +259,24 @@ def test_pdf_convert(transform):
     assert result.url == target_url
 
 
+def test_minify_css(transform):
+    target_url = '{}/{}/minify_css/{}'.format(config.CDN_URL, APIKEY, EXTERNAL_URL)
+    result = transform.minify_css()
+    assert result.url == target_url
+
+
+def test_minify_css_with_params(transform):
+    target_url = '{}/{}/minify_css=gzip:false,level:1/{}'.format(config.CDN_URL, APIKEY, EXTERNAL_URL)
+    result = transform.minify_css(level=1, gzip=False)
+    assert result.url == target_url
+
+
+def test_minify_js(transform):
+    target_url = '{}/{}/minify_js=gzip:false,targets:not dead,> 1%/{}'.format(config.CDN_URL, APIKEY, EXTERNAL_URL)
+    result = transform.minify_js(gzip=False, targets='not dead,> 1%')
+    assert result.url == target_url
+
+
 def quality(transform):
     target_url = '{}/{}/quality=value:75/{}'.format(config.CDN_URL, APIKEY, EXTERNAL_URL)
     quality = transform.quality(75)
@@ -292,10 +310,10 @@ def test_no_metadata(transform):
 def test_chain_tasks_and_store(post_mock, transform):
     post_mock.return_value = DummyHttpResponse(json_dict={'handle': HANDLE})
     transform_obj = transform.flip().resize(width=100)
-    new_filelink = transform_obj.store(filename='filename', location='S3', container='bucket')
+    new_filelink = transform_obj.store(filename='filename', location='S3', container='bucket', path='folder/image.jpg')
     assert new_filelink.handle == HANDLE
     post_mock.assert_called_once_with(
-        '{}/{}/flip/resize=width:100/store=container:bucket,filename:filename,location:S3/{}'.format(
+        '{}/{}/flip/resize=width:100/store=container:bucket,filename:filename,location:S3,path:"folder/image.jpg"/{}'.format(
             config.CDN_URL, APIKEY, EXTERNAL_URL
         )
     )
@@ -310,3 +328,9 @@ def test_av_convert(post_mock, transform):
     assert isinstance(new_av, AudioVisual)
     assert new_av.uuid == 'someuuid'
     assert new_av.timestamp == 'sometimestamp'
+
+
+def test_auto_image(transform):
+    target_url = '{}/{}/auto_image/{}'.format(config.CDN_URL, APIKEY, EXTERNAL_URL)
+    auto_image = transform.auto_image()
+    assert auto_image.url == target_url
diff --git a/tests/uploads/test_upload_external_url.py b/tests/uploads/test_upload_external_url.py
@@ -1,3 +1,4 @@
+import base64
 from unittest.mock import patch
 
 import pytest
@@ -8,6 +9,7 @@
 from filestack.uploads.external_url import upload_external_url
 
 url = 'http://image.url'
+encoded_url = 'b64://{}'.format(base64.urlsafe_b64encode(url.encode()).decode())
 apikey = 'TESTAPIKEY'
 
 
@@ -17,7 +19,7 @@ def test_upload(post_mock):
 
     handle = upload_external_url(url, apikey)
     assert handle == 'newHandle'
-    post_mock.assert_called_once_with('{}/{}/store/{}'.format(config.CDN_URL, apikey, url))
+    post_mock.assert_called_once_with('{}/{}/store/{}'.format(config.CDN_URL, apikey, encoded_url))
 
 
 @pytest.mark.parametrize('store_params, expected_store_task', [
@@ -47,7 +49,7 @@ def test_upload_with_security(post_mock):
     handle = upload_external_url(url, apikey, security=security)
     assert handle == 'newHandle'
     expected_url = '{}/{}/store/{}/{}'.format(
-        config.CDN_URL, apikey, security.as_url_string(), url
+        config.CDN_URL, apikey, security.as_url_string(), encoded_url
     )
     post_mock.assert_called_once_with(expected_url)
 
