Merge pull request #32 from sat-utils/develop

release 0.1.2

Author: matthewhanson

CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [v0.1.2] - 2019-02-14
+
+### Added
+- Item.download_assets() and Items.download_assets() works as download() except accepts a list of keys rather than a single key. Passing in `None` for keys will download all assets.
+- requestor_pays keyword option added to Item.download() (and Items.download()). Defaults to False. Use it to acknowledge paying egress costs when downloading data from a Reqeuestor Pays bucket (e.g., Sentinel-2). If the bucket is requestor pays and this is not set to True an AccessDenied error message will occur.
+
 ## [v0.1.1] - 2019-01-15
 
 ### Added
@@ -15,7 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 
 - Python 3 only. With Python 2.7 going unsupported in 2020 the time has come to stop supporting 2.7. There are too many additions in Python3 that continue to make backward compatability with Python 2.7 more difficult. In the case of this release the addition of caching using `functools` made sat-stac incompatible with Python 2.7.
-- More lenient version requirements for `requests` (now <=2.19.1). Otherwise can cause dependency incompatibility problems in some cases.
+- More lenient version requirements for `requests` (now >=2.19.1). Otherwise can cause dependency incompatibility problems in some cases.
 - Behavior of `path` and `filename` keyword arguments to Collection.add_item() has changed slightly. The components of `path` are now exclusively used to generate sub-catalogs, while `filename` is the relative filename (which could include a further subdirectory) from the last sub-catalog (it's parent). Before, it was assumed that Item files were always in a single subdirectory under it's parent catalog.
 - Tutorials updated
 
@@ -24,5 +30,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 Initial Release
 
 [Unreleased]: https://github.com/sat-utils/sat-stac/compare/master...develop
-[v0.1.1]: https://github.com/sat-utils/sat-api/compare/0.1.0...v0.1.1
-[v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0
+[v0.1.2]: https://github.com/sat-utils/sat-stac/compare/0.1.1...v0.1.2
+[v0.1.1]: https://github.com/sat-utils/sat-stac/compare/0.1.0...v0.1.1
+[v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0

README.md

@@ -33,7 +33,7 @@ $ pip install .
 The latest version of sat-stac is 0.1.1, which uses the STAC spec v0.6.0. To install other versions of sat-stac, install the matching version of sat-stac. 
 
 ```bash
-pip install satstac==0.1.0
+pip install sat-stac==0.1.0
 ```
 
 The table below shows the corresponding versions between sat-stac and STAC:

satstac/item.py

@@ -123,7 +123,16 @@ def substitute(self, string):
                 subs[key] = self[key.replace('_colon_', ':')]
         return Template(string).substitute(**subs)   
 
-    def download(self, key, overwrite=False, path='', filename='${id}'):
+    def download_assets(self, keys=None, **kwargs):
+        """ Download multiple assets """
+        if keys is None:
+            keys = self.data['assets'].keys()
+        filenames = []
+        for key in keys:
+            filenames.append(self.download(key, **kwargs))
+        return filenames
+
+    def download(self, key, overwrite=False, path='', filename='${id}', requestor_pays=False):
         """ Download this key (e.g., a band, or metadata file) from the scene """
         asset = self.asset(key)
         if asset is None:
@@ -137,7 +146,7 @@ def download(self, key, overwrite=False, path='', filename='${id}'):
             ext = os.path.splitext(asset['href'])[1]
             fout = os.path.join(_path, fname + '_' + key + ext)
             if not os.path.exists(fout) or overwrite:
-                _filename = utils.download_file(asset['href'], filename=fout)
+                _filename = utils.download_file(asset['href'], filename=fout, requestor_pays=requestor_pays)
             else:
                 _filename = fout
         except Exception as e:

satstac/items.py

@@ -118,6 +118,14 @@ def filter(self, key, values):
             items += list(filter(lambda x: x[key] == val, self._items))
         self._items = items
 
+    def download_assets(self, *args, **kwargs):
+        filenames = []
+        for i in self._items:
+            fnames = i.download_assets(*args, **kwargs)
+            if len(fnames) > 0:
+                filenames.append(fnames)
+        return filenames
+
     def download(self, *args, **kwargs):
         """ Download all Items """
         dls = []

satstac/utils.py

@@ -52,14 +52,14 @@ def dict_merge(dct, merge_dct, add_keys=True):
     return dct
 
 
-def download_file(url, filename=None):
+def download_file(url, filename=None, requestor_pays=False):
     """ Download a file as filename """
     filename = os.path.basename(url) if filename is None else filename
     logger.info('Downloading %s as %s' % (url, filename))
     headers = {}
     # check if on s3, if so try to sign it
     if 's3.amazonaws.com' in url:
-        signed_url, signed_headers = get_s3_signed_url(url)
+        signed_url, signed_headers = get_s3_signed_url(url, requestor_pays=requestor_pays)
         resp = requests.get(signed_url, headers=signed_headers, stream=True)
         if resp.status_code != 200:
             resp = requests.get(url, headers=headers, stream=True)

satstac/version.py

@@ -1 +1 @@
-__version__ = '0.1.1'
+__version__ = '0.1.2'

test/test_item.py

@@ -97,6 +97,13 @@ def test_download(self):
         fname = item.download(key='MTL', path=self.path)
         assert(os.path.exists(fname))
 
+    def test_download_assets(self):
+        """ Retrieve multiple data files """
+        item = Item.open(self.filename)
+        fnames = item.download_assets(keys=['MTL', 'ANG'], path=self.path)
+        for f in fnames:
+            assert(os.path.exists(f))
+
     def test_download_nonexist(self):
         """ Test downloading of non-existent file """
         item = Item.open(self.filename)

test/test_items.py

@@ -2,12 +2,21 @@
 import unittest
 
 from satstac import Items, Item
+from shutil import rmtree
 
 testpath = os.path.dirname(__file__)
 
 
 class Test(unittest.TestCase):
 
+    path = os.path.join(testpath, 'test-item')
+
+    @classmethod
+    def tearDownClass(cls):
+        """ Remove test files """
+        if os.path.exists(cls.path):
+            rmtree(cls.path)
+
     def load_items(self):
         return Items.load(os.path.join(testpath, 'items.json'))
 
@@ -89,14 +98,21 @@ def test_filter(self):
         items.filter('eo:cloud_cover', [100])
         assert(len(items) == 1)
 
+    def test_download_assets(self):
+        """ Download multiple assets from all items """
+        items = self.load_items()
+        filenames = items.download_assets(keys=['MTL', 'ANG'], path=self.path)
+        assert(len(filenames) == 2)
+        for fnames in filenames:
+            assert(len(fnames) == 2)
+            for f in fnames:
+                assert(os.path.exists(f))
+
     def test_download(self):
         """ Download a data file from all items """
         items = self.load_items()
 
-        fnames = items.download(key='MTL')
+        fnames = items.download(key='MTL', path=self.path)
         assert(len(fnames) == 2)
         for f in fnames:
             assert(os.path.exists(f))
-            os.remove(f)
-            assert(not os.path.exists(f))
-        #shutil.rmtree(os.path.join(testpath, 'landsat-8-l1'))