Merge pull request #17 from jkeifer/null-bucket-location

Handle us-east-1 null bucket location

Author: matthewhanson

.github/workflows/continuous-integration.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
     steps:
       - uses: actions/checkout@v2
 
@@ -36,4 +36,4 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml
-          fail_ci_if_error: false 
+          fail_ci_if_error: false

CHANGELOG.md

@@ -6,13 +6,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
-## [v0.3.3] - 2022-05-18
+## [v0.3.3] - 2022-09-14
 
 ### Added
 - s3.latest_inventory() takes manifest_age_days argument for how far back to look for manifest
 - s3.latest_inventory() takes is_latest argument for filtering on versioned files
 - s3.latest_inventory() takes key_contains array argument for filtering on key containing strings of the array
 
+### Changed
+- No longer testing against python 3.6
+- Now testing against python 3.9 and 3.10
+
+### Fixed
+- s3.delete works
+- Handle bucket locations in US Standard region
+
 ## [v0.3.2] - 2021-07-15
 
 ### Added

boto3utils/s3.py

@@ -76,6 +76,13 @@ def exists(self, url):
                 raise
             return False
 
+    def get_bucket_region(self, bucket_name):
+        region = self.s3.get_bucket_location(
+            Bucket=bucket_name)['LocationConstraint']
+        # US Standard region buckets will have a null location
+        # https://github.com/aws/aws-cli/issues/3864
+        return region if region else 'us-east-1'
+
     def upload(self, filename, url, public=False, extra={}, http_url=False):
         """ Upload object to S3 uri (bucket + prefix), keeping same base filename """
         logger.debug("Uploading %s to %s" % (filename, url))
@@ -89,9 +96,8 @@ def upload(self, filename, url, public=False, extra={}, http_url=False):
                                    parts['key'],
                                    ExtraArgs=extra)
         if http_url:
-            region = self.s3.get_bucket_location(
-                Bucket=parts['bucket'])['LocationConstraint']
-            return self.s3_to_https(url_out, region)
+            return self.s3_to_https(url_out,
+                                    self.get_bucket_region(parts['bucket']))
         else:
             return url_out
 
@@ -156,8 +162,8 @@ def read_json(self, url):
     def delete(self, url):
         """ Remove object from S3 """
         parts = self.urlparse(url)
-        response = self.s3.delete_object(Bucket=parts['Bucket'],
-                                         Key=parts['Key'])
+        response = self.s3.delete_object(Bucket=parts['bucket'],
+                                         Key=parts['key'])
         return response
 
     # function derived from https://alexwlchan.net/2018/01/listing-s3-keys-redux/
@@ -234,7 +240,8 @@ def fenddate(info):
             return True if dt < end_date else False
 
         def islatest(info):
-            if latest := info.get("IsLatest"):
+            latest = info.get("IsLatest")
+            if latest:
                 return latest not in ("false", False)
             return True
 
@@ -308,7 +315,8 @@ def latest_inventory(self, url, **kwargs):
                 str(key).strip() for key in manifest['fileSchema'].split(',')
             ]
 
-            for i, url in enumerate(self.latest_inventory_files(url, manifest)):
+            for i, url in enumerate(self.latest_inventory_files(url,
+                                                                manifest)):
                 logger.info('Reading inventory file %s' % (i + 1))
                 results = self.read_inventory_file(url, keys, **kwargs)
                 yield from results

tests/conftest.py

@@ -0,0 +1,37 @@
+import os
+
+import moto
+import boto3
+import pytest
+
+if 'AWS_DEFAULT_REGION' not in os.environ:
+    os.environ['AWS_DEFAULT_REGION'] = 'us-east-1'
+
+
+@pytest.fixture
+def aws_credentials():
+    """Mocked AWS Credentials for moto."""
+    os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
+    os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
+    os.environ['AWS_SECURITY_TOKEN'] = 'testing'
+    os.environ['AWS_SESSION_TOKEN'] = 'testing'
+    os.environ['AWS_DEFAULT_REGION'] = 'us-east-1'
+    os.environ['AWS_REGION'] = 'us-east-1'
+
+
+@pytest.fixture
+def s3(aws_credentials):
+    with moto.mock_s3():
+        yield boto3.client('s3', region_name='us-east-1')
+
+
+@pytest.fixture
+def s3_west(aws_credentials):
+    with moto.mock_s3():
+        yield boto3.client('s3', region_name='us-west-2')
+
+
+@pytest.fixture
+def secretsmanager(aws_credentials):
+    with moto.mock_secretsmanager():
+        yield boto3.client('secretsmanager', region_name='us-east-1')

tests/test_s3.py

@@ -1,6 +1,3 @@
-# this must be imported before any boto3 module
-from moto import mock_s3
-
 import boto3
 import os
 import pytest
@@ -9,22 +6,37 @@
 from shutil import rmtree
 
 BUCKET = 'testbucket'
+BUCKET_WEST = 'testbucket_west'
 KEY = 'testkey'
 
 
-@pytest.fixture(scope='function')
-def s3mock():
-    with mock_s3():
-        client = boto3.client('s3',
-                              region_name='us-east-1',
-                              aws_access_key_id='noid',
-                              aws_secret_access_key='nokey')
-        client.create_bucket(Bucket=BUCKET)
-        client.put_object(Body='helloworld', Bucket=BUCKET, Key=KEY)
-        client.upload_file(Filename=os.path.join(testpath, 'test.json'),
-                           Bucket=BUCKET,
-                           Key='test.json')
-        yield client
+def create_test_bucket(s3, bucket):
+    params = {
+        'Bucket': bucket,
+    }
+
+    if s3.meta.region_name != 'us-east-1':
+        params['CreateBucketConfiguration'] = {
+            'LocationConstraint': s3.meta.region_name,
+        }
+
+    s3.create_bucket(**params)
+    s3.put_object(Body='helloworld', Bucket=bucket, Key=KEY)
+    s3.upload_file(Filename=os.path.join(testpath, 'test.json'),
+                   Bucket=bucket,
+                   Key='test.json')
+
+
+@pytest.fixture
+def s3mock(s3):
+    create_test_bucket(s3, BUCKET)
+    yield s3
+
+
+@pytest.fixture
+def s3mock_west(s3_west):
+    create_test_bucket(s3_west, BUCKET_WEST)
+    yield s3_west
 
 
 testpath = os.path.dirname(__file__)
@@ -55,6 +67,16 @@ def test_s3_to_https():
     assert (url == 'https://bucket.s3.us-west-2.amazonaws.com/prefix/filename')
 
 
+def test_get_bucket_region_null(s3mock):
+    region = s3().get_bucket_region(BUCKET)
+    assert region == 'us-east-1'
+
+
+def test_get_bucket_region(s3mock_west):
+    region = s3().get_bucket_region(BUCKET_WEST)
+    assert region == 'us-west-2'
+
+
 def test_exists(s3mock):
     exists = s3().exists('s3://%s/%s' % (BUCKET, 'keymaster'))
     assert (exists is False)
@@ -85,6 +107,12 @@ def test_read_json(s3mock):
     assert (out['field'] == 'value')
 
 
+def test_delete(s3mock):
+    url = 's3://%s/test.json' % BUCKET
+    out = s3().delete(url)
+    assert (out['ResponseMetadata']['HTTPStatusCode'] == 204)
+
+
 def test_find(s3mock):
     url = 's3://%s/test' % BUCKET
     urls = list(s3().find(url))
@@ -93,10 +121,17 @@ def test_find(s3mock):
 
 
 def test_latest_inventory():
+    from botocore.handlers import disable_signing
+
     url = 's3://sentinel-inventory/sentinel-s1-l1c/sentinel-s1-l1c-inventory'
     suffix = 'productInfo.json'
     session = boto3.Session()
     _s3 = s3(session)
+
+    # as we are actually hitting S3 (which is not great), we need
+    # to prevent signing our request with the dummy test credentials
+    _s3.s3.meta.events.register('choose-signer.s3.*', disable_signing)
+
     for url in _s3.latest_inventory(url, suffix=suffix):
         # dt = datetime.strptime(f['LastModifiedDate'], "%Y-%m-%dT%H:%M:%S.%fZ")
         # hours = (datetime.today() - dt).seconds // 3600

tests/test_secrets.py

@@ -1,45 +1,44 @@
-# this must be imported before any boto3 module
-from moto import mock_secretsmanager
-
-import os
-import boto3
 import pytest
 import json
 import base64
 
 from boto3utils import secrets
 from botocore.exceptions import ClientError
 
-testpath = os.path.dirname(__file__)
-
 SECRET_NAME = 'secret'
 SECRET = {'mock_key': 'mock_val'}
 SECRET_STRING = json.dumps(SECRET)
 SECRET_BINARY = base64.b64encode(SECRET_STRING.encode())
 
 
-@mock_secretsmanager
-def test_get_secret_string():
-    boto3.session.Session().client('secretsmanager',
-                                   region_name='us-west-2').create_secret(
-                                       Name=SECRET_NAME,
-                                       SecretString=SECRET_STRING)
+@pytest.fixture
+def secret(secretsmanager):
+    secretsmanager.create_secret(
+        Name=SECRET_NAME,
+        SecretString=SECRET_STRING,
+    )
+    return secretsmanager
+
+
+@pytest.fixture
+def binary_secret(secretsmanager):
+    secretsmanager.create_secret(
+        Name=SECRET_NAME,
+        SecretBinary=SECRET_BINARY,
+    )
+    return secretsmanager
+
+
+def test_get_secret_string(secret):
     secret = secrets.get_secret(SECRET_NAME)
     assert (secret == SECRET)
 
 
-@mock_secretsmanager
-def test_get_secret_undef():
+def test_get_secret_undef(secretsmanager):
     with pytest.raises(ClientError):
         secrets.get_secret(SECRET_NAME)
 
 
-@mock_secretsmanager
-def test_get_secret_binary():
-    boto3.session.Session().client('secretsmanager',
-                                   region_name='us-west-2').create_secret(
-                                       Name=SECRET_NAME,
-                                       SecretBinary=SECRET_BINARY)
+def test_get_secret_binary(binary_secret):
     secret = secrets.get_secret(SECRET_NAME)
     assert (secret == SECRET)
-    # client.create_secret(Name=SECRET_NAME, SecretBinary=SECRET_BINARY)