Merge pull request #8 from sat-utils/develop

matthewhanson · web-flow · commit 5611c8147f1b · 2019-01-22T17:19:09.000-05:00
deploy
diff --git a/README.md b/README.md
@@ -1,6 +1,78 @@
 # sat-stac-sentinel
 
-This is a Python repository for the creation of a STAC Sentinel catalog using the data and metadata files from s3://sentinel2-l1c/
+This is a repository used for the creation and maintenance of a [STAC](https://github.com/radiantearth/stac-spec) compliant [Sentinel catalog](https://sentinel-stac.s3.amazonaws.com/catalog.json) for data from the [Sentinel on AWS project](https://registry.opendata.aws/sentinel-2/) (located at s3://sentinel-s2-l1c/).
+
+There are two pieces of this repository:
+
+- A Python library (satstac.sentinel) and CLI containing functions for reading Sentinel metadata, transforming to STAC Items, and adding to the Sentinel catalog.
+- An AWS Lambda handler that accepts an SNS message containing the s3 URL for a new Sentinel scene, transforms it, and adds it to the catalog.
+
+To create the Sentinel STAC catalog located at https://sentinel-stac.s3.amazonaws.com/catalog.json the sat-stac-sentinel CLI was used to create the initial catalog of historical data. The Lambda function is deployed and keeping the catalog up to date with new scenes.
+
+## Installation
+
+
+
+## Usage
+
+A command line tool is available for ingesting the existing Sentinel data on s3 and creating/adding to a STAC catalog.
+
+```bash
+$ sat-stac-sentinel -h
+usage: sat-stac-sentinel [-h] {ingest,inventory} ...
+
+sat-stac-sentinel (v0.1.0)
+
+positional arguments:
+  {ingest,inventory}
+    ingest            Ingest records into catalog
+    inventory         Get latest inventory of tileInfo.json files
+
+optional arguments:
+  -h, --help          show this help message and exit
+```
+
+There are two available commands:
+
+### `inventory`
+
+This will fetch the latest inventory files from s3://sentinel-inventory/sentinel-s2-l1c/sentinel-s2-l1c-inventory and save to a local file. This isn't necessary, files can be directly ingested from the latest inventory files, but saving the file first allows it to be broken up and run with several jobs.
+
+### `ingest`
+
+This will ingest records either from a local inventory file or, if not provided, the latest bucket inventory files
+
+
+The `catalog` argument is the URL to the root catalog which contains a child collection called 'sentinel-2-l1c'. If the 'sentinel-2-l1c' Collection does not exist in the Catalog it will be added. In the case of the catalog maintained by this repo it is located at https://sentinel-stac.s3.amazonaws.com/catalog.json.
+
+If `start` and/or `end` are provided the records are all scanned and only those meeting the date requirements are ingested.
+
+
+## Transforming Sentinel metadata to STAC
+
+The data that is ingested by the sat-stac-sentinel CLI starts with
+
+In addition to the inventories, an SNS message is published (arn:aws:sns:us-west-2:274514004127:NewSceneHTML) whenever a new `index.html` appears in the bucket. The sat-stac-sentinel Lambda function listens for this message to get the link of the s3 path with the new scene.
+
+
+
+
+## Development
+
+The `master` branch is the latest versioned release, while the `develop` branch is the latest development version. When making a new release:
+
+- Update the [version](satstac.sentinel.version.py)
+- Update [CHANGELOG.md](CHANGELOG.md)
+- Create PR and merge to master
+- Create new tag with the version and push to GitHub:
+
+```bash
+$ git tag `<version>`
+$ git push origin `<version>`
+```
+
+On a release (merge to `master`) CircleCI will package the Lambda code and deploy it to the production Lambda function that listens (via SNS) for new Sentinel scenes, creates STAC Items and adds them to the Catalog.
+
 
 ## About
 [sat-stac-sentinel](https://github.com/sat-utils/sat-stac-sentinel) was created by [Development Seed](<http://developmentseed.org>) and is part of a collection of tools called [sat-utils](https://github.com/sat-utils).
diff --git a/lambda/lambda_function.py b/lambda/lambda_function.py
@@ -35,7 +35,7 @@ def lambda_handler(event, context):
         # transform to STAC
         item = transform(metadata)
         logger.info('Item: %s' % json.dumps(item.data))
-        collection.add_item(item, path=SETTINGS['path_pattern'], filename=SETTINGS['fname_pattern'])
-        logger.info('Added %s as %s' % (item, item.filename))
+        #collection.add_item(item, path=SETTINGS['path_pattern'], filename=SETTINGS['fname_pattern'])
+        #logger.info('Added %s as %s' % (item, item.filename))
         client.publish(TopicArn=sns_arn, Message=json.dumps(item.data))
         logger.info('Published to %s' % sns_arn)
diff --git a/satstac/sentinel/cli.py b/satstac/sentinel/cli.py
@@ -35,11 +35,16 @@ def parse_args(args):
     valid_date = lambda d: datetime.strptime(d, '%Y-%m-%d').date()
     parser.add_argument('--start', help='Start date of ingestion', default=None, type=valid_date)
     parser.add_argument('--end', help='End date of ingestion', default=None, type=valid_date)
+    parser.add_argument('--prefix', help='Only ingest scenes with a path starting with prefix', default=None)
     parser.add_argument('--s3meta', help='Get metadata directly from S3 (requestor pays)', default=False, action='store_true')
+    parser.add_argument('--filename', help='Inventory filename to use (default to fetch latest from bucket Inventory files)', default=None)
+    parser.add_argument('--publish', help='ARN to publish new Items to', default=None)
 
     # command 2
-    #parser = subparsers.add_parser('cmd2', parents=[pparser], help='Command 2', formatter_class=dhf)
-    # parser.add_argument()
+    h = 'Get latest inventory of tileInfo.json files'
+    parser = subparsers.add_parser('inventory', parents=[pparser], help=h, formatter_class=dhf)
+    fout = str(datetime.now().date()) + '.csv'
+    parser.add_argument('--filename', help='Filename to save', default=fout)
 
     # turn Namespace into dictinary
     parsed_args = vars(parser0.parse_args(args))
@@ -54,7 +59,16 @@ def cli():
 
     if cmd == 'ingest':
         cat = Catalog.open(args['catalog'])
-        sentinel.add_items(cat, start_date=args['start'], end_date=args['end'], s3meta=args['s3meta'])
+        if args['filename'] is not None:
+            records = sentinel.read_inventory(args['filename'])
+        else:
+            records = sentinel.latest_inventory()
+        sentinel.add_items(cat, records, start_date=args['start'], end_date=args['end'],
+                           prefix=args['prefix'], s3meta=args['s3meta'], publish=args['publish'])
+    elif cmd == 'inventory':
+        with open(args['filename'], 'w') as f:
+            f.write('datetime,path\n')
+            [f.write('%s,%s\n' % (i['datetime'], i['path'])) for i in sentinel.latest_inventory()]
 
 
 if __name__ == "__main__":
diff --git a/satstac/sentinel/main.py b/satstac/sentinel/main.py
@@ -34,7 +34,7 @@
 }
 
 
-def add_items(catalog, start_date=None, end_date=None, s3meta=False):
+def add_items(catalog, records, start_date=None, end_date=None, s3meta=False, prefix=None, publish=None):
     """ Stream records to a collection with a transform function 
     
     Keyword arguments:
@@ -50,17 +50,26 @@ def add_items(catalog, start_date=None, end_date=None, s3meta=False):
         cols = {c.id: c for c in catalog.collections()}
     collection = cols['sentinel-2-l1c']
 
+    client = None
+    if publish:
+        parts = publish.split(':')
+        client = boto3.client('sns', region_name=parts[3])
+
     duration = []
     # iterate through records
-    for i, record in enumerate(records()):
+    for i, record in enumerate(records):
         start = datetime.now()
+        if i % 50000 == 0:
+            logger.info('%s: Scanned %s records' % (start, str(i)))
         dt = record['datetime'].date()
+        if prefix is not None:
+            # if path doesn't match provided prefix skip to next record
+            if record['path'][:len(prefix)] != prefix:
+                continue
         if s3meta:
             url = op.join(SETTINGS['s3_url'], record['path'])
         else:
             url = op.join(SETTINGS['roda_url'], record['path'])
-        if i % 10000 == 0:
-            print('Scanned %s records' % str(i+1))
         #if i == 10:
         #    break
         if (start_date is not None and dt < start_date) or (end_date is not None and dt > end_date):
@@ -79,13 +88,28 @@ def add_items(catalog, start_date=None, end_date=None, s3meta=False):
             continue
         try:
             collection.add_item(item, path=SETTINGS['path_pattern'], filename=SETTINGS['fname_pattern'])
+            if client:
+                client.publish(TopicArn=publish, Message=json.dumps(item.data))
             duration.append((datetime.now()-start).total_seconds())
             logger.info('Ingested %s in %s' % (item.filename, duration[-1]))
         except Exception as err:
             logger.error('Error adding %s: %s' % (item.id, err))
     logger.info('Read in %s records averaging %4.2f sec (%4.2f stddev)' % (i, np.mean(duration), np.std(duration)))
 
-def records():
+
+def read_inventory(filename):
+    """ Create generator from inventory file """
+    with open(filename) as f:
+        f.readline()
+        for line in f.readlines():
+            parts = line.split(',')
+            yield {
+                'datetime': parse(parts[0]),
+                'path': parts[1].strip('\n')
+            }
+
+
+def latest_inventory():
     """ Return generator function for list of scenes """
     s3 = boto3.client('s3')
     # get latest file
diff --git a/satstac/sentinel/sentinel-2-l1c.json b/satstac/sentinel/sentinel-2-l1c.json
@@ -59,86 +59,86 @@
         "eo:off_nadir": 0,
         "eo:bands": [
             {
-                "id": "B01",
+                "name": "B01",
                 "common_name": "coastal",
                 "gsd": 60.0,
                 "center_wavelength": 0.4439,
                 "full_width_half_max": 0.027
             },
             {
-                "id": "B02",
+                "name": "B02",
                 "common_name": "blue",
                 "gsd": 10.0,
                 "center_wavelength": 0.4966,
                 "full_width_half_max": 0.098
             },
             {
-                "id": "B03",
+                "name": "B03",
                 "common_name": "green",
                 "gsd": 10.0,
                 "center_wavelength": 0.56,
                 "full_width_half_max": 0.045
             },
             {
-                "id": "B04",
+                "name": "B04",
                 "common_name": "red",
                 "gsd": 10.0,
                 "center_wavelength": 0.6645,
                 "full_width_half_max": 0.038
             },
             {
-                "id": "B05",
+                "name": "B05",
                 "gsd": 20.0,
                 "center_wavelength": 0.7039,
                 "full_width_half_max": 0.019
             },
             {
-                "id": "B06",
+                "name": "B06",
                 "gsd": 20.0,
                 "center_wavelength": 0.7402,
                 "full_width_half_max": 0.018
             },
             {
-                "id": "B07",
+                "name": "B07",
                 "gsd": 20.0,
                 "center_wavelength": 0.7825,
                 "full_width_half_max": 0.028
             },
             {
-                "id": "B08",
+                "name": "B08",
                 "common_name": "nir",
                 "gsd": 10.0,
                 "center_wavelength": 0.8351,
                 "full_width_half_max": 0.145
             },
             {
-                "id": "B8A",
+                "name": "B8A",
                 "gsd": 20.0,
                 "center_wavelength": 0.8648,
                 "full_width_half_max": 0.033
             },
             {
-                "id": "B09",
+                "name": "B09",
                 "gsd": 60.0,
                 "center_wavelength": 0.945,
                 "full_width_half_max": 0.026
             },
             {
-                "id": "B10",
+                "name": "B10",
                 "common_name": "cirrus",
                 "gsd": 60.0,
                 "center_wavelength": 1.3735,
                 "full_width_half_max": 0.075
             },
             {
-                "id": "B11",
+                "name": "B11",
                 "common_name": "swir16",
                 "gsd": 20.0,
                 "center_wavelength": 1.6137,
                 "full_width_half_max": 0.143
             },
             {
-                "id": "B12",
+                "name": "B12",
                 "common_name": "swir22",
                 "gsd": 20.0,
                 "center_wavelength": 2.22024,
diff --git a/test/test_main.py b/test/test_main.py
@@ -26,7 +26,7 @@ def test_main(self):
         #fout = sentinel.main(sentinel, start_date=dt(2013, 10, 1).date())
 
     def test_records(self):
-        for r in sentinel.records():
+        for r in sentinel.latest_inventory():
             assert('datetime' in r)
             assert('path' in r)
             break
