Merge pull request #509 from stac-utils/jcw/extened-sns-attributes

Author: jwalgran

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - Publish ingest results to a post-ingest SNS topic
+- Add datetime and bbox attributes to post-ingest SNS messages
 
 ### Changed
 

src/lib/sns.js

@@ -1,6 +1,6 @@
 import { sns } from './aws-clients.js'
 import logger from './logger.js'
-import { isCollection, isItem } from './stac-utils.js'
+import { getBBox, getStartAndEndDates, isCollection, isItem } from './stac-utils.js'
 
 const attrsFromPayload = function (payload) {
   let type = 'unknown'
@@ -13,7 +13,7 @@ const attrsFromPayload = function (payload) {
     collection = payload.record.collection || ''
   }
 
-  return {
+  const attributes = {
     recordType: {
       DataType: 'String',
       StringValue: type
@@ -25,8 +25,61 @@ const attrsFromPayload = function (payload) {
     collection: {
       DataType: 'String',
       StringValue: collection
+    },
+  }
+
+  const bbox = getBBox(payload.record)
+  if (bbox) {
+    attributes['bbox.sw_lon'] = {
+      DataType: 'Number',
+      StringValue: bbox[0].toString(),
+    }
+    attributes['bbox.sw_lat'] = {
+      DataType: 'Number',
+      StringValue: bbox[1].toString(),
+    }
+    attributes['bbox.ne_lon'] = {
+      DataType: 'Number',
+      StringValue: bbox[2].toString(),
+    }
+    attributes['bbox.ne_lat'] = {
+      DataType: 'Number',
+      StringValue: bbox[3].toString(),
+    }
+  }
+
+  if (payload.record?.properties?.datetime) {
+    attributes.datetime = {
+      DataType: 'String',
+      StringValue: payload.record.properties.datetime
+    }
+  }
+
+  const { startDate, endDate } = getStartAndEndDates(payload.record)
+
+  if (startDate) {
+    attributes.start_datetime = {
+      DataType: 'String',
+      StringValue: startDate.toISOString()
+    }
+    attributes.start_unix_epoch_ms_offset = {
+      DataType: 'Number',
+      StringValue: startDate.getTime().toString()
     }
   }
+
+  if (endDate) {
+    attributes.end_datetime = {
+      DataType: 'String',
+      StringValue: endDate.toISOString()
+    }
+    attributes.end_unix_epoch_ms_offset = {
+      DataType: 'Number',
+      StringValue: endDate.getTime().toString()
+    }
+  }
+
+  return attributes
 }
 
 /* eslint-disable-next-line import/prefer-default-export */

src/lib/stac-utils.js

@@ -1,7 +1,62 @@
+import logger from './logger.js'
+
 export function isCollection(record) {
   return record && record.type === 'Collection'
 }
 
 export function isItem(record) {
   return record && record.type === 'Feature'
 }
+
+export function getStartAndEndDates(record) {
+  let startDate
+  let endDate
+
+  if (isCollection(record)) {
+    const interval = record.extent?.temporal?.interval || [[null, null]]
+    if (!interval) {
+      logger.info(
+        `Missing extent.temporal.interval in record ${record.id}`
+      )
+    }
+    // STAC spec 1.0.0 says
+    // "The first time interval always describes the overall temporal extent of the data"
+    // Nulls are allows for open-ended intervals and nulls for both ends is not forbidden
+    const [intervalStart, intervalEnd] = interval.length > 0 ? interval[0] : [null, null]
+    if (intervalStart) {
+      startDate = new Date(intervalStart)
+    }
+    if (intervalEnd) {
+      endDate = new Date(intervalEnd)
+    }
+  } else if (isItem(record)) {
+    const properties = record.properties || {}
+    if (properties.start_datetime && properties.end_datetime) {
+      startDate = new Date(properties.start_datetime)
+      endDate = new Date(properties.end_datetime)
+    } else if (properties.datetime) {
+      startDate = new Date(properties.datetime)
+      endDate = startDate
+    } else {
+      const propertiesString = JSON.stringify(properties)
+      logger.info(
+        `Missing properties in record ${record.id}:
+        Expected datetime or both start_datetime and end_datetime in ${propertiesString}`
+      )
+    }
+  }
+
+  return { startDate, endDate }
+}
+
+export function getBBox(record) {
+  if (isCollection(record)) {
+    return record.extent?.spatial?.bbox?.length > 0
+      ? record.extent.spatial.bbox[0]
+      : undefined
+  }
+  if (isItem(record)) {
+    return record.bbox || undefined
+  }
+  return undefined
+}

tests/system/test-ingest.js

@@ -348,6 +348,22 @@ test('Ingested collection is published to post-ingest SNS topic', async (t) => {
   t.is(attrs.collection.Value, collection.id)
   t.is(attrs.ingestStatus.Value, 'successful')
   t.is(attrs.recordType.Value, 'Collection')
+
+  const bbox = collection.extent.spatial.bbox[0]
+  t.is(bbox[0].toString(), attrs['bbox.sw_lon'].Value)
+  t.is(bbox[1].toString(), attrs['bbox.sw_lat'].Value)
+  t.is(bbox[2].toString(), attrs['bbox.ne_lon'].Value)
+  t.is(bbox[3].toString(), attrs['bbox.ne_lat'].Value)
+
+  const expectedStartOffsetValue = (new Date(collection.extent.temporal.interval[0][0]))
+    .getTime().toString()
+  t.is(expectedStartOffsetValue, attrs.start_unix_epoch_ms_offset.Value)
+  t.is(
+    (new Date(collection.extent.temporal.interval[0][0])).toISOString(),
+    attrs.start_datetime.Value
+  )
+  t.is(undefined, attrs.end_unix_epoch_ms_offset)
+  t.is(undefined, attrs.end_datetime)
 })
 
 test('Ingested collection is published to post-ingest SNS topic with updated links', async (t) => {
@@ -382,6 +398,10 @@ test('Ingest collection failure is published to post-ingest SNS topic', async (t
   t.is(attrs.collection.Value, 'badCollection')
   t.is(attrs.ingestStatus.Value, 'failed')
   t.is(attrs.recordType.Value, 'Collection')
+  t.is(undefined, attrs.start_unix_epoch_ms_offset)
+  t.is(undefined, attrs.start_datetime)
+  t.is(undefined, attrs.end_unix_epoch_ms_offset)
+  t.is(undefined, attrs.end_datetime)
 })
 
 async function emptyPostIngestQueue(t) {
@@ -422,16 +442,37 @@ test('Ingested item is published to post-ingest SNS topic', async (t) => {
 
   const item = await loadFixture(
     'stac/ingest-item.json',
-    { id: randomId('item'), collection: collection.id }
+    {
+      id: randomId('item'),
+      collection: collection.id
+    }
   )
 
+  item.properties.start_datetime = '1955-11-05T13:00:00.000Z'
+  item.properties.end_datetime = '1985-11-05T13:00:00.000Z'
+
   const { message, attrs } = await testPostIngestSNS(t, item)
 
   t.is(message.record.id, item.id)
   t.deepEqual(message.record.links, item.links)
   t.is(attrs.collection.Value, item.collection)
   t.is(attrs.ingestStatus.Value, 'successful')
   t.is(attrs.recordType.Value, 'Item')
+
+  t.is(item.bbox[0].toString(), attrs['bbox.sw_lon'].Value)
+  t.is(item.bbox[1].toString(), attrs['bbox.sw_lat'].Value)
+  t.is(item.bbox[2].toString(), attrs['bbox.ne_lon'].Value)
+  t.is(item.bbox[3].toString(), attrs['bbox.ne_lat'].Value)
+
+  t.is(message.record.properties.datetime, attrs.datetime.Value)
+
+  const expectedStartOffsetValue = (new Date(item.properties.start_datetime)).getTime().toString()
+  t.is(expectedStartOffsetValue, attrs.start_unix_epoch_ms_offset.Value)
+  t.is(item.properties.start_datetime, attrs.start_datetime.Value)
+
+  const expectedEndOffsetValue = (new Date(item.properties.end_datetime)).getTime().toString()
+  t.is(expectedEndOffsetValue, attrs.end_unix_epoch_ms_offset.Value)
+  t.is(item.properties.end_datetime, attrs.end_datetime.Value)
 })
 
 test('Ingest item failure is published to post-ingest SNS topic', async (t) => {

tests/unit/test-stac-utils.js

@@ -0,0 +1,137 @@
+// @ts-nocheck
+
+import test from 'ava'
+import { getStartAndEndDates } from '../../src/lib/stac-utils.js'
+
+test('getStartandEndDates uses item datetime', (t) => {
+  const stringDate = '1955-11-05T13:00:00Z'
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Feature',
+    id: 'test',
+    properties: {
+      datetime: stringDate
+    }
+  })
+  const date = new Date(stringDate)
+  t.deepEqual(date, startDate, 'startDate did not match datetime')
+  t.deepEqual(date, endDate, 'endDate did not match datetime')
+})
+
+test('getStartandEndDates uses item start_datetime and end_datetime', (t) => {
+  const datetime = '1955-11-05T13:00:00Z'
+  const startDatetime = '1985-11-05T13:00:00Z'
+  const endDatetime = '2015-11-05T13:00:00Z'
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Feature',
+    id: 'test',
+    properties: {
+      datetime,
+      start_datetime: startDatetime,
+      end_datetime: endDatetime,
+    }
+  })
+  t.deepEqual(new Date(startDatetime), startDate, 'startDate did not match start_datetime')
+  t.deepEqual(new Date(endDatetime), endDate, 'endDate did not match end_datetime')
+})
+
+test('getStartandEndDates uses item start_datetime and end_datetime with null datetime', (t) => {
+  const startDatetime = '1985-11-05T13:00:00Z'
+  const endDatetime = '2015-11-05T13:00:00Z'
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Feature',
+    id: 'test',
+    properties: {
+      datetime: null,
+      start_datetime: startDatetime,
+      end_datetime: endDatetime,
+    }
+  })
+  t.deepEqual(new Date(startDatetime), startDate, 'startDate did not match start_datetime')
+  t.deepEqual(new Date(endDatetime), endDate, 'endDate did not match end_datetime')
+})
+
+test('getStartandEndDates returns undefineds if item datetime is null', (t) => {
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Feature',
+    id: 'test',
+    properties: {
+      datetime: null
+    }
+  })
+  t.deepEqual(undefined, startDate, 'startDate is not undefined')
+  t.deepEqual(undefined, endDate, 'endDate is not undefined')
+})
+
+test('getStartandEndDates returns undefineds if collection interval is missing', (t) => {
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Collection',
+    id: 'test',
+    extent: {
+      temporal: {
+        interval: []
+      }
+    }
+  })
+  t.deepEqual(undefined, startDate, 'startDate is not undefined')
+  t.deepEqual(undefined, endDate, 'endDate is not undefined')
+})
+
+test('getStartandEndDates returns startDate if collection interval has start', (t) => {
+  const startDatetime = '1985-11-05T13:00:00Z'
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Collection',
+    id: 'test',
+    extent: {
+      temporal: {
+        interval: [[startDatetime, null]]
+      }
+    }
+  })
+  t.deepEqual(new Date(startDatetime), startDate, 'startDate was not returned')
+  t.deepEqual(undefined, endDate, 'endDate is not undefined')
+})
+
+test('getStartandEndDates returns endDate if collection interval has end', (t) => {
+  const endDatetime = '1985-11-05T13:00:00Z'
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Collection',
+    id: 'test',
+    extent: {
+      temporal: {
+        interval: [[null, endDatetime]]
+      }
+    }
+  })
+  t.deepEqual(undefined, startDate, 'startDate is not undefined')
+  t.deepEqual(new Date(endDatetime), endDate, 'endDate was not returned')
+})
+
+test('getStartandEndDates returns collection interval', (t) => {
+  const startDatetime = '1955-11-05T13:00:00Z'
+  const endDatetime = '1985-11-05T13:00:00Z'
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Collection',
+    id: 'test',
+    extent: {
+      temporal: {
+        interval: [[startDatetime, endDatetime]]
+      }
+    }
+  })
+  t.deepEqual(new Date(startDatetime), startDate, 'startDate was not returned')
+  t.deepEqual(new Date(endDatetime), endDate, 'endDate was not returned')
+})
+
+test('getStartandEndDates only looks at first collection interval', (t) => {
+  const { startDate, endDate } = getStartAndEndDates({
+    type: 'Collection',
+    id: 'test',
+    extent: {
+      temporal: {
+        interval: [[null, null], ['1955-11-05T13:00:00Z', '1985-11-05T13:00:00Z']]
+      }
+    }
+  })
+  t.deepEqual(undefined, startDate, 'startDate is not undefined')
+  t.deepEqual(undefined, endDate, 'endDate is not undefined')
+})