Merge pull request #101 from kuzzleio/0.4.0-proposal

# [0.4.0](https://github.com/kuzzleio/kuzzle-plugin-device-manager/releases/tag/0.4.0) (2021-10-06)


#### New features

- [ [#100](#100) ] Add historization   ([Aschen](https://github.com/Aschen))
---

Author: Aschen

doc/1/guides/assets/asset-data-model.png

@@ -31,7 +31,7 @@ An asset is uniquely identified by the `type` + `model` + `reference` triplet.
       "qos": {
         "battery": 2.3
       },
-      
+
       "latitude": 41.074688,
       "longitude": 28.9800192,
       "accuracy": 42,
@@ -85,3 +85,27 @@ By default, for each measurement type the following information are copied in ad
 ```
 
 It is possible to override the [Decoder.copyToAsset](/official-plugins/device-manager/1/classes/decoder/copy-to-asset) method to choose what to copy into the asset.
+
+## Historization
+
+Assets are historized in the `assets-history` collection when a new measure is received.
+
+Before historization, the `tenant:<tenant-id>:asset:measure:new` event is emitted. This event allows to enrich the asset before it is historized.
+
+The payload contains the new `measures` and the `assetId`.
+
+The content of `request.result.asset` will be used to update the asset measures and then create the history document.
+
+```js
+app.pipe.register(`tenant:<tenant-id>:asset:measures:new`, async (request: KuzzleRequest) => {
+  const measures: AssetMeasures = request.result.asset.measures;
+  const assetId: string = request.result.assetId;
+
+  request.result.asset.metadata = {
+    enriched: true,
+    assetId: request.result.assetId
+  };
+
+  return request;
+});
+```

doc/1/guides/assets/index.md

@@ -5,7 +5,7 @@ Feature: Engine Controller
     When I successfully execute the action "collection":"list" with args:
       | index | "tenant-kuzzle" |
     Then I should receive a result matching:
-      | collections | [{"name":"assets","type":"stored"},{"name":"config","type":"stored"},{"name":"devices","type":"stored"}] |
+      | collections | [{"name":"assets","type":"stored"},{"name":"assets-history","type":"stored"}, {"name":"config","type":"stored"},{"name":"devices","type":"stored"}] |
     When I successfully execute the action "device-manager/engine":"delete" with args:
       | index | "tenant-kuzzle" |
     And I successfully execute the action "collection":"list" with args:

features/Engine.feature

@@ -127,43 +127,43 @@ Feature: Payloads Controller
       | tenantId   | "tenant-ayse"                      |
 
   Scenario: Propagate device measures to asset
-    Given I successfully execute the action "device-manager/device":"linkAsset" with args:
-      | _id     | "DummyTemp-attached_ayse_unlinked" |
-      | assetId | "PERFO-unlinked"                   |
     When I successfully receive a "dummy-temp" payload with:
-      | deviceEUI    | "attached_ayse_unlinked" |
-      | register55   | 42.2                     |
-      | batteryLevel | 0.4                      |
-    Then The document "device-manager":"devices":"DummyTemp-attached_ayse_unlinked" content match:
-      | tenantId | "tenant-ayse"    |
-      | assetId  | "PERFO-unlinked" |
-    Then The document "tenant-ayse":"devices":"DummyTemp-attached_ayse_unlinked" content match:
-      | tenantId | "tenant-ayse"    |
-      | assetId  | "PERFO-unlinked" |
-    And The document "tenant-ayse":"assets":"PERFO-unlinked" content match:
-      | measures.temperature.id          | "DummyTemp-attached_ayse_unlinked" |
-      | measures.temperature.reference   | "attached_ayse_unlinked"           |
-      | measures.temperature.model       | "DummyTemp"                        |
-      | measures.temperature.updatedAt   | "_DATE_NOW_"                       |
-      | measures.temperature.payloadUuid | "_STRING_"                         |
-      | measures.temperature.degree      | 42.2                               |
-      | measures.temperature.qos.battery | 40                                 |
+      | deviceEUI    | "attached_ayse_linked" |
+      | register55   | 42.2                   |
+      | batteryLevel | 0.4                    |
+    Then The document "device-manager":"devices":"DummyTemp-attached_ayse_linked" content match:
+      | tenantId | "tenant-ayse" |
+      | assetId  | "MART-linked" |
+    Then The document "tenant-ayse":"devices":"DummyTemp-attached_ayse_linked" content match:
+      | tenantId | "tenant-ayse" |
+      | assetId  | "MART-linked" |
+    And The document "tenant-ayse":"assets":"MART-linked" content match:
+      | measures.temperature.id          | "DummyTemp-attached_ayse_linked" |
+      | measures.temperature.reference   | "attached_ayse_linked"           |
+      | measures.temperature.model       | "DummyTemp"                      |
+      | measures.temperature.updatedAt   | "_DATE_NOW_"                     |
+      | measures.temperature.payloadUuid | "_STRING_"                       |
+      | measures.temperature.degree      | 42.2                             |
+      | measures.temperature.qos.battery | 40                               |
+      # Enriched with the event
+      | metadata.enriched                | true                             |
+      | metadata.assetId                 | "MART-linked"                    |
     And I should receive a result matching:
-      | device._id | "DummyTemp-attached_ayse_unlinked" |
-      | asset._id  | "PERFO-unlinked"                   |
-      | tenantId   | "tenant-ayse"                      |
+      | device._id | "DummyTemp-attached_ayse_linked" |
+      | asset._id  | "MART-linked"                    |
+      | tenantId   | "tenant-ayse"                    |
 
-  Scenario: Trigger tenant specific events
-    Given I subscribe to "tests":"messages" notifications
-    And I successfully execute the action "device-manager/device":"linkAsset" with args:
-      | _id     | "DummyTemp-attached_ayse_unlinked" |
-      | assetId | "PERFO-unlinked"                   |
+  Scenario: Use event to enrich the asset and historize it
     When I successfully receive a "dummy-temp" payload with:
-      | deviceEUI    | "attached_ayse_unlinked" |
-      | register55   | 42.2                     |
-      | batteryLevel | 0.4                      |
-    Then I should receive realtime notifications for "tests":"messages" matching:
-      | result._source.device._id          | result._source.asset._id |
-      | "DummyTemp-attached_ayse_unlinked" | "PERFO-unlinked"         |
-
-
+      | deviceEUI    | "attached_ayse_linked" |
+      | register55   | 51.1                   |
+      | batteryLevel | 0.42                   |
+    And I refresh the collection "tenant-ayse":"assets-history"
+    And The last document from "tenant-ayse":"assets-history" content match:
+      | assetId                 | "MART-linked"   |
+      | measureTypes            | ["temperature"] |
+      | assetId                 | "MART-linked"   |
+      | asset.model             | "MART"          |
+      | asset.reference         | "linked"        |
+      | asset.metadata.enriched | true            |
+      | asset.metadata.assetId  | "MART-linked"   |

features/PayloadController.feature

@@ -1,4 +1,5 @@
-import { Backend, KuzzleRequest } from 'kuzzle';
+import { Backend, Kuzzle, KuzzleRequest } from 'kuzzle';
+import { AssetMeasures, BaseAssetContent } from 'lib/types';
 
 import { DeviceManagerPlugin } from '../../../index';
 import { DummyTempDecoder, DummyTempPositionDecoder } from './decoders';
@@ -58,6 +59,21 @@ deviceManager.assets.register('hevSuit', {
   freezing: { type: 'boolean' }
 }, { group: 'astronaut' });
 
+
+// Register a pipe to enrich a tenant asset
+app.pipe.register(`tenant:tenant-ayse:asset:measures:new`, async (request: KuzzleRequest) => {
+  if (request.result.assetId !== 'MART-linked') {
+    return request;
+  }
+
+  request.result.asset.metadata = {
+    enriched: true,
+    assetId: request.result.assetId
+  };
+
+  return request;
+});
+
 app.plugin.use(deviceManager);
 
 app.hook.register('request:onError', async (request: KuzzleRequest) => {
@@ -66,14 +82,8 @@ app.hook.register('request:onError', async (request: KuzzleRequest) => {
 
 app.config.set('plugins.kuzzle-plugin-logger.services.stdout.level', 'debug');
 
-/**
- * Register pipe for scenario used to test the tenant specific event propagation
- */
-app.pipe.register('tenant:tenant-ayse:device:new-payload', async eventParam => {
-  await app.sdk.realtime.publish('tests', 'messages', eventParam.result);
-
-  return eventParam;
-});
+// Reduce writing latency since we won't have significant load
+app.config.set('plugins.device-manager.writerInterval', 1);
 
 app.config.set('limits.documentsWriteCount', 5000);
 

features/fixtures/application/app.ts

@@ -40,6 +40,23 @@ module.exports = {
         tenantId: 'tenant-ayse',
         assetId: null
       },
+      { index: { _id: 'DummyTemp-attached_ayse_linked' } },
+      {
+        reference: 'attached_ayse_linked',
+        model: 'DummyTemp',
+        measures: {
+          temperature: {
+            updatedAt: 1610793427950,
+            payloadUuid: 'some-uuid',
+            degree: 42.2,
+          }
+        },
+        qos: {
+          battery: 80
+        },
+        tenantId: 'tenant-ayse',
+        assetId: 'MART-linked'
+      },
     ]
   },
   'tenant-ayse': {
@@ -61,13 +78,48 @@ module.exports = {
         tenantId: 'tenant-ayse',
         assetId: null
       },
+      { index: { _id: 'DummyTemp-attached_ayse_linked' } },
+      {
+        reference: 'attached_ayse_linked',
+        model: 'DummyTemp',
+        measures: {
+          temperature: {
+            updatedAt: 1610793427950,
+            payloadUuid: 'some-uuid',
+            degree: 42.2,
+          }
+        },
+        qos: {
+          battery: 80
+        },
+        tenantId: 'tenant-ayse',
+        assetId: 'MART-linked'
+      },
     ],
     assets: [
       { index: { _id: 'PERFO-unlinked' } },
       {
         model: 'PERFO',
         reference: 'unlinked',
         measures: {},
+      },
+      { index: { _id: 'MART-linked' } },
+      {
+        model: 'MART',
+        reference: 'linked',
+        measures: {
+          temperature: {
+            reference: 'attached_ayse_linked',
+            qos: {
+              battery: 40
+            },
+            payloadUuid: 'some-uuid',
+            degree: 42.2,
+            model: 'DummyTemp',
+            id: 'DummyTemp-attached_ayse_linked',
+            updatedAt: 1610793427950
+          }
+        },
       }
     ]
   },

features/fixtures/fixtures.js

@@ -180,3 +180,21 @@ Then('I delete the document {string}', async function (id) {
     this.props.collection,
     id);
 });
+
+Then('The last document from {string}:{string} content match:', async function (index, collection, dataTable) {
+  const expectedContent = this.parseObject(dataTable);
+
+  const result = await this.sdk.document.search(
+    index,
+    collection,
+    {
+      sort: { '_kuzzle_info.createdAt': 'desc' }
+    },
+     { size: 1 });
+
+  if (result.hits.length === 0) {
+    throw new Error('No document found');
+  }
+
+  should(result.hits[0]._source).matchObject(expectedContent);
+});

features/step_definitions/common/documents-steps.js

@@ -73,9 +73,11 @@ Before({ timeout: 30 * 1000 }, async function () {
     removeCatalogEntries(this.sdk, 'device-manager'),
 
     truncateCollection(this.sdk, 'tenant-kuzzle', 'assets'),
+    truncateCollection(this.sdk, 'tenant-kuzzle', 'assets-history'),
     truncateCollection(this.sdk, 'tenant-kuzzle', 'devices'),
 
     truncateCollection(this.sdk, 'tenant-ayse', 'assets'),
+    truncateCollection(this.sdk, 'tenant-ayse', 'assets-history'),
     truncateCollection(this.sdk, 'tenant-ayse', 'devices'),
     removeCatalogEntries(this.sdk, 'tenant-ayse'),
   ]);

features/support/hooks.js

@@ -31,10 +31,52 @@ import {
   assetsMappings,
   devicesMappings,
   catalogMappings,
+  assetsHistoryMappings,
 } from './models';
 
+export type DeviceManagerConfig = {
+  /**
+   * Administration index name
+   */
+  adminIndex: string;
+
+  /**
+   * Config collection name (in admin index)
+   */
+  configCollection: string;
+
+  /**
+   * Administration collections mappings (in admin index)
+   */
+  adminCollections: {
+    config: JSONObject;
+
+    devices: JSONObject;
+
+    payloads: JSONObject;
+  },
+
+  /**
+   * Tenants collections
+   */
+  collections: {
+    assets: JSONObject;
+
+    devices: JSONObject;
+
+    'assets-history': JSONObject;
+  },
+
+  /**
+   * Interval to write documents from the buffer
+   */
+  writerInterval: number;
+}
+
 export class DeviceManagerPlugin extends Plugin {
-  private defaultConfig: JSONObject;
+  public config: DeviceManagerConfig;
+
+  private defaultConfig: DeviceManagerConfig;
 
   private assetController: AssetController;
   private deviceController: DeviceController;
@@ -125,6 +167,7 @@ export class DeviceManagerPlugin extends Plugin {
       collections: {
         assets: assetsMappings,
         devices: devicesMappings,
+        'assets-history': assetsHistoryMappings,
       },
       writerInterval: 50
     };
@@ -138,9 +181,7 @@ export class DeviceManagerPlugin extends Plugin {
 
     this.context = context;
 
-    this.config.mappings = new Map<string, JSONObject>();
-
-    this.batchWriter = new BatchWriter(this.sdk, this.config.writerInterval);
+    this.batchWriter = new BatchWriter(this.sdk, { interval: this.config.writerInterval });
     this.batchWriter.begin();
 
     this.payloadService = new PayloadService(this, this.batchWriter);
@@ -234,7 +275,7 @@ export class DeviceManagerPlugin extends Plugin {
       }
 
       await Promise.all([
-        this.sdk.collection.create(this.config.adminIndex, 'config', this.config.adminCollections.config),
+        this.sdk.collection.create(this.config.adminIndex, this.config.configCollection, this.config.adminCollections.config),
         this.sdk.collection.create(this.config.adminIndex, 'devices', this.deviceMappings.get()),
         this.sdk.collection.create(this.config.adminIndex, 'payloads', this.getPayloadsMappings()),
       ]);
@@ -278,13 +319,13 @@ export class DeviceManagerPlugin extends Plugin {
   private async initializeConfig () {
     const exists = await this.sdk.document.exists(
       this.config.adminIndex,
-      'config',
+      this.config.configCollection,
       'plugin--device-manager');
 
     if (! exists) {
       await this.sdk.document.create(
         this.config.adminIndex,
-        'config',
+        this.config.configCollection,
         {
           type: 'device-manager',
           'device-manager': { autoProvisionning: true }
@@ -310,7 +351,7 @@ export class DeviceManagerPlugin extends Plugin {
   }
 
   private async generateConfigID (request: KuzzleRequest) {
-    if (request.getCollection() !== 'config') {
+    if (request.getCollection() !== this.config.configCollection) {
       return request;
     }