feat: end-2-end mqttt bridge health check (#25)

* feat: end-2-end mqttt bridge health check

* refactor: move health check into its own stack

* refactor: hard code amazon root cert into source code

* feat: create fake health check device

* fix: delete parameter store

When using DeleteParametersCommand to delete parameters more than 10,
it will error as Member must have length less than or equal to 10

* feat: add create health check device cli

Author: pudkrong

.github/workflows/test-and-release.yaml

@@ -65,6 +65,7 @@ jobs:
       - name: Fake nRF Cloud account device
         run: |
           ./cli.sh fake-nrfcloud-account-device
+          ./cli.sh create-fake-nrfcloud-health-check-device
 
       - name: Deploy test resources stack
         run: |

README.md

@@ -33,6 +33,7 @@ need to prepare nRF Cloud API key.
 ```bash
 ./cli.sh configure thirdParty nrfcloud apiKey <API key>
 ./cli.sh initialize-nrfcloud-account
+./cli.sh create-health-check-device
 ```
 
 ### Deploy

cdk/BackendLambdas.d.ts

@@ -10,4 +10,5 @@ type BackendLambdas = {
 	fetchDeviceShadow: PackedLambda
 	onDeviceMessage: PackedLambda
 	storeMessagesInTimestream: PackedLambda
+	healthCheck: PackedLambda
 }

cdk/backend.ts

@@ -41,6 +41,9 @@ const packagesInLayer: string[] = [
 	'lodash-es',
 	'@middy/core',
 ]
+
+const healthCheckPackagesInLayer: string[] = ['mqtt', 'ws']
+
 const certsDir = path.join(
 	process.cwd(),
 	'certificates',
@@ -84,6 +87,10 @@ new BackendApp({
 		id: 'baseLayer',
 		dependencies: packagesInLayer,
 	}),
+	healthCheckLayer: await packLayer({
+		id: 'healthCheckLayer',
+		dependencies: healthCheckPackagesInLayer,
+	}),
 	iotEndpoint: await getIoTEndpoint({ iot })(),
 	mqttBridgeCertificate,
 	caCertificate,

cdk/packBackendLambdas.ts

@@ -29,4 +29,5 @@ export const packBackendLambdas = async (): Promise<BackendLambdas> => ({
 		'storeMessagesInTimestream',
 		'lambda/storeMessagesInTimestream.ts',
 	),
+	healthCheck: await packLambdaFromPath('healthCheck', 'lambda/healthCheck.ts'),
 })

cdk/resources/HealthCheckMqttBridge.ts

@@ -0,0 +1,76 @@
+import {
+	Duration,
+	aws_events_targets as EventTargets,
+	aws_events as Events,
+	aws_iam as IAM,
+	aws_lambda as Lambda,
+	Stack,
+} from 'aws-cdk-lib'
+import { Construct } from 'constructs'
+import { type Settings as BridgeSettings } from '../../bridge/settings.js'
+import type { PackedLambda } from '../helpers/lambdas/packLambda.js'
+import type { DeviceStorage } from './DeviceStorage.js'
+import { LambdaLogGroup } from './LambdaLogGroup.js'
+import type { WebsocketAPI } from './WebsocketAPI.js'
+
+export type BridgeImageSettings = BridgeSettings
+
+export class HealthCheckMqttBridge extends Construct {
+	public constructor(
+		parent: Construct,
+		{
+			websocketAPI,
+			deviceStorage,
+			layers,
+			lambdaSources,
+		}: {
+			websocketAPI: WebsocketAPI
+			deviceStorage: DeviceStorage
+			layers: Lambda.ILayerVersion[]
+			lambdaSources: {
+				healthCheck: PackedLambda
+			}
+		},
+	) {
+		super(parent, 'healthCheckMqttBridge')
+
+		const scheduler = new Events.Rule(this, 'scheduler', {
+			description: `Scheduler to health check mqtt bridge`,
+			schedule: Events.Schedule.rate(Duration.minutes(1)),
+		})
+
+		// Lambda functions
+		const healthCheck = new Lambda.Function(this, 'healthCheck', {
+			handler: lambdaSources.healthCheck.handler,
+			architecture: Lambda.Architecture.ARM_64,
+			runtime: Lambda.Runtime.NODEJS_18_X,
+			timeout: Duration.seconds(15),
+			memorySize: 1792,
+			code: Lambda.Code.fromAsset(lambdaSources.healthCheck.zipFile),
+			description: 'End to end test for mqtt bridge',
+			environment: {
+				VERSION: this.node.tryGetContext('version'),
+				LOG_LEVEL: this.node.tryGetContext('logLevel'),
+				NODE_NO_WARNINGS: '1',
+				STACK_NAME: Stack.of(this).stackName,
+				DEVICES_TABLE_NAME: deviceStorage.devicesTable.tableName,
+				WEBSOCKET_URL: websocketAPI.websocketURI,
+			},
+			initialPolicy: [],
+			layers,
+		})
+		const ssmReadPolicy = new IAM.PolicyStatement({
+			effect: IAM.Effect.ALLOW,
+			actions: ['ssm:GetParametersByPath'],
+			resources: [
+				`arn:aws:ssm:${Stack.of(this).region}:${
+					Stack.of(this).account
+				}:parameter/${Stack.of(this).stackName}/thirdParty/nrfcloud`,
+			],
+		})
+		healthCheck.addToRolePolicy(ssmReadPolicy)
+		scheduler.addTarget(new EventTargets.LambdaFunction(healthCheck))
+		deviceStorage.devicesTable.grantWriteData(healthCheck)
+		new LambdaLogGroup(this, 'healthCheckLog', healthCheck)
+	}
+}

cdk/stacks/BackendStack.ts

@@ -13,6 +13,7 @@ import { ContinuousDeployment } from '../resources/ContinuousDeployment.js'
 import { ConvertDeviceMessages } from '../resources/ConvertDeviceMessages.js'
 import { DeviceShadow } from '../resources/DeviceShadow.js'
 import { DeviceStorage } from '../resources/DeviceStorage.js'
+import { HealthCheckMqttBridge } from '../resources/HealthCheckMqttBridge.js'
 import { HistoricalData } from '../resources/HistoricalData.js'
 import {
 	Integration,
@@ -28,6 +29,7 @@ export class BackendStack extends Stack {
 		{
 			lambdaSources,
 			layer,
+			healthCheckLayer,
 			iotEndpoint,
 			mqttBridgeCertificate,
 			caCertificate,
@@ -38,6 +40,7 @@ export class BackendStack extends Stack {
 		}: {
 			lambdaSources: BackendLambdas
 			layer: PackedLayer
+			healthCheckLayer: PackedLayer
 			iotEndpoint: string
 			mqttBridgeCertificate: CertificateFiles
 			caCertificate: CAFiles
@@ -72,6 +75,15 @@ export class BackendStack extends Stack {
 				'parameterStoreExtensionLayer',
 				parameterStoreLayerARN[Stack.of(this).region] as string,
 			)
+		const healthCheckLayerVersion = new Lambda.LayerVersion(
+			this,
+			'healthCheckLayer',
+			{
+				code: Lambda.Code.fromAsset(healthCheckLayer.layerZipFile),
+				compatibleArchitectures: [Lambda.Architecture.ARM_64],
+				compatibleRuntimes: [Lambda.Runtime.NODEJS_18_X],
+			},
+		)
 
 		const lambdaLayers: Lambda.ILayerVersion[] = [
 			baseLayer,
@@ -100,6 +112,13 @@ export class BackendStack extends Stack {
 			bridgeImageSettings,
 		})
 
+		new HealthCheckMqttBridge(this, {
+			websocketAPI,
+			deviceStorage,
+			layers: [...lambdaLayers, healthCheckLayerVersion],
+			lambdaSources,
+		})
+
 		new ConvertDeviceMessages(this, {
 			deviceStorage,
 			websocketAPI,

cli/cli.ts

@@ -14,7 +14,9 @@ import psjon from '../package.json'
 import type { CommandDefinition } from './commands/CommandDefinition'
 import { configureDeviceCommand } from './commands/configure-device.js'
 import { configureCommand } from './commands/configure.js'
-import { createFakeNrfCloudAccountDeviceCredentials } from './commands/createFakeNrfCloudAccountDeviceCredentials.js'
+import { createFakeNrfCloudAccountDeviceCredentials } from './commands/create-fake-nrfcloud-account-device-credentials.js'
+import { createFakeNrfCloudHealthCheckDevice } from './commands/create-fake-nrfcloud-health-check-device.js'
+import { createHealthCheckDevice } from './commands/create-health-check-device.js'
 import { importDevicesCommand } from './commands/import-devices.js'
 import { initializeNRFCloudAccountCommand } from './commands/initialize-nrfcloud-account.js'
 import { logsCommand } from './commands/logs.js'
@@ -66,6 +68,12 @@ const CLI = async ({ isCI }: { isCI: boolean }) => {
 				ssm,
 			}),
 		)
+		commands.push(
+			createFakeNrfCloudHealthCheckDevice({
+				iot,
+				ssm,
+			}),
+		)
 	} else {
 		commands.push(
 			initializeNRFCloudAccountCommand({
@@ -74,6 +82,13 @@ const CLI = async ({ isCI }: { isCI: boolean }) => {
 				stackName: STACK_NAME,
 			}),
 		)
+		commands.push(
+			createHealthCheckDevice({
+				ssm,
+				stackName: STACK_NAME,
+				env: accountEnv,
+			}),
+		)
 		try {
 			const outputs = await stackOutput(
 				new CloudFormationClient({}),

cli/commands/createFakeNrfCloudAccountDeviceCredentials.ts renamed to cli/commands/create-fake-nrfcloud-account-device-credentials.ts

@@ -18,13 +18,14 @@ import {
 	SSMClient,
 } from '@aws-sdk/client-ssm'
 import chalk from 'chalk'
+import { chunk } from 'lodash-es'
 import { randomUUID } from 'node:crypto'
 import { getIoTEndpoint } from '../../aws/getIoTEndpoint.js'
 import { STACK_NAME } from '../../cdk/stacks/stackConfig.js'
 import { updateSettings, type Settings } from '../../nrfcloud/settings.js'
 import { isString } from '../../util/isString.js'
 import { settingsPath } from '../../util/settings.js'
-import type { CommandDefinition } from './CommandDefinition'
+import type { CommandDefinition } from './CommandDefinition.js'
 
 export const createFakeNrfCloudAccountDeviceCredentials = ({
 	iot,
@@ -114,11 +115,14 @@ export const createFakeNrfCloudAccountDeviceCredentials = ({
 				...(parameters.Parameters?.map((p) => p.Name) ?? []),
 				fakeTenantParameter,
 			]
-			await ssm.send(
-				new DeleteParametersCommand({
-					Names: names as string[],
-				}),
-			)
+			const namesChunk = chunk(names, 10)
+			for (const names of namesChunk) {
+				await ssm.send(
+					new DeleteParametersCommand({
+						Names: names as string[],
+					}),
+				)
+			}
 			return
 		}
 		const tenantId = randomUUID()

cli/commands/create-fake-nrfcloud-health-check-device.ts

@@ -0,0 +1,77 @@
+import {
+	AttachPolicyCommand,
+	CreateKeysAndCertificateCommand,
+	IoTClient,
+} from '@aws-sdk/client-iot'
+import { GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm'
+import chalk from 'chalk'
+import { STACK_NAME } from '../../cdk/stacks/stackConfig.js'
+import {
+	updateSettings,
+	type Settings,
+} from '../../nrfcloud/healthCheckSettings.js'
+import { isString } from '../../util/isString.js'
+import type { CommandDefinition } from './CommandDefinition.js'
+
+export const createFakeNrfCloudHealthCheckDevice = ({
+	iot,
+	ssm,
+}: {
+	iot: IoTClient
+	ssm: SSMClient
+}): CommandDefinition => ({
+	command: 'create-fake-nrfcloud-health-check-device',
+	action: async () => {
+		const fakeTenantParameter = `/${STACK_NAME}/fakeTenant`
+		const tenantId = (
+			await ssm.send(
+				new GetParameterCommand({
+					Name: fakeTenantParameter,
+				}),
+			)
+		).Parameter?.Value
+		if (tenantId === undefined) {
+			throw new Error(`${STACK_NAME} has no fake nRF Cloud Account device`)
+		}
+
+		const policyName = `fake-nrfcloud-account-device-policy-${tenantId}`
+		console.debug(chalk.magenta(`Creating IoT certificate`))
+		const credentials = await iot.send(
+			new CreateKeysAndCertificateCommand({
+				setAsActive: true,
+			}),
+		)
+
+		console.debug(chalk.magenta(`Attaching policy to IoT certificate`))
+		await iot.send(
+			new AttachPolicyCommand({
+				policyName,
+				target: credentials.certificateArn,
+			}),
+		)
+
+		const pk = credentials.keyPair?.PrivateKey
+		if (
+			!isString(credentials.certificatePem) ||
+			!isString(pk) ||
+			!isString(credentials.certificateArn)
+		) {
+			throw new Error(`Failed to create certificate!`)
+		}
+
+		const settings: Settings = {
+			healthCheckClientCert: credentials.certificatePem,
+			healthCheckPrivateKey: pk,
+			healthCheckClientId: 'health-check',
+			healthCheckModel: 'PCA20035+solar',
+			healthCheckFingerPrint: '29a.ch3ckr',
+		}
+		await updateSettings({ ssm, stackName: STACK_NAME })(settings)
+
+		console.debug(chalk.white(`Fake nRF Cloud health check device settings:`))
+		Object.entries(settings).forEach(([k, v]) => {
+			console.debug(chalk.yellow(`${k}:`), chalk.blue(v))
+		})
+	},
+	help: 'Creates fake nRF Cloud health check device used by the stack to end-to-end health check',
+})