Added fault-tolerance (retryability) to all SimpleDB client requests

Author: squidfunk

CHANGELOG

@@ -1,3 +1,7 @@
+isotopes-0.4.0 (2018-08-05)
+
+  * Added fault-tolerance (retryability) to all SimpleDB client requests
+
 isotopes-0.3.3 (2018-08-02)
 
   * Added runtime check to Isotope.put for presence of item name

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "isotopes",
-  "version": "0.3.3",
+  "version": "0.4.0",
   "description": "Serverless and typed object store built on top of AWS SimpleDB",
   "keywords": [
     "aws",
@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "lodash": "^4.17.10",
+    "retry": "^0.12.0",
     "squel": "^5.12.2"
   },
   "devDependencies": {
@@ -43,6 +44,7 @@
     "@types/jasmine": "^2.8.6",
     "@types/lodash": "^4.14.113",
     "@types/node": "^9.6.2",
+    "@types/retry": "^0.10.2",
     "aws-sdk": "^2.279.1",
     "aws-sdk-mock": "^2.0.0",
     "chance": "^1.0.13",

package.json

@@ -20,8 +20,9 @@
  * IN THE SOFTWARE.
  */
 
-import { SimpleDB } from "aws-sdk"
+import { AWSError, SimpleDB } from "aws-sdk"
 import { castArray, toPairs } from "lodash/fp"
+import { operation, OperationOptions } from "retry"
 
 import { IsotopeDictionary } from "../format"
 
@@ -34,6 +35,7 @@ import { IsotopeDictionary } from "../format"
  */
 export interface IsotopeClientOptions {
   consistent?: boolean                 /* Whether to use consistent reads */
+  retry?: OperationOptions             /* Retry strategy options */
 }
 
 /**
@@ -52,15 +54,57 @@ export interface IsotopeClientItemList {
   next?: string                        /* Pagination token */
 }
 
+/* ----------------------------------------------------------------------------
+ * Functions
+ * ------------------------------------------------------------------------- */
+
+/**
+ * Make a function returning a Promise retryable
+ *
+ * Only 5xx errors need to be retried as we don't need to retry client errors,
+ * so fail immediately if the status code is below 500.
+ *
+ * @param action - Function returning a Promise
+ * @param options - Retry strategy options
+ *
+ * @return Promise resolving with the original Promise's result
+ */
+export function retryable<T>(
+  action: () => Promise<T>,
+  options: OperationOptions
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const op = operation(options)
+    op.attempt(async () => {
+      try {
+        resolve(await action())
+      } catch (err) {
+        const { statusCode } = err as AWSError
+        if (!statusCode || statusCode < 500 || !op.retry(err))
+          reject(err)
+      }
+    })
+  })
+}
+
 /* ----------------------------------------------------------------------------
  * Values
  * ------------------------------------------------------------------------- */
 
 /**
  * Default client options
+ *
+ * We're not using the exponential backoff strategy (as recommended) due to the
+ * observations made in this article: https://bit.ly/2AJQiNV
  */
-const defaultOptions: IsotopeClientOptions = {
-  consistent: false
+const defaultOptions: Required<IsotopeClientOptions> = {
+  consistent: false,
+  retry: {
+    minTimeout: 100,
+    maxTimeout: 250,
+    retries: 3,
+    factor: 1
+  }
 }
 
 /* ----------------------------------------------------------------------------
@@ -117,6 +161,11 @@ export function mapAttributesToDictionary(
  */
 export class IsotopeClient {
 
+  /**
+   * Isotope client options
+   */
+  protected options: Required<IsotopeClientOptions>
+
   /**
    * SimpleDB instance
    */
@@ -130,8 +179,9 @@ export class IsotopeClient {
    */
   public constructor(
     protected domain: string,
-    protected options: IsotopeClientOptions = defaultOptions
+    options?: IsotopeClientOptions
   ) {
+    this.options  = { ...defaultOptions, ...options }
     this.simpledb = new SimpleDB({ apiVersion: "2009-04-15" })
   }
 
@@ -141,9 +191,10 @@ export class IsotopeClient {
    * @return Promise resolving with no result
    */
   public async create(): Promise<void> {
-    await this.simpledb.createDomain({
-      DomainName: this.domain
-    }).promise()
+    await retryable(() =>
+      this.simpledb.createDomain({
+        DomainName: this.domain
+      }).promise(), this.options.retry)
   }
 
   /**
@@ -152,9 +203,10 @@ export class IsotopeClient {
    * @return Promise resolving with no result
    */
   public async destroy(): Promise<void> {
-    await this.simpledb.deleteDomain({
-      DomainName: this.domain
-    }).promise()
+    await retryable(() =>
+      this.simpledb.deleteDomain({
+        DomainName: this.domain
+      }).promise(), this.options.retry)
   }
 
   /**
@@ -168,12 +220,13 @@ export class IsotopeClient {
   public async get(
     id: string, names?: string[]
   ): Promise<IsotopeClientItem | undefined> {
-    const { Attributes } = await this.simpledb.getAttributes({
-      DomainName: this.domain,
-      ItemName: id,
-      AttributeNames: names,
-      ConsistentRead: this.options.consistent
-    }).promise()
+    const { Attributes } = await retryable(() =>
+      this.simpledb.getAttributes({
+        DomainName: this.domain,
+        ItemName: id,
+        AttributeNames: names,
+        ConsistentRead: this.options.consistent
+      }).promise(), this.options.retry)
 
     /* Item not found */
     if (!Attributes)
@@ -197,11 +250,12 @@ export class IsotopeClient {
   public async put(
     id: string, attrs: IsotopeDictionary
   ): Promise<void> {
-    await this.simpledb.putAttributes({
-      DomainName: this.domain,
-      ItemName: id,
-      Attributes: mapDictionaryToAttributes(attrs)
-    }).promise()
+    await retryable(() =>
+      this.simpledb.putAttributes({
+        DomainName: this.domain,
+        ItemName: id,
+        Attributes: mapDictionaryToAttributes(attrs)
+      }).promise(), this.options.retry)
   }
 
   /**
@@ -215,14 +269,15 @@ export class IsotopeClient {
   public async delete(
     id: string, names?: string[]
   ): Promise<void> {
-    await this.simpledb.deleteAttributes({
-      DomainName: this.domain,
-      ItemName: id,
-      Attributes: (names || [])
-        .map<SimpleDB.DeletableAttribute>(name => ({
-          Name: name
-        }))
-    }).promise()
+    await retryable(() =>
+      this.simpledb.deleteAttributes({
+        DomainName: this.domain,
+        ItemName: id,
+        Attributes: (names || [])
+          .map<SimpleDB.DeletableAttribute>(name => ({
+            Name: name
+          }))
+      }).promise(), this.options.retry)
   }
 
   /**
@@ -236,11 +291,12 @@ export class IsotopeClient {
   public async select(
     expr: string, next?: string
   ): Promise<IsotopeClientItemList> {
-    const { Items, NextToken } = await this.simpledb.select({
-      SelectExpression: expr,
-      NextToken: next,
-      ConsistentRead: this.options.consistent
-    }).promise()
+    const { Items, NextToken } = await retryable(() =>
+      this.simpledb.select({
+        SelectExpression: expr,
+        NextToken: next,
+        ConsistentRead: this.options.consistent
+      }).promise(), this.options.retry)
 
     /* No items found */
     if (!Items)

src/isotopes/client/index.ts

@@ -1,6 +1,8 @@
 {
   "extends": "../tsconfig.json",
   "compilerOptions": {
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
     "paths": {
       "_/*": ["../tests/*"]
     },