Detect UTF8 encoding in text responses

Author: janaz

package.json

@@ -34,7 +34,8 @@
   "main": "./dist/compile/lambda",
   "types": "./dist/compile/lambda.d.ts",
   "dependencies": {
-    "in-process-request": "^0.1.0"
+    "in-process-request": "^0.1.0",
+    "isutf8": "^2.1.0"
   },
   "devDependencies": {
     "@types/compression": "^1.7.0",

src/@types/isutf8/index.d.ts

@@ -0,0 +1,3 @@
+declare module 'isutf8' {
+  export default function isUtf8(b: Buffer): boolean
+}

src/response.ts

@@ -1,9 +1,11 @@
 import { InProcessResponse } from 'in-process-request';
 import { LambdaResponse } from './types';
 import fixResponseHeaders from './fixResponseHeaders'
+import isUtf8 from 'isutf8'
 
+type Encoding = 'base64' | 'utf8'
 export const inProcessResponseToLambdaResponse = (response: InProcessResponse, supportMultiHeaders: boolean, supportCookies: boolean): LambdaResponse => {
-  const encoding = response.isUTF8 ? 'utf8' : 'base64';
+  const encoding = getEncoding(response);
   return {
     statusCode: response.statusCode,
     body: response.body.toString(encoding),
@@ -12,6 +14,21 @@ export const inProcessResponseToLambdaResponse = (response: InProcessResponse, s
   };
 };
 
+const getEncoding = (response: InProcessResponse): Encoding => {
+  // APi Gateway REST API cannot handle html responses encoded as base64
+  if (response.isUTF8) {
+    return 'utf8';
+  }
+  const contentType = (response.headers['content-type'] as string || '').toLowerCase();
+  const isJson = (): boolean => contentType.startsWith('application/json')
+  const isText = (): boolean => contentType.startsWith('text/')
+  const maybeUtf8 = isJson() || isText()
+  if (maybeUtf8 && isUtf8(response.body)) {
+    return 'utf8'
+  }
+  return 'base64'
+}
+
 export const errorResponse = (): LambdaResponse => {
   return {
     statusCode: 500,

test/response.test.ts

@@ -2,14 +2,19 @@ import { inProcessResponseToLambdaResponse, errorResponse} from '../src/response
 import { InProcessResponse } from 'in-process-request';
 
 describe('inProcessResponseToLambdaResponse', () => {
-  const generateMockResponse = (isUTF8: boolean = true): InProcessResponse => ({
-    body: Buffer.from('hello'),
+  interface GenerateMockResponseOptions {
+    body?: Buffer
+    isUTF8?: boolean
+    contentType?: string
+  }
+  const generateMockResponse = (opts: GenerateMockResponseOptions = {}): InProcessResponse => ({
+    body: opts.body || Buffer.from('hello'),
     headers: {
       'set-cookie': ['chocolate=10; Path=/', 'peanut_butter=20; Path=/'],
-      'content-type': 'text/plain',
+      'content-type': opts.contentType || 'text/plain',
       'x-custom': '10',
     },
-    isUTF8,
+    isUTF8: opts.isUTF8 === true,
     statusCode: 200,
     statusMessage: 'OK',
   });
@@ -81,17 +86,37 @@ describe('inProcessResponseToLambdaResponse', () => {
 
   describe('when the body is not utf8', () => {
     it('has base64 body', () => {
-      const res = inProcessResponseToLambdaResponse(generateMockResponse(false), true, false);
+      const res = inProcessResponseToLambdaResponse(generateMockResponse({isUTF8: false, contentType: 'not-text/plain'}), true, false);
       expect(res.body).toEqual('aGVsbG8=');
     })
 
     it('is not base64 encoded', () => {
-      const res = inProcessResponseToLambdaResponse(generateMockResponse(false), true, false);
+      const res = inProcessResponseToLambdaResponse(generateMockResponse({isUTF8: false, contentType: 'not-text/plain'}), true, false);
       expect(res.isBase64Encoded).toEqual(true);
     })
 
+    it('has basee64 body if isUTF8 is set to false and content type starts with text/ and the content is not UTF8', () => {
+      const res = inProcessResponseToLambdaResponse(generateMockResponse({body: Buffer.from([1,2,3,4,5,6]), isUTF8: false, contentType: 'binary/octet-stream'}), false, false);
+      expect(res.isBase64Encoded).toEqual(true);
+      expect(res.body).toEqual('AQIDBAUG');
+    })
+
+  })
 
+  describe('UTF8 content', () => {
+    it('has utf8 body if isUTF8 is set to true', () => {
+      const res = inProcessResponseToLambdaResponse(generateMockResponse({isUTF8: true, contentType: 'not-text/plain'}), false, false);
+      expect(res.isBase64Encoded).toEqual(false);
+      expect(res.body).toEqual('hello');
+    })
+
+    it('has utf8 body if isUTF8 is set to false but content type starts with text/ and the content is text', () => {
+      const res = inProcessResponseToLambdaResponse(generateMockResponse({body: Buffer.from('text'), isUTF8: false, contentType: 'text/plain'}), false, false);
+      expect(res.isBase64Encoded).toEqual(false);
+      expect(res.body).toEqual('text');
+    })
   })
+
 })
 
 describe('errorResponse', () => {

yarn.lock

@@ -2100,6 +2100,11 @@ istanbul-reports@^3.0.2:
     html-escaper "^2.0.0"
     istanbul-lib-report "^3.0.0"
 
+isutf8@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/isutf8/-/isutf8-2.1.0.tgz#b6d08a02d4ce43bf3b4be39b9b60231b88dfeb2b"
+  integrity sha512-rEMU6f82evtJNtYMrtVODUbf+C654mos4l+9noOueesUMipSWK6x3tpt8DiXhcZh/ZOBWYzJ9h9cNAlcQQnMiQ==
+
 jake@^10.6.1:
   version "10.6.1"
   resolved "https://registry.yarnpkg.com/jake/-/jake-10.6.1.tgz#c9c476cfd6e726ef600ee9bb2b880d5425ff8c79"