From e5309766032c0fe328643bc8dd3f834623e03089 Mon Sep 17 00:00:00 2001 From: Paul Annekov Date: Tue, 18 Jun 2019 13:16:05 +0300 Subject: [PATCH 1/6] elasticio#1599 made sailor to use object-storage-client lib --- lib/amqp.js | 14 ++- lib/objectStorage.js | 81 ---------------- mocha_spec/integration_helpers.js | 15 ++- mocha_spec/objectStorage.spec.js | 136 --------------------------- mocha_spec/run.spec.js | 2 - package-lock.json | 148 +++++++++++++++++++++++++++++- package.json | 1 + 7 files changed, 167 insertions(+), 230 deletions(-) delete mode 100644 lib/objectStorage.js delete mode 100644 mocha_spec/objectStorage.spec.js diff --git a/lib/amqp.js b/lib/amqp.js index fa78f69b..1f8a0ce8 100644 --- a/lib/amqp.js +++ b/lib/amqp.js @@ -1,7 +1,7 @@ const log = require('./logging.js'); const amqplib = require('amqplib'); const encryptor = require('./encryptor.js'); -const ObjectStorage = require('./objectStorage.js'); +const { ObjectStorage } = require('@elastic.io/object-storage-client'); const co = require('co'); const _ = require('lodash'); const eventToPromise = require('event-to-promise'); @@ -12,7 +12,13 @@ const HEADER_ERROR_RESPONSE = 'x-eio-error-response'; class Amqp { constructor(settings) { this.settings = settings; - this.objectStorage = new ObjectStorage(settings); + this.objectStorage = new ObjectStorage({ + uri: settings.OBJECT_STORAGE_URI, + cipher: { + key: process.env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD, + iv: process.env.ELASTICIO_MESSAGE_CRYPTO_IV + } + }); } connect(uri) { @@ -61,7 +67,7 @@ class Amqp { return encryptor.decryptMessageContent(message.content); } - return await this.objectStorage.getObject(objectId); + return await this.objectStorage.getAsJSON(objectId, this.settings.OBJECT_STORAGE_TOKEN); } async encryptMessage(data, properties) { @@ -71,7 +77,7 @@ class Amqp { let message = ''; try { - properties.headers.objectId = await this.objectStorage.addObject(data); + properties.headers.objectId = await this.objectStorage.addAsJSON(data, this.settings.OBJECT_STORAGE_TOKEN); } catch (e) { log.error('Failed to add message to object storage: %s', e); message = encryptor.encryptMessageContent(data); diff --git a/lib/objectStorage.js b/lib/objectStorage.js deleted file mode 100644 index f6482c0f..00000000 --- a/lib/objectStorage.js +++ /dev/null @@ -1,81 +0,0 @@ -const log = require('./logging.js'); -const encryptor = require('./encryptor.js'); -const uuid = require('uuid'); -const axios = require('axios'); -const http = require('http'); -const https = require('https'); - -class ObjectStorage { - constructor(settings) { - this.api = axios.create({ - baseURL: `${settings.OBJECT_STORAGE_URI}/`, - httpAgent: new http.Agent({ keepAlive: true }), - httpsAgent: new https.Agent({ keepAlive: true }), - headers: { Authorization: `Bearer ${settings.OBJECT_STORAGE_TOKEN}` }, - validateStatus: null - }); - } - - async requestRetry({ maxAttempts, delay, request, onResponse }) { - let attempts = 0; - let res; - let err; - while (attempts < maxAttempts) { - err = null; - res = null; - attempts++; - try { - res = await request(); - } catch (e) { - err = e; - } - if (onResponse && onResponse(err, res)) { - continue; - } - if (err || res.status >= 400) { - log.warn('Error during object get: %s', err ? err : `${res.status} (${res.statusText})`); - await new Promise((resolve) => setTimeout(resolve, delay)); - continue; - } - break; - } - if (err || res.status >= 400) { - throw err || new Error(`HTTP error during object get: ${res.status} (${res.statusText})`); - } - return res; - } - - async getObject(objectId) { - const res = await this.requestRetry({ - maxAttempts: 3, - delay: 100, - request: () => this.api.get(`/objects/${objectId}`, { responseType: 'stream' }) - }); - - return await encryptor.decryptMessageContentStream(res.data); - } - - async addObject(data) { - let objectId = uuid.v4(); - await this.requestRetry({ - maxAttempts: 3, - delay: 100, - request: () => this.api.put( - `/objects/${objectId}`, - encryptor.encryptMessageContentStream(data), - { headers: { 'content-type': 'application/octet-stream' } } - ), - onResponse: (err, res) => { - if (!err && res.status === 409) { - log.warn('Generated already existing UUID'); - objectId = uuid.v4(); - return true; - } - } - }); - - return objectId; - } -} - -module.exports = ObjectStorage; diff --git a/mocha_spec/integration_helpers.js b/mocha_spec/integration_helpers.js index 402d1d7d..6d071477 100644 --- a/mocha_spec/integration_helpers.js +++ b/mocha_spec/integration_helpers.js @@ -6,7 +6,8 @@ const { EventEmitter } = require('events'); const PREFIX = 'sailor_nodejs_integration_test'; const nock = require('nock'); const getStream = require('get-stream'); -const encryptor = require('../lib/encryptor.js'); +const { Readable } = require('stream'); +const { Message } = require('@elastic.io/object-storage-client'); const env = process.env; @@ -184,6 +185,9 @@ function prepareEnv() { env.ELASTICIO_OBJECT_STORAGE_URI = 'http://ma.es.ter'; env.ELASTICIO_OBJECT_STORAGE_ENABLED = ''; env.ELASTICIO_OBJECT_STORAGE_TOKEN = 'jwt'; + + env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD = 'testCryptoPassword'; + env.ELASTICIO_MESSAGE_CRYPTO_IV = 'iv=any16_symbols'; } function mockApiTaskStepResponse(response) { @@ -203,7 +207,14 @@ function mockApiTaskStepResponse(response) { } async function encryptForObjectStorage(input) { - const stream = encryptor.encryptMessageContentStream(input); + const dataStream = new Readable(); + dataStream.push(JSON.stringify(input)); + dataStream.push(null); + const message = new Message({ + key: env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD, + iv: env.ELASTICIO_MESSAGE_CRYPTO_IV + }); + const stream = message.packStream(dataStream); return await getStream.buffer(stream); } diff --git a/mocha_spec/objectStorage.spec.js b/mocha_spec/objectStorage.spec.js deleted file mode 100644 index e9a2376b..00000000 --- a/mocha_spec/objectStorage.spec.js +++ /dev/null @@ -1,136 +0,0 @@ -const nock = require('nock'); -const sinonjs = require('sinon'); -const expect = require('chai').expect; -const getStream = require('get-stream'); -const logging = require('../lib/logging.js'); - -describe('ObjectStorage', () => { - - process.env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD = 'testCryptoPassword'; - process.env.ELASTICIO_MESSAGE_CRYPTO_IV = 'iv=any16_symbols'; - - const envVars = {}; - envVars.ELASTICIO_AMQP_URI = 'amqp://test2/test2'; - envVars.ELASTICIO_FLOW_ID = '5559edd38968ec0736000003'; - envVars.ELASTICIO_STEP_ID = 'step_1'; - envVars.ELASTICIO_EXEC_ID = 'some-exec-id'; - - envVars.ELASTICIO_USER_ID = '5559edd38968ec0736000002'; - envVars.ELASTICIO_COMP_ID = '5559edd38968ec0736000456'; - envVars.ELASTICIO_FUNCTION = 'list'; - - envVars.ELASTICIO_HOOK_SHUTDOWN = true; - - envVars.ELASTICIO_API_URI = 'http://apihost.com'; - envVars.ELASTICIO_API_USERNAME = 'test@test.com'; - envVars.ELASTICIO_API_KEY = '5559edd'; - - envVars.ELASTICIO_OBJECT_STORAGE_URI = 'http://ma.es.ter'; - envVars.ELASTICIO_OBJECT_STORAGE_TOKEN = 'jwt'; - envVars.ELASTICIO_OBJECT_FLOW_ID = 1; - - const ObjectStorage = require('../lib/objectStorage.js'); - const encryptor = require('../lib/encryptor.js'); - const settings = require('../lib/settings.js').readFrom(envVars); - - let sinon; - beforeEach(() => { - sinon = sinonjs.sandbox.create(); - }); - afterEach(() => { - sinon.restore(); - }); - - it('should fail after 3 retries', async () => { - const log = sinon.stub(logging, 'warn'); - const objectStorage = new ObjectStorage(settings); - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .get('/objects/1') - .replyWithError({ code: 'ETIMEDOUT' }) - .get('/objects/1') - .reply(404) - .get('/objects/1') - .replyWithError({ code: 'ENOTFOUND' }); - - let err; - try { - await objectStorage.getObject(1); - } catch (e) { - err = e; - } - - expect(objectStorageCalls.isDone()).to.be.true; - expect(err.code).to.be.equal('ENOTFOUND'); - expect(log.getCall(1).args[1].toString()).to.include('404'); - expect(log.callCount).to.be.equal(3); - }); - - it('should retry get request 3 times on errors', async () => { - const objectStorage = new ObjectStorage(settings); - const data = { test: 'test' }; - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .get('/objects/1') - .reply(500) - .get('/objects/1') - .replyWithError({ code: 'ECONNRESET' }) - .get('/objects/1') - .reply(200, await getStream.buffer(encryptor.encryptMessageContentStream(data))); - - const out = await objectStorage.getObject(1); - - expect(objectStorageCalls.isDone()).to.be.true; - expect(out).to.be.deep.equal(data); - }); - - it('should retry put request 3 times on errors', async () => { - const objectStorage = new ObjectStorage(settings); - const data = { test: 'test' }; - const put = await getStream.buffer(encryptor.encryptMessageContentStream(data)); - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .put(/^\/objects\/[0-9a-z-]+$/, put) - .replyWithError({ code: 'ECONNREFUSED' }) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(400) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(200); - - await objectStorage.addObject(data); - - expect(objectStorageCalls.isDone()).to.be.true; - }); - - it('should retry put request on 409 error with different objectId', async () => { - const log = sinon.stub(logging, 'warn'); - const objectStorage = new ObjectStorage(settings); - const data = { test: 'test' }; - const put = await getStream.buffer(encryptor.encryptMessageContentStream(data)); - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .matchHeader('content-type', 'application/octet-stream') - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(503) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(409) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(200); - const urls = []; - objectStorageCalls.on('request', req => { - urls.push(req.path); - }); - - await objectStorage.addObject(data); - - expect(objectStorageCalls.isDone()).to.be.true; - expect(urls[0]).to.be.equal(urls[1]); - expect(urls[1]).to.not.equal(urls[2]); - expect(log.getCall(1).args[0]).to.include('UUID'); - expect(log.callCount).to.be.equal(2); - }); -}); diff --git a/mocha_spec/run.spec.js b/mocha_spec/run.spec.js index 6fb6e131..bd7135e0 100644 --- a/mocha_spec/run.spec.js +++ b/mocha_spec/run.spec.js @@ -214,7 +214,6 @@ describe('Integration Test', () => { .get('/customers') .reply(200, customers); - const log = sinon.stub(logging, 'warn'); const objectStorageGet = nock(process.env.ELASTICIO_OBJECT_STORAGE_URI) .get(`/objects/${objectId}`) .matchHeader('authorization', /Bearer/) @@ -251,7 +250,6 @@ describe('Integration Test', () => { expect(body).to.be.null; expect(objectStorageGet.isDone()).to.be.true; expect(objectStoragePut.isDone()).to.be.true; - expect(log.getCall(1).args[1].toString()).to.include('400'); done(); }); diff --git a/package-lock.json b/package-lock.json index 8b29c09d..9b14e7bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,58 @@ { "name": "elasticio-sailor-nodejs", - "version": "2.4.0-dev.29", + "version": "2.4.2-dev.0", "lockfileVersion": 1, "requires": true, "dependencies": { + "@elastic.io/bunyan-logger": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@elastic.io/bunyan-logger/-/bunyan-logger-1.0.5.tgz", + "integrity": "sha512-FcoaG7nTA2H/VuE+0TC1ZKxwEMv3eTVJ42HwrzOY3x3UIpJ9RorG+Sk7G6SBoNuEiBRslGA6Iy9ddE4J3lR2og==", + "requires": { + "bunyan": "1.8.12" + } + }, + "@elastic.io/object-storage-client": { + "version": "0.0.1-dev.7", + "resolved": "https://registry.npmjs.org/@elastic.io/object-storage-client/-/object-storage-client-0.0.1-dev.7.tgz", + "integrity": "sha512-1LSfLxpfhkCiub1NrpCjRghFk9dz8wCMhkXLpZtLg25mP4IMAhjZlx9oJUgVBwRDJH/beJeQSZ/5o6qDLmtjBA==", + "requires": { + "@elastic.io/bunyan-logger": "1.0.5", + "axios": "0.19.0", + "get-stream": "5.1.0", + "jsonwebtoken": "8.5.1", + "uuid": "3.3.2" + }, + "dependencies": { + "axios": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz", + "integrity": "sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==", + "requires": { + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" + } + }, + "follow-redirects": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "requires": { + "debug": "=3.1.0" + } + }, + "is-buffer": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "abbrev": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", @@ -470,6 +519,11 @@ "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", "dev": true }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -1044,6 +1098,14 @@ "safer-buffer": "^2.1.0" } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -3335,6 +3397,30 @@ "integrity": "sha1-PAQ0dD35Pi9cQq7nsZvLSDV19OE=", "dev": true }, + "jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "requires": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -3347,6 +3433,25 @@ "verror": "1.10.0" } }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "kind-of": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", @@ -3586,6 +3691,11 @@ "lodash._root": "^3.0.0" } }, + "lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" + }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -3598,6 +3708,21 @@ "integrity": "sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U=", "dev": true }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" + }, + "lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" + }, "lodash.isobject": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-2.4.1.tgz", @@ -3607,6 +3732,16 @@ "lodash._objecttypes": "~2.4.1" } }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" + }, "lodash.keys": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz", @@ -3618,6 +3753,11 @@ "lodash.isarray": "^3.0.0" } }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, "lodash.restparam": { "version": "3.6.1", "resolved": "https://registry.npmjs.org/lodash.restparam/-/lodash.restparam-3.6.1.tgz", @@ -5138,8 +5278,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safe-json-stringify": { "version": "1.2.0", @@ -5179,8 +5318,7 @@ "semver": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", - "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", - "dev": true + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" }, "send": { "version": "0.14.2", diff --git a/package.json b/package.json index f74b83a1..a32a7284 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "node": ">=10.15.0" }, "dependencies": { + "@elastic.io/object-storage-client": "0.0.1-dev.7", "amqplib": "0.5.1", "axios": "0.18.0", "bunyan": "^1.8.10", From 4bfd9703bccbb9bae7e900f249920565a2651975 Mon Sep 17 00:00:00 2001 From: Paul Annekov Date: Tue, 18 Jun 2019 13:16:05 +0300 Subject: [PATCH 2/6] elasticio#1599 made sailor to use object-storage-client lib --- lib/amqp.js | 14 ++- lib/objectStorage.js | 81 ---------------- mocha_spec/integration_helpers.js | 15 ++- mocha_spec/objectStorage.spec.js | 136 --------------------------- mocha_spec/run.spec.js | 2 - package-lock.json | 148 +++++++++++++++++++++++++++++- package.json | 1 + 7 files changed, 167 insertions(+), 230 deletions(-) delete mode 100644 lib/objectStorage.js delete mode 100644 mocha_spec/objectStorage.spec.js diff --git a/lib/amqp.js b/lib/amqp.js index 61b9ba86..596ffb38 100644 --- a/lib/amqp.js +++ b/lib/amqp.js @@ -1,7 +1,7 @@ const log = require('./logging.js'); const amqplib = require('amqplib'); const encryptor = require('./encryptor.js'); -const ObjectStorage = require('./objectStorage.js'); +const { ObjectStorage } = require('@elastic.io/object-storage-client'); const co = require('co'); const _ = require('lodash'); const eventToPromise = require('event-to-promise'); @@ -12,7 +12,13 @@ const HEADER_ERROR_RESPONSE = 'x-eio-error-response'; class Amqp { constructor(settings) { this.settings = settings; - this.objectStorage = new ObjectStorage(settings); + this.objectStorage = new ObjectStorage({ + uri: settings.OBJECT_STORAGE_URI, + cipher: { + key: process.env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD, + iv: process.env.ELASTICIO_MESSAGE_CRYPTO_IV + } + }); } connect(uri) { @@ -61,7 +67,7 @@ class Amqp { return encryptor.decryptMessageContent(message.content); } - return await this.objectStorage.getObject(objectId); + return await this.objectStorage.getAsJSON(objectId, this.settings.OBJECT_STORAGE_TOKEN); } async encryptMessage(data, properties) { @@ -71,7 +77,7 @@ class Amqp { let message = ''; try { - properties.headers.objectId = await this.objectStorage.addObject(data); + properties.headers.objectId = await this.objectStorage.addAsJSON(data, this.settings.OBJECT_STORAGE_TOKEN); } catch (e) { log.error('Failed to add message to object storage: %s', e); message = encryptor.encryptMessageContent(data); diff --git a/lib/objectStorage.js b/lib/objectStorage.js deleted file mode 100644 index f6482c0f..00000000 --- a/lib/objectStorage.js +++ /dev/null @@ -1,81 +0,0 @@ -const log = require('./logging.js'); -const encryptor = require('./encryptor.js'); -const uuid = require('uuid'); -const axios = require('axios'); -const http = require('http'); -const https = require('https'); - -class ObjectStorage { - constructor(settings) { - this.api = axios.create({ - baseURL: `${settings.OBJECT_STORAGE_URI}/`, - httpAgent: new http.Agent({ keepAlive: true }), - httpsAgent: new https.Agent({ keepAlive: true }), - headers: { Authorization: `Bearer ${settings.OBJECT_STORAGE_TOKEN}` }, - validateStatus: null - }); - } - - async requestRetry({ maxAttempts, delay, request, onResponse }) { - let attempts = 0; - let res; - let err; - while (attempts < maxAttempts) { - err = null; - res = null; - attempts++; - try { - res = await request(); - } catch (e) { - err = e; - } - if (onResponse && onResponse(err, res)) { - continue; - } - if (err || res.status >= 400) { - log.warn('Error during object get: %s', err ? err : `${res.status} (${res.statusText})`); - await new Promise((resolve) => setTimeout(resolve, delay)); - continue; - } - break; - } - if (err || res.status >= 400) { - throw err || new Error(`HTTP error during object get: ${res.status} (${res.statusText})`); - } - return res; - } - - async getObject(objectId) { - const res = await this.requestRetry({ - maxAttempts: 3, - delay: 100, - request: () => this.api.get(`/objects/${objectId}`, { responseType: 'stream' }) - }); - - return await encryptor.decryptMessageContentStream(res.data); - } - - async addObject(data) { - let objectId = uuid.v4(); - await this.requestRetry({ - maxAttempts: 3, - delay: 100, - request: () => this.api.put( - `/objects/${objectId}`, - encryptor.encryptMessageContentStream(data), - { headers: { 'content-type': 'application/octet-stream' } } - ), - onResponse: (err, res) => { - if (!err && res.status === 409) { - log.warn('Generated already existing UUID'); - objectId = uuid.v4(); - return true; - } - } - }); - - return objectId; - } -} - -module.exports = ObjectStorage; diff --git a/mocha_spec/integration_helpers.js b/mocha_spec/integration_helpers.js index 402d1d7d..6d071477 100644 --- a/mocha_spec/integration_helpers.js +++ b/mocha_spec/integration_helpers.js @@ -6,7 +6,8 @@ const { EventEmitter } = require('events'); const PREFIX = 'sailor_nodejs_integration_test'; const nock = require('nock'); const getStream = require('get-stream'); -const encryptor = require('../lib/encryptor.js'); +const { Readable } = require('stream'); +const { Message } = require('@elastic.io/object-storage-client'); const env = process.env; @@ -184,6 +185,9 @@ function prepareEnv() { env.ELASTICIO_OBJECT_STORAGE_URI = 'http://ma.es.ter'; env.ELASTICIO_OBJECT_STORAGE_ENABLED = ''; env.ELASTICIO_OBJECT_STORAGE_TOKEN = 'jwt'; + + env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD = 'testCryptoPassword'; + env.ELASTICIO_MESSAGE_CRYPTO_IV = 'iv=any16_symbols'; } function mockApiTaskStepResponse(response) { @@ -203,7 +207,14 @@ function mockApiTaskStepResponse(response) { } async function encryptForObjectStorage(input) { - const stream = encryptor.encryptMessageContentStream(input); + const dataStream = new Readable(); + dataStream.push(JSON.stringify(input)); + dataStream.push(null); + const message = new Message({ + key: env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD, + iv: env.ELASTICIO_MESSAGE_CRYPTO_IV + }); + const stream = message.packStream(dataStream); return await getStream.buffer(stream); } diff --git a/mocha_spec/objectStorage.spec.js b/mocha_spec/objectStorage.spec.js deleted file mode 100644 index e9a2376b..00000000 --- a/mocha_spec/objectStorage.spec.js +++ /dev/null @@ -1,136 +0,0 @@ -const nock = require('nock'); -const sinonjs = require('sinon'); -const expect = require('chai').expect; -const getStream = require('get-stream'); -const logging = require('../lib/logging.js'); - -describe('ObjectStorage', () => { - - process.env.ELASTICIO_MESSAGE_CRYPTO_PASSWORD = 'testCryptoPassword'; - process.env.ELASTICIO_MESSAGE_CRYPTO_IV = 'iv=any16_symbols'; - - const envVars = {}; - envVars.ELASTICIO_AMQP_URI = 'amqp://test2/test2'; - envVars.ELASTICIO_FLOW_ID = '5559edd38968ec0736000003'; - envVars.ELASTICIO_STEP_ID = 'step_1'; - envVars.ELASTICIO_EXEC_ID = 'some-exec-id'; - - envVars.ELASTICIO_USER_ID = '5559edd38968ec0736000002'; - envVars.ELASTICIO_COMP_ID = '5559edd38968ec0736000456'; - envVars.ELASTICIO_FUNCTION = 'list'; - - envVars.ELASTICIO_HOOK_SHUTDOWN = true; - - envVars.ELASTICIO_API_URI = 'http://apihost.com'; - envVars.ELASTICIO_API_USERNAME = 'test@test.com'; - envVars.ELASTICIO_API_KEY = '5559edd'; - - envVars.ELASTICIO_OBJECT_STORAGE_URI = 'http://ma.es.ter'; - envVars.ELASTICIO_OBJECT_STORAGE_TOKEN = 'jwt'; - envVars.ELASTICIO_OBJECT_FLOW_ID = 1; - - const ObjectStorage = require('../lib/objectStorage.js'); - const encryptor = require('../lib/encryptor.js'); - const settings = require('../lib/settings.js').readFrom(envVars); - - let sinon; - beforeEach(() => { - sinon = sinonjs.sandbox.create(); - }); - afterEach(() => { - sinon.restore(); - }); - - it('should fail after 3 retries', async () => { - const log = sinon.stub(logging, 'warn'); - const objectStorage = new ObjectStorage(settings); - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .get('/objects/1') - .replyWithError({ code: 'ETIMEDOUT' }) - .get('/objects/1') - .reply(404) - .get('/objects/1') - .replyWithError({ code: 'ENOTFOUND' }); - - let err; - try { - await objectStorage.getObject(1); - } catch (e) { - err = e; - } - - expect(objectStorageCalls.isDone()).to.be.true; - expect(err.code).to.be.equal('ENOTFOUND'); - expect(log.getCall(1).args[1].toString()).to.include('404'); - expect(log.callCount).to.be.equal(3); - }); - - it('should retry get request 3 times on errors', async () => { - const objectStorage = new ObjectStorage(settings); - const data = { test: 'test' }; - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .get('/objects/1') - .reply(500) - .get('/objects/1') - .replyWithError({ code: 'ECONNRESET' }) - .get('/objects/1') - .reply(200, await getStream.buffer(encryptor.encryptMessageContentStream(data))); - - const out = await objectStorage.getObject(1); - - expect(objectStorageCalls.isDone()).to.be.true; - expect(out).to.be.deep.equal(data); - }); - - it('should retry put request 3 times on errors', async () => { - const objectStorage = new ObjectStorage(settings); - const data = { test: 'test' }; - const put = await getStream.buffer(encryptor.encryptMessageContentStream(data)); - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .put(/^\/objects\/[0-9a-z-]+$/, put) - .replyWithError({ code: 'ECONNREFUSED' }) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(400) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(200); - - await objectStorage.addObject(data); - - expect(objectStorageCalls.isDone()).to.be.true; - }); - - it('should retry put request on 409 error with different objectId', async () => { - const log = sinon.stub(logging, 'warn'); - const objectStorage = new ObjectStorage(settings); - const data = { test: 'test' }; - const put = await getStream.buffer(encryptor.encryptMessageContentStream(data)); - - const objectStorageCalls = nock(settings.OBJECT_STORAGE_URI) - .matchHeader('authorization', 'Bearer jwt') - .matchHeader('content-type', 'application/octet-stream') - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(503) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(409) - .put(/^\/objects\/[0-9a-z-]+$/, put) - .reply(200); - const urls = []; - objectStorageCalls.on('request', req => { - urls.push(req.path); - }); - - await objectStorage.addObject(data); - - expect(objectStorageCalls.isDone()).to.be.true; - expect(urls[0]).to.be.equal(urls[1]); - expect(urls[1]).to.not.equal(urls[2]); - expect(log.getCall(1).args[0]).to.include('UUID'); - expect(log.callCount).to.be.equal(2); - }); -}); diff --git a/mocha_spec/run.spec.js b/mocha_spec/run.spec.js index 3ec3deed..6115983b 100644 --- a/mocha_spec/run.spec.js +++ b/mocha_spec/run.spec.js @@ -245,7 +245,6 @@ describe('Integration Test', () => { .get('/customers') .reply(200, customers); - const log = sinon.stub(logging, 'warn'); const objectStorageGet = nock(process.env.ELASTICIO_OBJECT_STORAGE_URI) .get(`/objects/${objectId}`) .matchHeader('authorization', /Bearer/) @@ -282,7 +281,6 @@ describe('Integration Test', () => { expect(body).to.be.null; expect(objectStorageGet.isDone()).to.be.true; expect(objectStoragePut.isDone()).to.be.true; - expect(log.getCall(1).args[1].toString()).to.include('400'); done(); }); diff --git a/package-lock.json b/package-lock.json index 8b29c09d..9b14e7bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,58 @@ { "name": "elasticio-sailor-nodejs", - "version": "2.4.0-dev.29", + "version": "2.4.2-dev.0", "lockfileVersion": 1, "requires": true, "dependencies": { + "@elastic.io/bunyan-logger": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@elastic.io/bunyan-logger/-/bunyan-logger-1.0.5.tgz", + "integrity": "sha512-FcoaG7nTA2H/VuE+0TC1ZKxwEMv3eTVJ42HwrzOY3x3UIpJ9RorG+Sk7G6SBoNuEiBRslGA6Iy9ddE4J3lR2og==", + "requires": { + "bunyan": "1.8.12" + } + }, + "@elastic.io/object-storage-client": { + "version": "0.0.1-dev.7", + "resolved": "https://registry.npmjs.org/@elastic.io/object-storage-client/-/object-storage-client-0.0.1-dev.7.tgz", + "integrity": "sha512-1LSfLxpfhkCiub1NrpCjRghFk9dz8wCMhkXLpZtLg25mP4IMAhjZlx9oJUgVBwRDJH/beJeQSZ/5o6qDLmtjBA==", + "requires": { + "@elastic.io/bunyan-logger": "1.0.5", + "axios": "0.19.0", + "get-stream": "5.1.0", + "jsonwebtoken": "8.5.1", + "uuid": "3.3.2" + }, + "dependencies": { + "axios": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz", + "integrity": "sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==", + "requires": { + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" + } + }, + "follow-redirects": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "requires": { + "debug": "=3.1.0" + } + }, + "is-buffer": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw==" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "abbrev": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", @@ -470,6 +519,11 @@ "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", "dev": true }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -1044,6 +1098,14 @@ "safer-buffer": "^2.1.0" } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -3335,6 +3397,30 @@ "integrity": "sha1-PAQ0dD35Pi9cQq7nsZvLSDV19OE=", "dev": true }, + "jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "requires": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -3347,6 +3433,25 @@ "verror": "1.10.0" } }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "kind-of": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", @@ -3586,6 +3691,11 @@ "lodash._root": "^3.0.0" } }, + "lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" + }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -3598,6 +3708,21 @@ "integrity": "sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U=", "dev": true }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=" + }, + "lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=" + }, "lodash.isobject": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-2.4.1.tgz", @@ -3607,6 +3732,16 @@ "lodash._objecttypes": "~2.4.1" } }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" + }, "lodash.keys": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz", @@ -3618,6 +3753,11 @@ "lodash.isarray": "^3.0.0" } }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, "lodash.restparam": { "version": "3.6.1", "resolved": "https://registry.npmjs.org/lodash.restparam/-/lodash.restparam-3.6.1.tgz", @@ -5138,8 +5278,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safe-json-stringify": { "version": "1.2.0", @@ -5179,8 +5318,7 @@ "semver": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", - "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", - "dev": true + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" }, "send": { "version": "0.14.2", diff --git a/package.json b/package.json index f74b83a1..a32a7284 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "node": ">=10.15.0" }, "dependencies": { + "@elastic.io/object-storage-client": "0.0.1-dev.7", "amqplib": "0.5.1", "axios": "0.18.0", "bunyan": "^1.8.10", From 438787ed5171aff85335c80b995c0d70ad2d8662 Mon Sep 17 00:00:00 2001 From: Paul Annekov Date: Tue, 18 Jun 2019 18:15:01 +0300 Subject: [PATCH 3/6] elasticio#1599 fixed tests --- mocha_spec/integration_helpers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mocha_spec/integration_helpers.js b/mocha_spec/integration_helpers.js index 6d071477..7e3ef193 100644 --- a/mocha_spec/integration_helpers.js +++ b/mocha_spec/integration_helpers.js @@ -8,6 +8,7 @@ const nock = require('nock'); const getStream = require('get-stream'); const { Readable } = require('stream'); const { Message } = require('@elastic.io/object-storage-client'); +const encryptor = require('../lib/encryptor'); const env = process.env; @@ -135,7 +136,7 @@ class AmqpHelper extends EventEmitter { this.publishChannel.ack(message); const content = message.content.toString(); - const emittedMessage = content ? JSON.parse(content) : content; + const emittedMessage = content ? encryptor.decryptMessageContent(message.content) : content; const data = { properties: message.properties, From 221449f09f070e744613713ddb5ddea40b11b872 Mon Sep 17 00:00:00 2001 From: Paul Annekov Date: Tue, 18 Jun 2019 18:17:02 +0300 Subject: [PATCH 4/6] elasticio#1599 2.4.2-dev.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a32a7284..987ebbf6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticio-sailor-nodejs", "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors", - "version": "2.4.2-dev.0", + "version": "2.4.2-dev.2", "main": "run.js", "scripts": { "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js", From edec8d00935a54ffed4491905e039a9eb37dd183 Mon Sep 17 00:00:00 2001 From: Paul Annekov Date: Wed, 19 Jun 2019 18:43:29 +0300 Subject: [PATCH 5/6] 2.4.2-dev.3 --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 987ebbf6..006962a7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticio-sailor-nodejs", "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors", - "version": "2.4.2-dev.2", + "version": "2.4.2-dev.3", "main": "run.js", "scripts": { "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js", @@ -15,7 +15,7 @@ "node": ">=10.15.0" }, "dependencies": { - "@elastic.io/object-storage-client": "0.0.1-dev.7", + "@elastic.io/object-storage-client": "0.0.1-dev.8", "amqplib": "0.5.1", "axios": "0.18.0", "bunyan": "^1.8.10", From 70a0706a77cfc218b2a649e4920dfc86488efc5c Mon Sep 17 00:00:00 2001 From: Paul Annekov Date: Wed, 19 Jun 2019 19:18:03 +0300 Subject: [PATCH 6/6] elasticio#1599 2.4.2-dev.4 --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 006962a7..127a4d41 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticio-sailor-nodejs", "description": "The official elastic.io library for bootstrapping and executing for Node.js connectors", - "version": "2.4.2-dev.3", + "version": "2.4.2-dev.4", "main": "run.js", "scripts": { "lint": "./node_modules/.bin/eslint lib spec mocha_spec lib run.js runService.js", @@ -15,7 +15,7 @@ "node": ">=10.15.0" }, "dependencies": { - "@elastic.io/object-storage-client": "0.0.1-dev.8", + "@elastic.io/object-storage-client": "0.0.1-dev.9", "amqplib": "0.5.1", "axios": "0.18.0", "bunyan": "^1.8.10",