Fixed issue with attachments in Bulk (#58)

Author: if0s

.circleci/config.yml

@@ -77,12 +77,12 @@ jobs:
       - checkout
       - node/install:
           node-version: << pipeline.parameters.node-version >>
-      - run:
-          name: Audit Dependencies
-          command: npm audit --production --audit-level=high
       - node/install-packages:
           cache-path: ./node_modules
           override-ci-command: npm install
+      - run:
+          name: Audit Dependencies
+          command: npm run audit
       - run:
           name: Running Mocha Tests
           command: npm test
@@ -118,6 +118,8 @@ workflows:
     jobs:
       - build:
           name: "Build and publish docker image"
+          context:
+            - componentspusher
           filters:
             branches:
               ignore: /.*/

.grype-ignore.yaml

@@ -0,0 +1,10 @@
+ignore:
+  - vulnerability: CVE-2022-3996
+    package:
+      name: libssl3
+      version: 3.0.7-r0
+
+  - vulnerability: CVE-2022-3996
+    package:
+      name: libcrypto3
+      version: 3.0.7-r0

.nsprc

@@ -0,0 +1,6 @@
+{
+    "GHSA-27h2-hvpr-p74q": {
+            "active": true,
+            "notes": "We don't use verify function from jsonwebtoken, so not affected"
+    }
+}

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.5.0 (January 13, 2023)
+* Fixed issue with attachments in `Bulk Create/Update/Delete/Upsert` action
+* Added ability to directly provide url to csv file in `Bulk Create/Update/Delete/Upsert` action
+* Update Sailor version to 2.7.1
+* Update component-commons-library version to 3.1.5
+
 ## 2.4.2 (November 18, 2022)
 * Improved error handling in `Lookup Objects` action
 

component.json

@@ -3,7 +3,7 @@
   "description": "Customer relationship management (CRM) software & cloud computing from the leader in CRM solutions for businesses large & small.",
   "docsUrl": "https://github.com/elasticio/salesforce-component-v2",
   "url": "http://www.salesforce.com/",
-  "version": "2.4.2",
+  "version": "2.5.0",
   "authClientTypes": [
     "oauth2"
   ],

lib/actions/bulk_cud.js

@@ -1,11 +1,11 @@
 const { messages } = require('elasticio-node');
-const { Readable } = require('stream');
-const util = require('../util');
+const { AttachmentProcessor } = require('@elastic.io/component-commons-library');
+const { getUserAgent } = require('../util');
 const { callJSForceMethod } = require('../helpers/wrapper');
 
 const DEFAULT_TIMEOUT = 600; // 10 min
 
-exports.objectTypes = async function objectTypes(configuration) {
+module.exports.objectTypes = async function objectTypes(configuration) {
   switch (configuration.operation) {
     case 'insert': {
       return callJSForceMethod.call(this, configuration, 'getCreateableObjectTypes');
@@ -20,27 +20,26 @@ exports.objectTypes = async function objectTypes(configuration) {
   }
 };
 
-exports.process = async function bulkCUD(message, configuration) {
+module.exports.process = async function bulkCUD(message, configuration) {
   this.logger.info('Starting Bulk %s action', configuration.operation);
   // Get CSV from attachment
-  if (!message.attachments || Object.keys(message.attachments).length === 0) {
-    this.logger.error('Attachment not found');
-    return messages.newMessageWithBody({ });
-  }
-
-  const key = Object.keys(message.attachments)[0];
 
   let timeout = parseInt(configuration.timeout, 10);
   if (Number.isNaN(timeout) || (timeout < 0)) {
     this.logger.warn(`Timeout is incorrect. Set default value: ${DEFAULT_TIMEOUT} sec.`);
     timeout = DEFAULT_TIMEOUT;
   }
+  let { attachmentUrl } = message.body;
+  if (!attachmentUrl) {
+    if (!message.attachments || Object.keys(message.attachments).length === 0 || !message.attachments[Object.keys(message.attachments)[0]].url) {
+      this.logger.error('Attachment not found');
+      return messages.newMessageWithBody({});
+    }
+    attachmentUrl = message.attachments[Object.keys(message.attachments)[0]].url;
+  }
 
-  const result = await util.downloadAttachment(message.attachments[key].url);
-
-  const csvStream = new Readable();
-  csvStream.push(result);
-  csvStream.push(null);
+  const attachmentProcessor = new AttachmentProcessor(getUserAgent(), message.id);
+  const { data: csvStream } = await attachmentProcessor.getAttachment(attachmentUrl, 'stream');
 
   let extra;
   if (configuration.operation === 'upsert') {
@@ -54,33 +53,36 @@ exports.process = async function bulkCUD(message, configuration) {
   };
   const job = await callJSForceMethod.call(this, configuration, 'bulkCreateJob', batchOptions);
   const batch = job.createBatch();
-  return new Promise((resolve, reject) => {
-    // Upload CSV to SF
-    batch.execute(csvStream)
-      // eslint-disable-next-line no-unused-vars
-      .on('queue', () => {
-        // Check while job status become JobComplete or Failed, Aborted
-        batch.poll(1000, timeout * 1000);
-      }).on('response', (rets) => {
-        // Retrieve the results of the completed job
-        // emit results
-        this.logger.debug('Result receiver. Records count:', rets.length);
-        const out = messages.newEmptyMessage();
-        out.body = { result: rets };
-        resolve(out);
-      }).on('error', (err) => {
-        this.logger.error('Job error');
-        reject(new Error(err.message));
-        throw Error(`Job error: ${err.message}`);
-      });
-  });
+  try {
+    const result = await new Promise((resolve, reject) => {
+      batch.execute(csvStream)
+        .on('queue', () => {
+          batch.poll(1000, timeout * 1000);
+        }).on('response', (rets) => {
+          resolve(rets);
+        }).on('error', (err) => {
+          reject(err);
+        });
+    });
+    this.logger.debug(`Result receiver. Records count: ${result.length}`);
+    return messages.newMessageWithBody({ result });
+  } catch (err) {
+    this.logger.error(`Job error: ${err.message}`);
+    throw err;
+  }
 };
 
-exports.getMetaModel = async function getMetaModel(configuration) {
+module.exports.getMetaModel = async function getMetaModel(configuration) {
   const meta = {
     in: {
       type: 'object',
-      properties: {},
+      properties: {
+        attachmentUrl: {
+          type: 'string',
+          required: false,
+          title: 'Attachment Url',
+        },
+      },
     },
     out: {
       type: 'object',

lib/util.js

@@ -47,18 +47,6 @@ module.exports.uploadAttachment = async (url, payload) => {
   });
 };
 
-module.exports.downloadAttachment = async (url) => {
-  const ax = axios.create();
-  addRetryCountInterceptorToAxios(ax);
-  const response = await ax.get(url, {
-    method: 'GET',
-    timeout: REQUEST_TIMEOUT,
-    retry: REQUEST_MAX_RETRY,
-    delay: REQUEST_RETRY_DELAY,
-  });
-  return response.data;
-};
-
 function getSecretUri(secretId, isRefresh) {
   const parsedUrl = new URL(process.env.ELASTICIO_API_URI);
   parsedUrl.username = process.env.ELASTICIO_API_USERNAME;