perf(export): improve memory usage when exporting data (#1029)

DayTF · web-flow · commit ed6af03b1add · 2024-05-17T09:35:18.000+02:00
diff --git a/package.json b/package.json
@@ -35,7 +35,7 @@
     "body-parser": "^1.20.1",
     "compose-middleware": "5.0.1",
     "cors": "2.8.5",
-    "csv-stringify": "1.0.4",
+    "csv-stringify": "6.5.0",
     "express": "^4.18.2",
     "express-jwt": "8.4.1",
     "forest-ip-utils": "1.0.1",
diff --git a/src/routes/associations.js b/src/routes/associations.js
@@ -77,7 +77,7 @@ module.exports = function Associations(app, model, Implementation, integrator, o
       .catch(next);
   }
 
-  function exportCSV(request, response, next) {
+  async function exportCSV(request, response, next) {
     const { params, associationModel } = getContext(request);
 
     const recordsExporter = new Implementation.ResourcesExporter(
diff --git a/src/routes/resources.js b/src/routes/resources.js
@@ -48,7 +48,7 @@ module.exports = function Resources(app, model, { configStore } = inject()) {
       .catch(next);
   };
 
-  this.exportCSV = (request, response, next) => {
+  this.exportCSV = async (request, response, next) => {
     const params = request.query;
     const recordsExporter = new Implementation.ResourcesExporter(
       model,
diff --git a/src/services/csv-exporter.js b/src/services/csv-exporter.js
@@ -1,21 +1,40 @@
-const P = require('bluebird');
 const moment = require('moment');
-const stringify = require('csv-stringify');
+// eslint-disable-next-line import/no-unresolved
+const { stringify } = require('csv-stringify/sync');
 const { inject } = require('@forestadmin/context');
 const ParamsFieldsDeserializer = require('../deserializers/params-fields');
 const SmartFieldsValuesInjector = require('./smart-fields-values-injector');
 
 // NOTICE: Prevent bad date formatting into timestamps.
 const CSV_OPTIONS = {
-  formatters: {
+  cast: {
     date: (value) => moment(value).format(),
   },
 };
 
 function CSVExporter(params, response, modelName, recordsExporter) {
   const { configStore } = inject();
 
-  this.perform = () => {
+  function getValueForAttribute(record, attribute) {
+    let value;
+    if (params.fields[attribute]) {
+      if (record[attribute]) {
+        if (params.fields[attribute] && record[attribute][params.fields[attribute]]) {
+          value = record[attribute][params.fields[attribute]];
+        } else {
+          // eslint-disable-next-line
+              value = record[attribute].id || record[attribute]._id;
+        }
+      }
+    } else {
+      value = record[attribute];
+    }
+
+    return value || '';
+  }
+
+  // eslint-disable-next-line sonarjs/cognitive-complexity
+  this.perform = async () => {
     const filename = `${params.filename}.csv`;
     response.setHeader('Content-Type', 'text/csv; charset=utf-8');
     response.setHeader('Content-disposition', `attachment; filename=${filename}`);
@@ -32,47 +51,24 @@ function CSVExporter(params, response, modelName, recordsExporter) {
 
     const fieldsPerModel = new ParamsFieldsDeserializer(params.fields).perform();
 
-    return recordsExporter
-      .perform((records) => P
-        .map(records, (record) =>
-          new SmartFieldsValuesInjector(record, modelName, fieldsPerModel).perform())
-        .then((recordsWithSmartFieldsValues) =>
-          new P((resolve) => {
-            if (configStore.Implementation.Flattener) {
-              recordsWithSmartFieldsValues = configStore.Implementation.Flattener
-                .flattenRecordsForExport(modelName, recordsWithSmartFieldsValues);
-            }
+    await recordsExporter
+      .perform(async (records) => {
+        await Promise.all(
+          // eslint-disable-next-line max-len
+          records.map((record) => new SmartFieldsValuesInjector(record, modelName, fieldsPerModel).perform()),
+        );
 
-            const CSVLines = [];
-            recordsWithSmartFieldsValues.forEach((record) => {
-              const CSVLine = [];
-              CSVAttributes.forEach((attribute) => {
-                let value;
-                if (params.fields[attribute]) {
-                  if (record[attribute]) {
-                    if (params.fields[attribute] && record[attribute][params.fields[attribute]]) {
-                      value = record[attribute][params.fields[attribute]];
-                    } else {
-                      // eslint-disable-next-line
-                      value = record[attribute].id || record[attribute]._id;
-                    }
-                  }
-                } else {
-                  value = record[attribute];
-                }
-                CSVLine.push(value || '');
-              });
-              CSVLines.push(CSVLine);
-            });
+        if (configStore.Implementation.Flattener) {
+          records = configStore.Implementation.Flattener
+            .flattenRecordsForExport(modelName, records);
+        }
 
-            stringify(CSVLines, CSV_OPTIONS, (error, csv) => {
-              response.write(csv);
-              resolve();
-            });
-          })))
-      .then(() => {
-        response.end();
+        records.forEach((record) => {
+          // eslint-disable-next-line max-len
+          response.write(stringify([CSVAttributes.map((attribute) => getValueForAttribute(record, attribute, params))], CSV_OPTIONS));
+        });
       });
+    response.end();
   };
 }
 
diff --git a/src/services/smart-fields-values-injector.js b/src/services/smart-fields-values-injector.js
@@ -1,5 +1,4 @@
 const _ = require('lodash');
-const P = require('bluebird');
 const logger = require('./logger');
 const Schemas = require('../generators/schemas');
 
@@ -74,60 +73,63 @@ function SmartFieldsValuesInjector(
       && fieldsPerModel[modelNameToCheck].indexOf(fieldName) !== -1;
   }
 
-  this.perform = () =>
-    P.each(schema.fields, (field) => {
-      const fieldWasRequested = isRequestedField(requestedField || modelName, field.field);
+  // eslint-disable-next-line sonarjs/cognitive-complexity
+  function injectSmartFieldValue(field) {
+    const fieldWasRequested = isRequestedField(requestedField || modelName, field.field);
 
-      if (record && field.isVirtual && (field.get || field.value)) {
-        if (fieldsPerModel && !fieldWasRequested) {
-          return null;
-        }
-
-        return setSmartFieldValue(record, field, modelName);
+    if (record && field.isVirtual && (field.get || field.value)) {
+      if (fieldsPerModel && !fieldWasRequested) {
+        return null;
       }
 
-      if (
-        !record[field.field]
+      return setSmartFieldValue(record, field, modelName);
+    }
+
+    if (
+      !record[field.field]
         && _.isArray(field.type)
         && (field.relationship || field.isVirtual)) {
-        // Add empty arrays on relation fields so that JsonApiSerializer add the relevant
-        // `data.x.relationships` section in the response.
-        //
-        // The field must match the following condition
-        // - field is a real or a smart HasMany / BelongsToMany relation
-        // - field is NOT an 'embedded' relationship (@see mongoose)
-
-        record[field.field] = [];
-      } else if (field.reference && !_.isArray(field.type)) {
-        // NOTICE: Set Smart Fields values to "belongsTo" associated records.
-        const modelNameAssociation = getReferencedModelName(field);
-        const schemaAssociation = Schemas.schemas[modelNameAssociation];
-
-        if (schemaAssociation && !_.isArray(field.type)) {
-          return P.each(schemaAssociation.fields, (fieldAssociation) => {
-            if (record
+      // Add empty arrays on relation fields so that JsonApiSerializer add the relevant
+      // `data.x.relationships` section in the response.
+      //
+      // The field must match the following condition
+      // - field is a real or a smart HasMany / BelongsToMany relation
+      // - field is NOT an 'embedded' relationship (@see mongoose)
+
+      record[field.field] = [];
+    } else if (field.reference && !_.isArray(field.type)) {
+      // NOTICE: Set Smart Fields values to "belongsTo" associated records.
+      const modelNameAssociation = getReferencedModelName(field);
+      const schemaAssociation = Schemas.schemas[modelNameAssociation];
+
+      if (schemaAssociation && !_.isArray(field.type)) {
+        return Promise.all(schemaAssociation.fields.map((fieldAssociation) => {
+          if (record
                 && record[field.field]
                 && fieldAssociation.isVirtual
                 && (fieldAssociation.get || fieldAssociation.value)) {
-              if (fieldsPerModel && !isRequestedField(field.field, fieldAssociation.field)) {
-                return null;
-              }
-
-              return setSmartFieldValue(
-                record[field.field],
-                fieldAssociation,
-                modelNameAssociation,
-              );
+            if (fieldsPerModel && !isRequestedField(field.field, fieldAssociation.field)) {
+              return null;
             }
 
-            return null;
-          });
-        }
+            return setSmartFieldValue(
+              record[field.field],
+              fieldAssociation,
+              modelNameAssociation,
+            );
+          }
+
+          return null;
+        }));
       }
+    }
+
+    return null;
+  }
 
-      return null;
-    })
-      .thenReturn(record);
+  this.perform = async () => Promise.all(
+    schema.fields.map((field) => injectSmartFieldValue(field)),
+  );
 }
 
 module.exports = SmartFieldsValuesInjector;
diff --git a/yarn.lock b/yarn.lock
@@ -3746,12 +3746,10 @@ cssesc@^3.0.0:
   resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee"
   integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==
 
-csv-stringify@1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-1.0.4.tgz#bc18bab9ad4cef3195fd257980b58b479c42d3e5"
-  integrity sha1-vBi6ua1M7zGV/SV5gLWLR5xC0+U=
-  dependencies:
-    lodash.get "^4.0.0"
+csv-stringify@6.5.0:
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.0.tgz#7b1491893c917e018a97de9bf9604e23b88647c2"
+  integrity sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==
 
 dargs@^7.0.0:
   version "7.0.0"
@@ -6570,7 +6568,7 @@ lodash.escaperegexp@^4.1.2:
   resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347"
   integrity sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==
 
-lodash.get@^4.0.0, lodash.get@^4.4.2:
+lodash.get@^4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
   integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=

Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ module.exports = function Associations(app, model, Implementation, integrator, o`
`77`	`77`	`.catch(next);`
`78`	`78`	`}`
`79`	`79`
`80`		`- function exportCSV(request, response, next) {`
	`80`	`+ async function exportCSV(request, response, next) {`
`81`	`81`	`const { params, associationModel } = getContext(request);`
`82`	`82`
`83`	`83`	`const recordsExporter = new Implementation.ResourcesExporter(`