feat: rework the nodesToString function to output expected element tags (#234)

cheton · web-flow · commit 9e91b0ca74aa · 2022-07-08T20:54:51.000+08:00
* feat: rework the "nodes-to-string" function to output expected element tags

* chore: bump to v4

* fix: filter out empty nodes

* rollback: rollback to CJS exports

* chore: fix eslint errors

* docs: update README.md

* docs: update README.md
diff --git a/README.md b/README.md
@@ -123,10 +123,15 @@ module.exports = {
             fallbackKey: function(ns, value) {
                 return value;
             },
+
+            // https://react.i18next.com/latest/trans-component#usage-with-simple-html-elements-like-less-than-br-greater-than-and-others-v10.4.0
+            supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
+            keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.
+
+            // https://github.com/acornjs/acorn/tree/master/acorn#interface
             acorn: {
                 ecmaVersion: 2020,
                 sourceType: 'module', // defaults to 'module'
-                // Check out https://github.com/acornjs/acorn/tree/master/acorn#interface for additional options
             }
         },
         lngs: ['en','de'],
@@ -148,7 +153,9 @@ module.exports = {
         interpolation: {
             prefix: '{{',
             suffix: '}}'
-        }
+        },
+        metadata: {},
+        allowDynamicKeys: false,
     },
     transform: function customTransform(file, enc, done) {
         "use strict";
@@ -498,18 +505,28 @@ Below are the configuration options with their default values:
     sort: false,
     attr: {
         list: ['data-i18n'],
-        extensions: ['.html', '.htm']
+        extensions: ['.html', '.htm'],
     },
     func: {
         list: ['i18next.t', 'i18n.t'],
-        extensions: ['.js', '.jsx']
+        extensions: ['.js', '.jsx'],
     },
     trans: {
         component: 'Trans',
         i18nKey: 'i18nKey',
         defaultsKey: 'defaults',
         extensions: ['.js', '.jsx'],
-        fallbackKey: false
+        fallbackKey: false,
+
+        // https://react.i18next.com/latest/trans-component#usage-with-simple-html-elements-like-less-than-br-greater-than-and-others-v10.4.0
+        supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
+        keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.
+
+        // https://github.com/acornjs/acorn/tree/master/acorn#interface
+        acorn: {
+            ecmaVersion: 2020,
+            sourceType: 'module', // defaults to 'module'
+        },
     },
     lngs: ['en'],
     ns: ['translation'],
@@ -520,7 +537,7 @@ Below are the configuration options with their default values:
         loadPath: 'i18n/{{lng}}/{{ns}}.json',
         savePath: 'i18n/{{lng}}/{{ns}}.json',
         jsonIndent: 2,
-        lineEnding: '\n'
+        lineEnding: '\n',
     },
     nsSeparator: ':',
     keySeparator: '.',
@@ -529,8 +546,10 @@ Below are the configuration options with their default values:
     contextDefaultValues: [],
     interpolation: {
         prefix: '{{',
-        suffix: '}}'
-    }
+        suffix: '}}',
+    },
+    metadata: {},
+    allowDynamicKeys: false,
 }
 ```
 
@@ -606,7 +625,17 @@ If an `Object` is supplied, you can specify a list of extensions, or override th
         i18nKey: 'i18nKey',
         defaultsKey: 'defaults',
         extensions: ['.js', '.jsx'],
-        fallbackKey: false
+        fallbackKey: false,
+
+        // https://react.i18next.com/latest/trans-component#usage-with-simple-html-elements-like-less-than-br-greater-than-and-others-v10.4.0
+        supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
+        keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.
+
+        // https://github.com/acornjs/acorn/tree/master/acorn#interface
+        acorn: {
+            ecmaVersion: 2020,
+            sourceType: 'module', // defaults to 'module'
+        },
     }
 }
 ```
@@ -819,9 +848,6 @@ interpolation options
 }
 ```
 
-## Integration Guide
-Checkout [Integration Guide](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide) to learn how to integrate with [React](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide#react), [Gettext Style I18n](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide#gettext-style-i18n), and [Handlebars](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide#handlebars).
-
 #### metadata
 
 Type: `Object` Default: `{}`
@@ -881,6 +907,9 @@ Example Usage:
     done();
 ```
 
+## Integration Guide
+Checkout [Integration Guide](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide) to learn how to integrate with [React](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide#react), [Gettext Style I18n](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide#gettext-style-i18n), and [Handlebars](https://github.com/i18next/i18next-scanner/wiki/Integration-Guide#handlebars).
+
 ## License
 
 MIT
diff --git a/bin/cli.js b/bin/cli.js
@@ -2,10 +2,10 @@
 
 const path = require('path');
 const program = require('commander');
-const ensureArray = require('ensure-array');
+const { ensureArray } = require('ensure-type');
 const sort = require('gulp-sort');
 const vfs = require('vinyl-fs');
-const scanner = require('../lib').default;
+const scanner = require('../lib');
 const pkg = require('../package.json');
 
 program
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "i18next-scanner",
-  "version": "3.3.0",
+  "version": "4.0.0",
   "description": "Scan your code, extract translation keys/values, and merge them into i18n resource files.",
   "homepage": "https://github.com/i18next/i18next-scanner",
   "author": "Cheton Wu <cheton@gmail.com>",
@@ -57,7 +57,7 @@
     "clone-deep": "^4.0.0",
     "commander": "^9.0.0",
     "deepmerge": "^4.0.0",
-    "ensure-array": "^1.0.0",
+    "ensure-type": "^1.5.0",
     "eol": "^0.9.1",
     "esprima-next": "^5.7.0",
     "gulp-sort": "^2.0.0",
diff --git a/src/index.js b/src/index.js
@@ -1,4 +1,4 @@
-/* eslint-disable no-buffer-constructor */
+/* eslint-disable import/no-import-module-exports */
 import fs from 'fs';
 import path from 'path';
 import eol from 'eol';
@@ -93,7 +93,7 @@ const flush = (parser, customFlush) => {
           contents = Buffer.from(text);
         } catch (e) {
           // Fallback to "new Buffer(string[, encoding])" which is deprecated since Node.js v6.0.0
-          contents = new Buffer(text);
+          contents = new Buffer(text); // eslint-disable-line no-buffer-constructor
         }
 
         this.push(new VirtualFile({
@@ -121,9 +121,11 @@ const createStream = (options, customTransform, customFlush) => {
   return stream;
 };
 
-export default (...args) => createStream(...args);
+// Convenience API
+module.exports = (...args) => module.exports.createStream(...args);
 
-export {
-  createStream,
-  Parser,
-};
+// Basic API
+module.exports.createStream = createStream;
+
+// Parser
+module.exports.Parser = Parser;
diff --git a/src/nodes-to-string.js b/src/nodes-to-string.js
@@ -1,3 +1,4 @@
+import { ensureArray, ensureBoolean, ensureString } from 'ensure-type';
 import _get from 'lodash/get';
 
 const isJSXText = (node) => {
@@ -32,16 +33,26 @@ const isObjectExpression = (node) => {
   return node.type === 'ObjectExpression';
 };
 
-const nodesToString = (nodes, code) => {
-  let memo = '';
-  let nodeIndex = 0;
-  nodes.forEach((node, i) => {
-    if (isJSXText(node) || isStringLiteral(node)) {
-      const value = (node.value)
-        .replace(/^[\r\n]+\s*/g, '') // remove leading spaces containing a leading newline character
-        .replace(/[\r\n]+\s*$/g, '') // remove trailing spaces containing a leading newline character
-        .replace(/[\r\n]+\s*/g, ' '); // replace spaces containing a leading newline character with a single space character
+const trimValue = value => ensureString(value)
+  .replace(/^[\r\n]+\s*/g, '') // remove leading spaces containing a leading newline character
+  .replace(/[\r\n]+\s*$/g, '') // remove trailing spaces containing a leading newline character
+  .replace(/[\r\n]+\s*/g, ' '); // replace spaces containing a leading newline character with a single space character
+
+const nodesToString = (nodes, options) => {
+  const supportBasicHtmlNodes = ensureBoolean(options?.supportBasicHtmlNodes);
+  const keepBasicHtmlNodesFor = ensureArray(options?.keepBasicHtmlNodesFor);
+  const filteredNodes = ensureArray(nodes)
+    .filter(node => {
+      if (isJSXText(node)) {
+        return trimValue(node.value);
+      }
+      return true;
+    });
 
+  let memo = '';
+  filteredNodes.forEach((node, nodeIndex) => {
+    if (isJSXText(node)) {
+      const value = trimValue(node.value);
       if (!value) {
         return;
       }
@@ -55,17 +66,44 @@ const nodesToString = (nodes, code) => {
       } if (isStringLiteral(expression)) {
         memo += expression.value;
       } else if (isObjectExpression(expression) && (_get(expression, 'properties[0].type') === 'Property')) {
-        memo += `<${nodeIndex}>{{${expression.properties[0].key.name}}}</${nodeIndex}>`;
+        memo += `{{${expression.properties[0].key.name}}}`;
       } else {
         console.error(`Unsupported JSX expression. Only static values or {{interpolation}} blocks are supported. Got ${expression.type}:`);
-        console.error(code.slice(node.start, node.end));
+        console.error(ensureString(options?.code).slice(node.start, node.end));
         console.error(node.expression);
       }
     } else if (node.children) {
-      memo += `<${nodeIndex}>${nodesToString(node.children, code)}</${nodeIndex}>`;
-    }
+      const nodeType = node.openingElement?.name?.name;
+      const selfClosing = node.openingElement?.selfClosing;
+      const attributeCount = ensureArray(node.openingElement?.attributes).length;
+      const filteredChildNodes = ensureArray(node.children)
+        .filter(childNode => {
+          if (isJSXText(childNode)) {
+            return trimValue(childNode.value);
+          }
+          return true;
+        });
+      const childCount = filteredChildNodes.length;
+      const firstChildNode = filteredChildNodes[0];
+      const shouldKeepChild = supportBasicHtmlNodes && keepBasicHtmlNodesFor.indexOf(node.openingElement?.name?.name) > -1;
 
-    ++nodeIndex;
+      if (selfClosing && shouldKeepChild && (attributeCount === 0)) {
+        // actual e.g. lorem <br/> ipsum
+        // expected e.g. lorem <br/> ipsum
+        memo += `<${nodeType}/>`;
+      } else if ((childCount === 0 && !shouldKeepChild) || (childCount === 0 && attributeCount !== 0)) {
+        // actual e.g. lorem <hr className="test" /> ipsum
+        // expected e.g. lorem <0></0> ipsum
+        memo += `<${nodeIndex}></${nodeIndex}>`;
+      } else if (shouldKeepChild && (attributeCount === 0) && (childCount === 1) && (isJSXText(firstChildNode) || isStringLiteral(firstChildNode?.expression))) {
+        // actual e.g. dolor <strong>bold</strong> amet
+        // expected e.g. dolor <strong>bold</strong> amet
+        memo += `<${nodeType}>${nodesToString(node.children, options)}</${nodeType}>`;
+      } else {
+        // regular case mapping the inner children
+        memo += `<${nodeIndex}>${nodesToString(node.children, options)}</${nodeIndex}>`;
+      }
+    }
   });
 
   return memo;
diff --git a/src/parser.js b/src/parser.js
@@ -7,7 +7,7 @@ import acornStage3 from 'acorn-stage3';
 import chalk from 'chalk';
 import cloneDeep from 'clone-deep';
 import deepMerge from 'deepmerge';
-import ensureArray from 'ensure-array';
+import { ensureArray } from 'ensure-type';
 import { parse } from 'esprima-next';
 import _ from 'lodash';
 import parse5 from 'parse5';
@@ -43,11 +43,13 @@ const defaults = {
     defaultsKey: 'defaults',
     extensions: ['.js', '.jsx'],
     fallbackKey: false,
+    supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys.
+    keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.
     acorn: {
       ecmaVersion: 2020, // defaults to 2020
       sourceType: 'module', // defaults to 'module'
       // Check out https://github.com/acornjs/acorn/tree/master/acorn#interface for additional options
-    }
+    },
   },
 
   lngs: ['en'], // array of supported languages
@@ -171,6 +173,12 @@ const normalizeOptions = (options) => {
     if (_.isUndefined(_.get(options, 'trans.acorn'))) {
       _.set(options, 'trans.acorn', defaults.trans.acorn);
     }
+    if (_.isUndefined(_.get(options, 'trans.supportBasicHtmlNodes'))) {
+      _.set(options, 'trans.supportBasicHtmlNodes', defaults.trans.supportBasicHtmlNodes);
+    }
+    if (_.isUndefined(_.get(options, 'trans.keepBasicHtmlNodesFor'))) {
+      _.set(options, 'trans.keepBasicHtmlNodesFor', defaults.trans.keepBasicHtmlNodesFor);
+    }
   }
 
   // Resource
@@ -538,6 +546,8 @@ class Parser {
       defaultsKey = this.options.trans.defaultsKey, // string
       fallbackKey, // boolean|function
       acorn: acornOptions = this.options.trans.acorn, // object
+      supportBasicHtmlNodes = this.options.trans.supportBasicHtmlNodes, // boolean
+      keepBasicHtmlNodesFor = this.options.trans.keepBasicHtmlNodesFor, // array
     } = { ...opts };
 
     const parseJSXElement = (node, code) => {
@@ -634,7 +644,11 @@ class Parser {
       const tOptions = attr.tOptions;
       const options = {
         ...tOptions,
-        defaultValue: defaultsString || nodesToString(node.children, code),
+        defaultValue: defaultsString || nodesToString(node.children, {
+          code,
+          supportBasicHtmlNodes,
+          keepBasicHtmlNodesFor,
+        }),
         fallbackKey: fallbackKey || this.options.trans.fallbackKey
       };
 
diff --git a/test/jsx-parser.test.js b/test/jsx-parser.test.js
diff --git a/test/parser.test.js b/test/parser.test.js
diff --git a/test/transform-stream.test.js b/test/transform-stream.test.js
