feat(options): ignore whitespace while parsing html and wrap top level text nodes in paragraphs

Author: oleast

src/converters.ts

@@ -104,3 +104,14 @@ export const convertTextNodeToText: TextConverter = (node, marks) => {
     data: {},
   };
 };
+
+export const convertTextNodeToParagraphedText: TagConverter<Block> = (
+  node,
+  next,
+) => {
+  return {
+    nodeType: BLOCKS.PARAGRAPH,
+    data: {},
+    content: next(node),
+  };
+};

src/htmlStringToDocument.ts

@@ -16,14 +16,17 @@ import {
   BLOCKS,
 } from "@contentful/rich-text-types";
 import type {
+  HTMLElementNode,
   HTMLNode,
   HTMLTagName,
+  HTMLTextNode,
   Next,
   Options,
   OptionsWithDefaults,
   TagConverter,
+  TextConverter,
 } from "./types";
-import { createDocumentNode, getAsList } from "./utils";
+import { createDocumentNode, getAsList, isWhiteSpace } from "./utils";
 
 const DEFAULT_TAG_CONVERTERS: Partial<
   Record<HTMLTagName, TagConverter<Block | Inline | Text>>
@@ -58,11 +61,9 @@ const mapHtmlNodeToRichTextNode = (
   node: HTMLNode,
   marks: Mark[],
   options: OptionsWithDefaults,
+  isTopLevel = false,
 ) => {
   const { convertText, convertTag } = options;
-  if (node.type === "text") {
-    return convertText(node, marks);
-  }
 
   const mapChildren: Next = (node, mark) => {
     const newMarks = mark ? getAsList(mark) : [];
@@ -74,10 +75,11 @@ const mapHtmlNodeToRichTextNode = (
     }
     return getAsList(mapHtmlNodeToRichTextNode(node, allMarks, options));
   };
+  const next = mapChildren;
 
-  const next: Next = (node, marks) => {
-    return mapChildren(node, marks);
-  };
+  if (node.type === "text") {
+    return convertText(node, marks);
+  }
 
   const tagConverter = convertTag?.[node.tagName] ?? convertTagToChildren;
   const convertedNode = tagConverter(node, next);
@@ -94,10 +96,13 @@ export const htmlStringToDocument = (
       ...options.convertTag,
     },
     convertText: options.convertText ?? convertTextNodeToText,
+    wrapTopLevelTextNodesInParagraphs: false,
+    ignoreWhiteSpace: false,
+    isWhiteSpace: options.isWhiteSpace ?? isWhiteSpace,
   };
   const parsedHtml = parseHtml(htmlString);
   const richTextNodes = parsedHtml.flatMap((node) =>
-    mapHtmlNodeToRichTextNode(node, [], optionsWithDefaults),
+    mapHtmlNodeToRichTextNode(node, [], optionsWithDefaults, true),
   );
 
   const richTextNodesWithTopLevelTextNodesConverted: TopLevelBlock[] =

src/parseHtml.ts

@@ -6,7 +6,7 @@ import {
   TextNode,
   CommentNode,
 } from "parse5/dist/tree-adapters/default";
-import { isNotNull } from "./utils";
+import { isNotNull, isWhiteSpace } from "./utils";
 
 import type { HTMLNode, HTMLTagName } from "./types";
 
@@ -28,6 +28,10 @@ const isChildNodeDocumentType = (
   return childNode.nodeName === "#documentType";
 };
 
+const isTextNodePureWhiteSpace = (textNode: TextNode): boolean => {
+  return isWhiteSpace(textNode.value);
+}
+
 const mapChildNodeToHtmlNode = (childNode: ChildNode): HTMLNode | null => {
   if (
     isChildNodeComment(childNode) ||
@@ -37,6 +41,9 @@ const mapChildNodeToHtmlNode = (childNode: ChildNode): HTMLNode | null => {
     return null;
   }
   if (isChildNodeTextNode(childNode)) {
+    if (isTextNodePureWhiteSpace(childNode)) {
+      return null;
+    }
     return {
       type: "text",
       value: childNode.value,

src/test/index.test.ts

@@ -72,18 +72,18 @@ describe("Parse HTML string to Contentful Document", () => {
 
     const matchNode = helpers.createBlock(
       BLOCKS.PARAGRAPH,
-      helpers.createText(matchText),
+      helpers.createText(matchText)
     );
 
     expect(htmlNodes).toMatchObject(
-      createDocumentNode([matchNode as TopLevelBlock]),
+      createDocumentNode([matchNode as TopLevelBlock])
     );
   });
 
   it("Handles a complex convert option from 'span' with bold class to 'paragraph' and 'bold' mark", () => {
     const styledSpanToMarkedParagraphConverter: TagConverter<Block> = (
       node,
-      next,
+      next
     ) => {
       const isBold = node.attrs.class === "bold";
       const marks = isBold ? ({ type: "bold" } satisfies Mark) : undefined;
@@ -100,13 +100,13 @@ describe("Parse HTML string to Contentful Document", () => {
         convertTag: {
           span: styledSpanToMarkedParagraphConverter,
         },
-      },
+      }
     );
 
     const matchNode = createDocumentNode([
       helpers.createBlock(
         BLOCKS.PARAGRAPH,
-        helpers.createText(matchText, { type: "bold" }),
+        helpers.createText(matchText, { type: "bold" })
       ),
     ] as TopLevelBlock[]);
 
@@ -115,7 +115,7 @@ describe("Parse HTML string to Contentful Document", () => {
 
   it("converts an invalid top level text node to a paragraph node", () => {
     expect(
-      htmlStringToDocument("<div>Text under top level div</div>"),
+      htmlStringToDocument("<div>Text under top level div</div>")
     ).toMatchObject({
       content: [
         {
@@ -141,8 +141,8 @@ describe("Parse HTML string to Contentful Document", () => {
       htmlStringToDocument(
         "Some unwrapped text prefixing a p tag." +
           "<p>Paragraph content <span>I am a text node</span></p>" +
-          "Some unwrapped text suffixing a p tag",
-      ),
+          "Some unwrapped text suffixing a p tag"
+      )
     ).toMatchObject({
       content: [
         {
@@ -192,4 +192,47 @@ describe("Parse HTML string to Contentful Document", () => {
       nodeType: "document",
     });
   });
+
+  it("Handles text nodes with only whitespace by ingoring them", () => {
+    const html = `<h2>Heading on the first line</h2>\n\n<p>Text on the third line.</p>`;
+
+    const htmlNodes = htmlStringToDocument(html, {
+      ignoreWhiteSpace: true,
+    });
+
+    const matchNode = createDocumentNode([
+      helpers.createBlock(
+        BLOCKS.HEADING_2,
+        helpers.createText("Heading on the first line")
+      ),
+      helpers.createBlock(
+        BLOCKS.PARAGRAPH,
+        helpers.createText("Text on the third line.")
+      ),
+    ] as TopLevelBlock[]);
+
+    expect(htmlNodes).toMatchObject(matchNode);
+  });
+
+  it("Handles text nodes with only whitespace by including them", () => {
+    const html = `<h2>Heading on the first line</h2>\n\n<p>Text on the third line.</p>`;
+
+    const htmlNodes = htmlStringToDocument(html, {
+      ignoreWhiteSpace: false,
+    });
+
+    const matchNode = createDocumentNode([
+      helpers.createBlock(
+        BLOCKS.HEADING_2,
+        helpers.createText("Heading on the first line")
+      ),
+      helpers.createText("\n\n"),
+      helpers.createBlock(
+        BLOCKS.PARAGRAPH,
+        helpers.createText("Text on the third line.")
+      ),
+    ] as TopLevelBlock[]);
+
+    expect(htmlNodes).toMatchObject(matchNode);
+  });
 });

src/types.ts

@@ -58,6 +58,9 @@ export type ConvertTagOptions = Record<HTMLTagName | string, TagConverter>;
 export interface OptionsWithDefaults {
   convertTag: ConvertTagOptions;
   convertText: TextConverter;
+  wrapTopLevelTextNodesInParagraphs: boolean;
+  ignoreWhiteSpace: boolean;
+  isWhiteSpace: (content: string) => boolean;
 }
 
 export type Options = Partial<OptionsWithDefaults>;

src/utils.ts

@@ -14,6 +14,8 @@ import { MARK_TYPES, INLINE_TYPES, BLOCK_TYPES } from "./constants";
 export const isNotNull = <T>(value: T): value is Exclude<T, null> =>
   value !== null;
 
+export const isWhiteSpace = (content: string): boolean => /^\s*$/.test(content);
+
 export const getAsList = <T>(value: T | T[]): T[] => {
   if (Array.isArray(value)) {
     return value;