fix: keep NBSP instead of collapsing it like ASCII whitespace (#87)

src/nodes.ts

@@ -1,5 +1,6 @@
 import * as NHParser from 'node-html-parser';
 import { CommentNode, NodeType } from 'node-html-parser';
+import { isCollapsibleWhitespace } from './utilities';
 
 
 /* ****************************************************************************************************************** */
@@ -29,4 +30,9 @@ export const isTextNode = (node: HtmlNode): node is TextNode => node.nodeType ==
 export const isCommentNode = (node: HtmlNode): node is CommentNode => node.nodeType === NodeType.COMMENT_NODE;
 export const isElementNode = (node: HtmlNode): node is ElementNode => node.nodeType === NodeType.ELEMENT_NODE;
 
+/**
+ * Like node-html-parser's `isWhitespace`, but by the whitespace HTML collapses, which leaves out NBSP and ` `
+ */
+export const isCollapsibleWhitespaceNode = (node: TextNode) => isCollapsibleWhitespace(node.rawText);
+
 // endregion

src/utilities.ts

@@ -11,6 +11,15 @@ export const trimNewLines = (s: string) => s.replace(/^\n+|\n+$/g, '');
 export const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`;
 export const isWhiteSpaceOnly = (s: string) => !/\S/.test(s);
 
+/**
+ * Whitespace that HTML collapses: ASCII only (https://infra.spec.whatwg.org/#ascii-whitespace).
+ * Unlike `\s`, it leaves out NBSP (U+00A0, ` `), which exists to not be collapsed.
+ */
+const collapsibleWhitespace = '[ \\t\\n\\f\\r]';
+export const collapsibleWhitespaceRegex = new RegExp(`${collapsibleWhitespace}+`, 'g');
+const collapsibleWhitespaceOnlyRegex = new RegExp(`^${collapsibleWhitespace}*$`);
+export const isCollapsibleWhitespace = (s: string) => collapsibleWhitespaceOnlyRegex.test(s);
+
 /**
  * Split string, preserving specific newline used for each line
  */
@@ -43,6 +52,13 @@ export function splitSpecial(s: string) {
   return lines;
 }
 
+/**
+ * Whitespace kept beside a surrounded tag: the character next to the content, unless the run holds NBSP, which
+ * HTML doesn't collapse, so the whole run is kept with only its collapsible whitespace collapsed
+ */
+const edgeWhitespace = (run: string, adjacent: string | undefined) =>
+  run.includes('\u00A0') ? run.replace(collapsibleWhitespaceRegex, ' ') : (adjacent ?? '');
+
 /**
  * Surround tag content with delimiter (moving any leading/trailing space to outside the tag
  */
@@ -87,8 +103,8 @@ export function tagSurround(content: string, surroundStr: string) {
 
     if (endPos === undefined) endPos = text.length - 1;
 
-    const leadingSpace = startPos > 0 ? text[startPos - 1] : '';
-    const trailingSpace = endPos < (text.length - 1) ? text[endPos + 1] : '';
+    const leadingSpace = edgeWhitespace(text.slice(0, startPos), text[startPos - 1]);
+    const trailingSpace = edgeWhitespace(text.slice(endPos + 1), text[endPos + 1]);
 
     const slicedText = text.slice(startPos, endPos + 1)
 

src/visitor.ts

@@ -1,6 +1,8 @@
 import { NodeHtmlMarkdown } from './main';
-import { ElementNode, HtmlNode, isElementNode, isTextNode } from './nodes';
-import { getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines } from './utilities';
+import { ElementNode, HtmlNode, isCollapsibleWhitespaceNode, isElementNode, isTextNode } from './nodes';
+import {
+  collapsibleWhitespaceRegex, getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines
+} from './utilities';
 import {
   createTranslatorContext, isTranslatorConfig, PostProcessResult, TranslatorConfig, TranslatorConfigFactory,
   TranslatorConfigObject, TranslatorContext
@@ -137,7 +139,7 @@ export class Visitor {
    */
   private processText(text: string, metadata: NodeMetadata | undefined) {
     let res = text;
-    if (!metadata?.preserveWhitespace) res = res.replace(/\s+/g, ' ');
+    if (!metadata?.preserveWhitespace) res = res.replace(collapsibleWhitespaceRegex, ' ');
     if (metadata?.noEscape) return res;
 
     const { lineStartEscape, globalEscape, textReplace } = this.options;
@@ -164,7 +166,7 @@ export class Visitor {
         (<any>node).trimmedText ??= trimNewLines((<any>node).wholeText);
       }
 
-      if (node.isWhitespace && !metadata?.preserveWhitespace) {
+      if (isCollapsibleWhitespaceNode(node) && !metadata?.preserveWhitespace) {
         return (!result.text.length || result.trailingNewlineStats.whitespace > 0) ? void 0 : this.appendResult(' ');
       }
 

test/options.test.ts

@@ -302,13 +302,13 @@ text`);
     instance.options.useLinkReferenceDefinitions = false;
     let res = translate(html);
     expect(res).toBe(
-      `Hello: [a b**c**](${url}) a**b** [link2](${url}/other) [repeat link](${url}) <${url}> Goodbye!`
+      `Hello:\u00A0 [a b**c**](${url}) a**b** [link2](${url}/other) [repeat link](${url}) <${url}>\u00A0Goodbye!`
     );
 
     instance.options.useLinkReferenceDefinitions = true;
     res = translate(html);
     expect(res).toBe(
-      `Hello: [a b**c**][1] a**b** [link2][2] [repeat link][1] <${url}> Goodbye! \n\n[1]: ${url}\n[2]: ${url}/other`
+      `Hello:\u00A0 [a b**c**][1] a**b** [link2][2] [repeat link][1] <${url}>\u00A0Goodbye! \n\n[1]: ${url}\n[2]: ${url}/other`
     );
 
     instance.options.useLinkReferenceDefinitions = originalUseLinkReferenceDefinitions;
@@ -327,12 +327,12 @@ text`);
 
     instance.options.useInlineLinks = false;
     let res = translate(html);
-    expect(res).toBe(`Hello: [${url}](${url}) a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`);
+    expect(res).toBe(`Hello:\u00A0 [${url}](${url}) \u00A0 a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`);
 
     instance.options.useInlineLinks = true;
     res = translate(html);
     expect(res).toBe(
-      `Hello: <${url}> a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`
+      `Hello:\u00A0 <${url}> \u00A0 a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`
     );
 
     instance.options.useInlineLinks = originalUseInlineLinksDefinitions;

test/special-cases.test.ts

@@ -47,8 +47,24 @@ describe(`Special Cases`, () => {
     test.each(textFormatTags)(`%s`, tag => {
       const delim = delims[tag];
 
-      expect(translate(`<p><${tag}> &nbsp;Label:&nbsp; </${tag}>Value</p>`)).toBe(` ${delim}Label:${delim} Value`);
-      expect(translate(`<p><${tag}>&nbsp; Label: &nbsp;</${tag}>Value</p>`)).toBe(` ${delim}Label:${delim} Value`);
+      expect(translate(`<p><${tag}> &nbsp;Label:&nbsp; </${tag}>Value</p>`)).toBe(` \u00A0${delim}Label:${delim}\u00A0 Value`);
+      expect(translate(`<p><${tag}>&nbsp; Label: &nbsp;</${tag}>Value</p>`)).toBe(`\u00A0 ${delim}Label:${delim} \u00A0Value`);
+    });
+  });
+
+  // See: https://github.com/crosstype/node-html-markdown/issues/87
+  describe(`Keeps NBSP rather than collapsing it like ASCII whitespace`, () => {
+    test(`as an entity or as the character`, () => {
+      expect(translate(`<h2>A&nbsp;&nbsp;&nbsp;B</h2>`)).toBe(`## A\u00A0\u00A0\u00A0B`);
+      expect(translate(`<p>A\u00A0\u00A0B</p>`)).toBe(`A\u00A0\u00A0B`);
+    });
+
+    test(`when it is a whole text node between tags`, () => {
+      expect(translate(`<p><b>a</b>&nbsp;&nbsp;<b>b</b></p>`)).toBe(`**a**\u00A0\u00A0**b**`);
+    });
+
+    test(`while still collapsing the ASCII whitespace around it`, () => {
+      expect(translate(`<p>a \t\n&nbsp;\n b</p>`)).toBe(`a \u00A0 b`);
     });
   });