From e8807fe7374d23a29c37e04e679c3e843ee4e4f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 20:31:59 +0000 Subject: [PATCH 1/4] fix: keep NBSP instead of collapsing it like ASCII whitespace (#87) Text collapsed with `\s`, which in JavaScript includes U+00A0, so ` ` runs came out as a single space. HTML only collapses ASCII whitespace, and NBSP exists to not be collapsed. - processText collapses `[ \t\n\f\r]+` instead of `\s+` - a text node counts as whitespace-only by the same set, rather than node-html-parser's `isWhitespace`, which includes NBSP and ` ` - tagSurround keeps an edge whitespace run whole when it holds NBSP, instead of only the one character next to the content Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018xDzwbcJFd3dMc24MCMT5W --- src/utilities.ts | 18 ++++++++++++++++-- src/visitor.ts | 9 ++++++--- test/options.test.ts | 8 ++++---- test/special-cases.test.ts | 20 ++++++++++++++++++-- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/utilities.ts b/src/utilities.ts index 4442b5d..00e6afe 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -11,6 +11,13 @@ export const trimNewLines = (s: string) => s.replace(/^\n+|\n+$/g, ''); export const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`; export const isWhiteSpaceOnly = (s: string) => !/\S/.test(s); +/** + * Whitespace that HTML collapses: ASCII only (https://infra.spec.whatwg.org/#ascii-whitespace). + * Unlike `\s`, it leaves out NBSP (U+00A0, ` `), which exists to not be collapsed. + */ +export const collapsibleWhitespaceRegex = /[ \t\n\f\r]+/g; +export const isCollapsibleWhitespace = (s: string) => /^[ \t\n\f\r]*$/.test(s); + /** * Split string, preserving specific newline used for each line */ @@ -43,6 +50,13 @@ export function splitSpecial(s: string) { return lines; } +/** + * Whitespace kept beside a surrounded tag: the character next to the content, unless the run holds NBSP, which + * HTML doesn't collapse, so the whole run is kept with only its collapsible whitespace collapsed + */ +const edgeWhitespace = (run: string, adjacent: string | undefined) => + run.includes('\u00A0') ? run.replace(collapsibleWhitespaceRegex, ' ') : (adjacent ?? ''); + /** * Surround tag content with delimiter (moving any leading/trailing space to outside the tag */ @@ -87,8 +101,8 @@ export function tagSurround(content: string, surroundStr: string) { if (endPos === undefined) endPos = text.length - 1; - const leadingSpace = startPos > 0 ? text[startPos - 1] : ''; - const trailingSpace = endPos < (text.length - 1) ? text[endPos + 1] : ''; + const leadingSpace = edgeWhitespace(text.slice(0, startPos), text[startPos - 1]); + const trailingSpace = edgeWhitespace(text.slice(endPos + 1), text[endPos + 1]); const slicedText = text.slice(startPos, endPos + 1) diff --git a/src/visitor.ts b/src/visitor.ts index 9384546..e1318ca 100644 --- a/src/visitor.ts +++ b/src/visitor.ts @@ -1,6 +1,9 @@ import { NodeHtmlMarkdown } from './main'; import { ElementNode, HtmlNode, isElementNode, isTextNode } from './nodes'; -import { getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines } from './utilities'; +import { + collapsibleWhitespaceRegex, getChildNodes, getTrailingWhitespaceInfo, isCollapsibleWhitespace, perfStart, perfStop, + trimNewLines +} from './utilities'; import { createTranslatorContext, isTranslatorConfig, PostProcessResult, TranslatorConfig, TranslatorConfigFactory, TranslatorConfigObject, TranslatorContext @@ -137,7 +140,7 @@ export class Visitor { */ private processText(text: string, metadata: NodeMetadata | undefined) { let res = text; - if (!metadata?.preserveWhitespace) res = res.replace(/\s+/g, ' '); + if (!metadata?.preserveWhitespace) res = res.replace(collapsibleWhitespaceRegex, ' '); if (metadata?.noEscape) return res; const { lineStartEscape, globalEscape, textReplace } = this.options; @@ -164,7 +167,7 @@ export class Visitor { (node).trimmedText ??= trimNewLines((node).wholeText); } - if (node.isWhitespace && !metadata?.preserveWhitespace) { + if (isCollapsibleWhitespace(node.rawText) && !metadata?.preserveWhitespace) { return (!result.text.length || result.trailingNewlineStats.whitespace > 0) ? void 0 : this.appendResult(' '); } diff --git a/test/options.test.ts b/test/options.test.ts index 7174a4c..c17eb0a 100644 --- a/test/options.test.ts +++ b/test/options.test.ts @@ -302,13 +302,13 @@ text`); instance.options.useLinkReferenceDefinitions = false; let res = translate(html); expect(res).toBe( - `Hello: [a b**c**](${url}) a**b** [link2](${url}/other) [repeat link](${url}) <${url}> Goodbye!` + `Hello:\u00A0 [a b**c**](${url}) a**b** [link2](${url}/other) [repeat link](${url}) <${url}>\u00A0Goodbye!` ); instance.options.useLinkReferenceDefinitions = true; res = translate(html); expect(res).toBe( - `Hello: [a b**c**][1] a**b** [link2][2] [repeat link][1] <${url}> Goodbye! \n\n[1]: ${url}\n[2]: ${url}/other` + `Hello:\u00A0 [a b**c**][1] a**b** [link2][2] [repeat link][1] <${url}>\u00A0Goodbye! \n\n[1]: ${url}\n[2]: ${url}/other` ); instance.options.useLinkReferenceDefinitions = originalUseLinkReferenceDefinitions; @@ -327,12 +327,12 @@ text`); instance.options.useInlineLinks = false; let res = translate(html); - expect(res).toBe(`Hello: [${url}](${url}) a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`); + expect(res).toBe(`Hello:\u00A0 [${url}](${url}) \u00A0 a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`); instance.options.useInlineLinks = true; res = translate(html); expect(res).toBe( - `Hello: <${url}> a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!` + `Hello:\u00A0 <${url}> \u00A0 a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!` ); instance.options.useInlineLinks = originalUseInlineLinksDefinitions; diff --git a/test/special-cases.test.ts b/test/special-cases.test.ts index bf52f3b..be81060 100644 --- a/test/special-cases.test.ts +++ b/test/special-cases.test.ts @@ -47,8 +47,24 @@ describe(`Special Cases`, () => { test.each(textFormatTags)(`%s`, tag => { const delim = delims[tag]; - expect(translate(`

<${tag}>  Label:  Value

`)).toBe(` ${delim}Label:${delim} Value`); - expect(translate(`

<${tag}>  Label:  Value

`)).toBe(` ${delim}Label:${delim} Value`); + expect(translate(`

<${tag}>  Label:  Value

`)).toBe(` \u00A0${delim}Label:${delim}\u00A0 Value`); + expect(translate(`

<${tag}>  Label:  Value

`)).toBe(`\u00A0 ${delim}Label:${delim} \u00A0Value`); + }); + }); + + // See: https://github.com/crosstype/node-html-markdown/issues/87 + describe(`Keeps NBSP rather than collapsing it like ASCII whitespace`, () => { + test(`as an entity or as the character`, () => { + expect(translate(`

A   B

`)).toBe(`## A\u00A0\u00A0\u00A0B`); + expect(translate(`

A\u00A0\u00A0B

`)).toBe(`A\u00A0\u00A0B`); + }); + + test(`when it is a whole text node between tags`, () => { + expect(translate(`

a  b

`)).toBe(`**a**\u00A0\u00A0**b**`); + }); + + test(`while still collapsing the ASCII whitespace around it`, () => { + expect(translate(`

a \t\n \n b

`)).toBe(`a \u00A0 b`); }); }); From 390ab7106090f01d578177df2071f84ce007c141 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:03:23 +0000 Subject: [PATCH 2/4] refactor: share the collapsible whitespace class between its two regexes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WcmVi7QkUu1TeYUfXMN5n --- src/utilities.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utilities.ts b/src/utilities.ts index 00e6afe..c8576f9 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -15,8 +15,10 @@ export const isWhiteSpaceOnly = (s: string) => !/\S/.test(s); * Whitespace that HTML collapses: ASCII only (https://infra.spec.whatwg.org/#ascii-whitespace). * Unlike `\s`, it leaves out NBSP (U+00A0, ` `), which exists to not be collapsed. */ -export const collapsibleWhitespaceRegex = /[ \t\n\f\r]+/g; -export const isCollapsibleWhitespace = (s: string) => /^[ \t\n\f\r]*$/.test(s); +const collapsibleWhitespace = '[ \\t\\n\\f\\r]'; +export const collapsibleWhitespaceRegex = new RegExp(`${collapsibleWhitespace}+`, 'g'); +const collapsibleWhitespaceOnlyRegex = new RegExp(`^${collapsibleWhitespace}*$`); +export const isCollapsibleWhitespace = (s: string) => collapsibleWhitespaceOnlyRegex.test(s); /** * Split string, preserving specific newline used for each line From cbbdaeeb3285f2a1f3db590c1ca6b90f52269544 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:09:04 +0000 Subject: [PATCH 3/4] refactor: expose the whitespace-only check as TextNode.isCollapsibleWhitespace Mirrors node-html-parser's isWhitespace getter, which the visitor used before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WcmVi7QkUu1TeYUfXMN5n --- src/nodes.ts | 19 ++++++++++++++++++- src/visitor.ts | 5 ++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/nodes.ts b/src/nodes.ts index ec2501a..66bbd07 100644 --- a/src/nodes.ts +++ b/src/nodes.ts @@ -1,5 +1,6 @@ import * as NHParser from 'node-html-parser'; import { CommentNode, NodeType } from 'node-html-parser'; +import { isCollapsibleWhitespace } from './utilities'; /* ****************************************************************************************************************** */ @@ -16,7 +17,23 @@ type NodeBase = { preserve?: boolean } export type HtmlNode = (NHParser.Node | Node) & NodeBase export type ElementNode = (NHParser.HTMLElement | HTMLElement) & NodeBase -export type TextNode = (NHParser.TextNode) & NodeBase +export type TextNode = (NHParser.TextNode) & NodeBase & { readonly isCollapsibleWhitespace: boolean } + +// endregion + + +/* ****************************************************************************************************************** */ +// region: Node Extensions +/* ****************************************************************************************************************** */ + +/** + * Like node-html-parser's `isWhitespace`, but by the whitespace HTML collapses, which leaves out NBSP and ` ` + */ +if (!Object.prototype.hasOwnProperty.call(NHParser.TextNode.prototype, 'isCollapsibleWhitespace')) + Object.defineProperty(NHParser.TextNode.prototype, 'isCollapsibleWhitespace', { + get(this: NHParser.TextNode) { return isCollapsibleWhitespace(this.rawText); }, + configurable: true + }); // endregion diff --git a/src/visitor.ts b/src/visitor.ts index e1318ca..722e74b 100644 --- a/src/visitor.ts +++ b/src/visitor.ts @@ -1,8 +1,7 @@ import { NodeHtmlMarkdown } from './main'; import { ElementNode, HtmlNode, isElementNode, isTextNode } from './nodes'; import { - collapsibleWhitespaceRegex, getChildNodes, getTrailingWhitespaceInfo, isCollapsibleWhitespace, perfStart, perfStop, - trimNewLines + collapsibleWhitespaceRegex, getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines } from './utilities'; import { createTranslatorContext, isTranslatorConfig, PostProcessResult, TranslatorConfig, TranslatorConfigFactory, @@ -167,7 +166,7 @@ export class Visitor { (node).trimmedText ??= trimNewLines((node).wholeText); } - if (isCollapsibleWhitespace(node.rawText) && !metadata?.preserveWhitespace) { + if (node.isCollapsibleWhitespace && !metadata?.preserveWhitespace) { return (!result.text.length || result.trailingNewlineStats.whitespace > 0) ? void 0 : this.appendResult(' '); } From 23c23defaa4461a009f47e32d38cbcbce5b650be Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 21:10:59 +0000 Subject: [PATCH 4/4] refactor: use a node helper for the whitespace-only check instead of patching TextNode Patching node-html-parser's prototype changed it for every importer in the process. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016WcmVi7QkUu1TeYUfXMN5n --- src/nodes.ts | 23 ++++++----------------- src/visitor.ts | 4 ++-- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/src/nodes.ts b/src/nodes.ts index 66bbd07..6b6c24f 100644 --- a/src/nodes.ts +++ b/src/nodes.ts @@ -17,23 +17,7 @@ type NodeBase = { preserve?: boolean } export type HtmlNode = (NHParser.Node | Node) & NodeBase export type ElementNode = (NHParser.HTMLElement | HTMLElement) & NodeBase -export type TextNode = (NHParser.TextNode) & NodeBase & { readonly isCollapsibleWhitespace: boolean } - -// endregion - - -/* ****************************************************************************************************************** */ -// region: Node Extensions -/* ****************************************************************************************************************** */ - -/** - * Like node-html-parser's `isWhitespace`, but by the whitespace HTML collapses, which leaves out NBSP and ` ` - */ -if (!Object.prototype.hasOwnProperty.call(NHParser.TextNode.prototype, 'isCollapsibleWhitespace')) - Object.defineProperty(NHParser.TextNode.prototype, 'isCollapsibleWhitespace', { - get(this: NHParser.TextNode) { return isCollapsibleWhitespace(this.rawText); }, - configurable: true - }); +export type TextNode = (NHParser.TextNode) & NodeBase // endregion @@ -46,4 +30,9 @@ export const isTextNode = (node: HtmlNode): node is TextNode => node.nodeType == export const isCommentNode = (node: HtmlNode): node is CommentNode => node.nodeType === NodeType.COMMENT_NODE; export const isElementNode = (node: HtmlNode): node is ElementNode => node.nodeType === NodeType.ELEMENT_NODE; +/** + * Like node-html-parser's `isWhitespace`, but by the whitespace HTML collapses, which leaves out NBSP and ` ` + */ +export const isCollapsibleWhitespaceNode = (node: TextNode) => isCollapsibleWhitespace(node.rawText); + // endregion diff --git a/src/visitor.ts b/src/visitor.ts index 722e74b..f86acc8 100644 --- a/src/visitor.ts +++ b/src/visitor.ts @@ -1,5 +1,5 @@ import { NodeHtmlMarkdown } from './main'; -import { ElementNode, HtmlNode, isElementNode, isTextNode } from './nodes'; +import { ElementNode, HtmlNode, isCollapsibleWhitespaceNode, isElementNode, isTextNode } from './nodes'; import { collapsibleWhitespaceRegex, getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines } from './utilities'; @@ -166,7 +166,7 @@ export class Visitor { (node).trimmedText ??= trimNewLines((node).wholeText); } - if (node.isCollapsibleWhitespace && !metadata?.preserveWhitespace) { + if (isCollapsibleWhitespaceNode(node) && !metadata?.preserveWhitespace) { return (!result.text.length || result.trailingNewlineStats.whitespace > 0) ? void 0 : this.appendResult(' '); }