diff --git a/src/nodes.ts b/src/nodes.ts index ec2501a..6b6c24f 100644 --- a/src/nodes.ts +++ b/src/nodes.ts @@ -1,5 +1,6 @@ import * as NHParser from 'node-html-parser'; import { CommentNode, NodeType } from 'node-html-parser'; +import { isCollapsibleWhitespace } from './utilities'; /* ****************************************************************************************************************** */ @@ -29,4 +30,9 @@ export const isTextNode = (node: HtmlNode): node is TextNode => node.nodeType == export const isCommentNode = (node: HtmlNode): node is CommentNode => node.nodeType === NodeType.COMMENT_NODE; export const isElementNode = (node: HtmlNode): node is ElementNode => node.nodeType === NodeType.ELEMENT_NODE; +/** + * Like node-html-parser's `isWhitespace`, but by the whitespace HTML collapses, which leaves out NBSP and ` ` + */ +export const isCollapsibleWhitespaceNode = (node: TextNode) => isCollapsibleWhitespace(node.rawText); + // endregion diff --git a/src/utilities.ts b/src/utilities.ts index 4442b5d..c8576f9 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -11,6 +11,15 @@ export const trimNewLines = (s: string) => s.replace(/^\n+|\n+$/g, ''); export const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`; export const isWhiteSpaceOnly = (s: string) => !/\S/.test(s); +/** + * Whitespace that HTML collapses: ASCII only (https://infra.spec.whatwg.org/#ascii-whitespace). + * Unlike `\s`, it leaves out NBSP (U+00A0, ` `), which exists to not be collapsed. + */ +const collapsibleWhitespace = '[ \\t\\n\\f\\r]'; +export const collapsibleWhitespaceRegex = new RegExp(`${collapsibleWhitespace}+`, 'g'); +const collapsibleWhitespaceOnlyRegex = new RegExp(`^${collapsibleWhitespace}*$`); +export const isCollapsibleWhitespace = (s: string) => collapsibleWhitespaceOnlyRegex.test(s); + /** * Split string, preserving specific newline used for each line */ @@ -43,6 +52,13 @@ export function splitSpecial(s: string) { return lines; } +/** + * Whitespace kept beside a surrounded tag: the character next to the content, unless the run holds NBSP, which + * HTML doesn't collapse, so the whole run is kept with only its collapsible whitespace collapsed + */ +const edgeWhitespace = (run: string, adjacent: string | undefined) => + run.includes('\u00A0') ? run.replace(collapsibleWhitespaceRegex, ' ') : (adjacent ?? ''); + /** * Surround tag content with delimiter (moving any leading/trailing space to outside the tag */ @@ -87,8 +103,8 @@ export function tagSurround(content: string, surroundStr: string) { if (endPos === undefined) endPos = text.length - 1; - const leadingSpace = startPos > 0 ? text[startPos - 1] : ''; - const trailingSpace = endPos < (text.length - 1) ? text[endPos + 1] : ''; + const leadingSpace = edgeWhitespace(text.slice(0, startPos), text[startPos - 1]); + const trailingSpace = edgeWhitespace(text.slice(endPos + 1), text[endPos + 1]); const slicedText = text.slice(startPos, endPos + 1) diff --git a/src/visitor.ts b/src/visitor.ts index 9384546..f86acc8 100644 --- a/src/visitor.ts +++ b/src/visitor.ts @@ -1,6 +1,8 @@ import { NodeHtmlMarkdown } from './main'; -import { ElementNode, HtmlNode, isElementNode, isTextNode } from './nodes'; -import { getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines } from './utilities'; +import { ElementNode, HtmlNode, isCollapsibleWhitespaceNode, isElementNode, isTextNode } from './nodes'; +import { + collapsibleWhitespaceRegex, getChildNodes, getTrailingWhitespaceInfo, perfStart, perfStop, trimNewLines +} from './utilities'; import { createTranslatorContext, isTranslatorConfig, PostProcessResult, TranslatorConfig, TranslatorConfigFactory, TranslatorConfigObject, TranslatorContext @@ -137,7 +139,7 @@ export class Visitor { */ private processText(text: string, metadata: NodeMetadata | undefined) { let res = text; - if (!metadata?.preserveWhitespace) res = res.replace(/\s+/g, ' '); + if (!metadata?.preserveWhitespace) res = res.replace(collapsibleWhitespaceRegex, ' '); if (metadata?.noEscape) return res; const { lineStartEscape, globalEscape, textReplace } = this.options; @@ -164,7 +166,7 @@ export class Visitor { (node).trimmedText ??= trimNewLines((node).wholeText); } - if (node.isWhitespace && !metadata?.preserveWhitespace) { + if (isCollapsibleWhitespaceNode(node) && !metadata?.preserveWhitespace) { return (!result.text.length || result.trailingNewlineStats.whitespace > 0) ? void 0 : this.appendResult(' '); } diff --git a/test/options.test.ts b/test/options.test.ts index 7174a4c..c17eb0a 100644 --- a/test/options.test.ts +++ b/test/options.test.ts @@ -302,13 +302,13 @@ text`); instance.options.useLinkReferenceDefinitions = false; let res = translate(html); expect(res).toBe( - `Hello: [a b**c**](${url}) a**b** [link2](${url}/other) [repeat link](${url}) <${url}> Goodbye!` + `Hello:\u00A0 [a b**c**](${url}) a**b** [link2](${url}/other) [repeat link](${url}) <${url}>\u00A0Goodbye!` ); instance.options.useLinkReferenceDefinitions = true; res = translate(html); expect(res).toBe( - `Hello: [a b**c**][1] a**b** [link2][2] [repeat link][1] <${url}> Goodbye! \n\n[1]: ${url}\n[2]: ${url}/other` + `Hello:\u00A0 [a b**c**][1] a**b** [link2][2] [repeat link][1] <${url}>\u00A0Goodbye! \n\n[1]: ${url}\n[2]: ${url}/other` ); instance.options.useLinkReferenceDefinitions = originalUseLinkReferenceDefinitions; @@ -327,12 +327,12 @@ text`); instance.options.useInlineLinks = false; let res = translate(html); - expect(res).toBe(`Hello: [${url}](${url}) a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`); + expect(res).toBe(`Hello:\u00A0 [${url}](${url}) \u00A0 a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!`); instance.options.useInlineLinks = true; res = translate(html); expect(res).toBe( - `Hello: <${url}> a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!` + `Hello:\u00A0 <${url}> \u00A0 a**b** [link2](${url}/other) [repeat link](${url}) Goodbye!` ); instance.options.useInlineLinks = originalUseInlineLinksDefinitions; diff --git a/test/special-cases.test.ts b/test/special-cases.test.ts index bf52f3b..be81060 100644 --- a/test/special-cases.test.ts +++ b/test/special-cases.test.ts @@ -47,8 +47,24 @@ describe(`Special Cases`, () => { test.each(textFormatTags)(`%s`, tag => { const delim = delims[tag]; - expect(translate(`

<${tag}>  Label:  Value

`)).toBe(` ${delim}Label:${delim} Value`); - expect(translate(`

<${tag}>  Label:  Value

`)).toBe(` ${delim}Label:${delim} Value`); + expect(translate(`

<${tag}>  Label:  Value

`)).toBe(` \u00A0${delim}Label:${delim}\u00A0 Value`); + expect(translate(`

<${tag}>  Label:  Value

`)).toBe(`\u00A0 ${delim}Label:${delim} \u00A0Value`); + }); + }); + + // See: https://github.com/crosstype/node-html-markdown/issues/87 + describe(`Keeps NBSP rather than collapsing it like ASCII whitespace`, () => { + test(`as an entity or as the character`, () => { + expect(translate(`

A   B

`)).toBe(`## A\u00A0\u00A0\u00A0B`); + expect(translate(`

A\u00A0\u00A0B

`)).toBe(`A\u00A0\u00A0B`); + }); + + test(`when it is a whole text node between tags`, () => { + expect(translate(`

a  b

`)).toBe(`**a**\u00A0\u00A0**b**`); + }); + + test(`while still collapsing the ASCII whitespace around it`, () => { + expect(translate(`

a \t\n \n b

`)).toBe(`a \u00A0 b`); }); });