From 83d61b62c23e00220fb662dfe3cab64c44912a35 Mon Sep 17 00:00:00 2001 From: Ivan Todoroski Date: Tue, 25 Aug 2026 19:22:37 -0700 Subject: [PATCH 1/2] fix(parser): incorrect parsing of hexadecimal NCRs When you have a decimal NCR (Numeric Character Reference) in your text: before Ӓ after it gets parsed like this: TEXT => "before" SEA_WS => " " CharRef => "Ӓ" SEA_WS => " " TEXT => "after" However, if you have a hexadecimal NCR: before � after it gets parsed like this instead: TEXT => "before" SEA_WS => " " CharRef => "" TEXT => "23ABC;" SEA_WS => " " TEXT => "after" Note the extra `TEXT => "23ABC;"` token after the CharRef, it shouldn't be there. The NCR got split into two tokens, instead of emitted as one. The root cause was an apparent typo in the regex for CharRef tokens. It was missing a plus sign and a semicolon on the hexadecimal side of the alternation, which caused it to match only the first hexadecimal digit, instead of the entire NCR all the way to the semicolon. I added a hex NCR to the `valid/entity-char-ref` test case and verified that the full CharRef token appeared correctly in the output before updating the snapshot. I also checked that all subsequent offsets were shifted by 15, which corresponds to the 14 chars added plus the newline. Finally, I added a new `invalild/entity-char-ref` test case. The old regex would produce some CharRef tokens on this text, even though none of the NCRs in it were valid. With this fix, the text output is now "chardata" only, no CharRef tokens. --- packages/parser/lib/lexer.js | 2 +- .../invalid/entity-char-ref/input.xml | 12 + .../invalid/entity-char-ref/output.js | 206 ++++++++++++++++++ .../invalid/entity-char-ref/sample.spec.js | 7 + .../snapshots/valid/entity-char-ref/input.xml | 1 + .../snapshots/valid/entity-char-ref/output.js | 92 +++++--- 6 files changed, 287 insertions(+), 33 deletions(-) create mode 100644 packages/parser/test/snapshots/invalid/entity-char-ref/input.xml create mode 100644 packages/parser/test/snapshots/invalid/entity-char-ref/output.js create mode 100644 packages/parser/test/snapshots/invalid/entity-char-ref/sample.spec.js diff --git a/packages/parser/lib/lexer.js b/packages/parser/lib/lexer.js index e7e0686e..d6c18170 100644 --- a/packages/parser/lib/lexer.js +++ b/packages/parser/lib/lexer.js @@ -75,7 +75,7 @@ const EntityRef = createToken({ const CharRef = createToken({ name: "CharRef", - pattern: /&#\d+;|&#x[a-fA-F0-9]/, + pattern: /&#\d+;|&#x[a-fA-F0-9]+;/, }); const SEA_WS = createToken({ diff --git a/packages/parser/test/snapshots/invalid/entity-char-ref/input.xml b/packages/parser/test/snapshots/invalid/entity-char-ref/input.xml new file mode 100644 index 00000000..47306477 --- /dev/null +++ b/packages/parser/test/snapshots/invalid/entity-char-ref/input.xml @@ -0,0 +1,12 @@ + + &foo + &foo ; + + Ӓ + Ӓ ; + Y3; + + � + � ; + ģYBC; + diff --git a/packages/parser/test/snapshots/invalid/entity-char-ref/output.js b/packages/parser/test/snapshots/invalid/entity-char-ref/output.js new file mode 100644 index 00000000..36343b61 --- /dev/null +++ b/packages/parser/test/snapshots/invalid/entity-char-ref/output.js @@ -0,0 +1,206 @@ +module.exports = { + cst: { + name: "document", + children: { + element: [ + { + name: "element", + children: { + OPEN: [ + { image: "<", startOffset: 0, endOffset: 0, tokenType: "OPEN" }, + ], + Name: [ + { + image: "note", + startOffset: 1, + endOffset: 4, + tokenType: "Name", + }, + ], + START_CLOSE: [ + { image: ">", startOffset: 5, endOffset: 5, tokenType: "CLOSE" }, + ], + content: [ + { + name: "content", + children: { + chardata: [ + { + name: "chardata", + children: { + SEA_WS: [ + { + image: "\n ", + startOffset: 6, + endOffset: 8, + tokenType: "SEA_WS", + }, + ], + }, + location: { startOffset: 6, endOffset: 8 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "foo\n ", + startOffset: 10, + endOffset: 15, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 10, endOffset: 15 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "foo ;\n\n ", + startOffset: 17, + endOffset: 25, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 17, endOffset: 25 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "#1234\n ", + startOffset: 27, + endOffset: 34, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 27, endOffset: 34 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "#1234 ;\n ", + startOffset: 36, + endOffset: 45, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 36, endOffset: 45 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "#12Y3;\n\n ", + startOffset: 47, + endOffset: 56, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 47, endOffset: 56 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "#x123ABC\n ", + startOffset: 58, + endOffset: 68, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 58, endOffset: 68 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "#x123ABC ;\n ", + startOffset: 70, + endOffset: 82, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 70, endOffset: 82 }, + }, + { + name: "chardata", + children: { + TEXT: [ + { + image: "#x123YBC;\n", + startOffset: 84, + endOffset: 93, + tokenType: "TEXT", + }, + ], + }, + location: { startOffset: 84, endOffset: 93 }, + }, + ], + }, + location: { startOffset: 6, endOffset: 93 }, + }, + ], + SLASH_OPEN: [ + { + image: "", + startOffset: 100, + endOffset: 100, + tokenType: "CLOSE", + }, + ], + }, + location: { startOffset: 0, endOffset: 100 }, + }, + ], + misc: [ + { + name: "misc", + children: { + SEA_WS: [ + { + image: "\n", + startOffset: 101, + endOffset: 101, + tokenType: "SEA_WS", + }, + ], + }, + location: { startOffset: 101, endOffset: 101 }, + }, + ], + }, + location: { startOffset: 0, endOffset: 101 }, + }, +}; diff --git a/packages/parser/test/snapshots/invalid/entity-char-ref/sample.spec.js b/packages/parser/test/snapshots/invalid/entity-char-ref/sample.spec.js new file mode 100644 index 00000000..7e6e9a0b --- /dev/null +++ b/packages/parser/test/snapshots/invalid/entity-char-ref/sample.spec.js @@ -0,0 +1,7 @@ +const { + executeInValidSampleTest, + testNameFromDir, +} = require("../../../sample-test"); +describe(`${testNameFromDir(__dirname)}`, () => { + executeInValidSampleTest(__dirname); +}); diff --git a/packages/parser/test/snapshots/valid/entity-char-ref/input.xml b/packages/parser/test/snapshots/valid/entity-char-ref/input.xml index 816e0202..c2666418 100644 --- a/packages/parser/test/snapshots/valid/entity-char-ref/input.xml +++ b/packages/parser/test/snapshots/valid/entity-char-ref/input.xml @@ -1,5 +1,6 @@ &to; Ӓ + � Tim diff --git a/packages/parser/test/snapshots/valid/entity-char-ref/output.js b/packages/parser/test/snapshots/valid/entity-char-ref/output.js index de9ebca3..615f2c72 100644 --- a/packages/parser/test/snapshots/valid/entity-char-ref/output.js +++ b/packages/parser/test/snapshots/valid/entity-char-ref/output.js @@ -67,19 +67,33 @@ module.exports = { }, location: { startOffset: 27, endOffset: 31 }, }, + { + name: "chardata", + children: { + SEA_WS: [ + { + image: "\n ", + startOffset: 42, + endOffset: 46, + tokenType: "SEA_WS", + }, + ], + }, + location: { startOffset: 42, endOffset: 46 }, + }, { name: "chardata", children: { SEA_WS: [ { image: "\n", - startOffset: 48, - endOffset: 48, + startOffset: 63, + endOffset: 63, tokenType: "SEA_WS", }, ], }, - location: { startOffset: 48, endOffset: 48 }, + location: { startOffset: 63, endOffset: 63 }, }, ], reference: [ @@ -111,6 +125,20 @@ module.exports = { }, location: { startOffset: 20, endOffset: 26 }, }, + { + name: "reference", + children: { + CharRef: [ + { + image: "�", + startOffset: 32, + endOffset: 41, + tokenType: "CharRef", + }, + ], + }, + location: { startOffset: 32, endOffset: 41 }, + }, ], element: [ { @@ -119,24 +147,24 @@ module.exports = { OPEN: [ { image: "<", - startOffset: 32, - endOffset: 32, + startOffset: 47, + endOffset: 47, tokenType: "OPEN", }, ], Name: [ { image: "from", - startOffset: 33, - endOffset: 36, + startOffset: 48, + endOffset: 51, tokenType: "Name", }, ], START_CLOSE: [ { image: ">", - startOffset: 37, - endOffset: 37, + startOffset: 52, + endOffset: 52, tokenType: "CLOSE", }, ], @@ -151,77 +179,77 @@ module.exports = { TEXT: [ { image: "Tim", - startOffset: 38, - endOffset: 40, + startOffset: 53, + endOffset: 55, tokenType: "TEXT", }, ], }, - location: { startOffset: 38, endOffset: 40 }, + location: { startOffset: 53, endOffset: 55 }, }, ], }, - location: { startOffset: 38, endOffset: 40 }, + location: { startOffset: 53, endOffset: 55 }, }, ], SLASH_OPEN: [ { image: "", - startOffset: 47, - endOffset: 47, + startOffset: 62, + endOffset: 62, tokenType: "CLOSE", }, ], }, - location: { startOffset: 32, endOffset: 47 }, + location: { startOffset: 47, endOffset: 62 }, }, ], }, - location: { startOffset: 6, endOffset: 48 }, + location: { startOffset: 6, endOffset: 63 }, }, ], SLASH_OPEN: [ { image: "", - startOffset: 55, - endOffset: 55, + startOffset: 70, + endOffset: 70, tokenType: "CLOSE", }, ], }, - location: { startOffset: 0, endOffset: 55 }, + location: { startOffset: 0, endOffset: 70 }, }, ], misc: [ @@ -231,16 +259,16 @@ module.exports = { SEA_WS: [ { image: "\n", - startOffset: 56, - endOffset: 56, + startOffset: 71, + endOffset: 71, tokenType: "SEA_WS", }, ], }, - location: { startOffset: 56, endOffset: 56 }, + location: { startOffset: 71, endOffset: 71 }, }, ], }, - location: { startOffset: 0, endOffset: 56 }, + location: { startOffset: 0, endOffset: 71 }, }, }; From 6ba36cc49998255a2da21673ee9bdd039ba38426 Mon Sep 17 00:00:00 2001 From: Ivan Todoroski Date: Tue, 25 Aug 2026 19:32:46 -0700 Subject: [PATCH 2/2] chore(parser): increase Mocha diff size from 8k (default) to 64k When editing test fixtures and looking at the resulting test output, the diffs might get large due to a bunch of irrelevant offset changes. By the time you get to the interesting part of the diff, it gets cut off due to the 8k limit. --- packages/parser/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/parser/package.json b/packages/parser/package.json index 9a27b583..c8f2a81e 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -25,7 +25,7 @@ "scripts": { "ci": "npm-run-all clean type-check coverage:*", "clean": "rimraf ./coverage ./nyc_output", - "test": "mocha \"./test/**/*spec.js\"", + "test": "mocha -reporter-option maxDiffSize=65536 \"./test/**/*spec.js\"", "coverage:run": "nyc mocha \"./test/**/*spec.js\"", "coverage:check": "nyc check-coverage --lines 100 --branches 100 --statements 100 --functions 100", "snapshots:update": "node ./scripts/update-snapshots.js",