diff --git a/src/grammar/grammar.ts b/src/grammar/grammar.ts index 0f74bdba..3315f40f 100644 --- a/src/grammar/grammar.ts +++ b/src/grammar/grammar.ts @@ -409,6 +409,13 @@ export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): IRawGrammar { grammar = clone(grammar); + // Raw grammars may contain metadata named id. Only compiled rules may use it. + clearRuleIds(grammar); + if (grammar.injections) { + for (const selector in grammar.injections) { + clearRuleIds(grammar.injections[selector]); + } + } grammar.repository = grammar.repository || {}; grammar.repository.$self = { @@ -420,6 +427,25 @@ function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): I return grammar; } +function clearRuleIds(rule: IRawRule): void { + delete rule.id; + if (rule.patterns) { + for (const pattern of rule.patterns) { + clearRuleIds(pattern); + } + } + for (const key of ['repository', 'captures', 'beginCaptures', 'endCaptures', 'whileCaptures'] as const) { + const rules = rule[key]; + if (rules) { + for (const name in rules) { + if (name !== '$vscodeTextmateLocation' && rules[name]) { + clearRuleIds(rules[name]); + } + } + } + } +} + export class AttributedScopeStack { static fromExtension(namesScopeList: AttributedScopeStack | null, contentNameScopesList: AttributedScopeStackFrame[]): AttributedScopeStack | null { let current = namesScopeList; diff --git a/src/tests/grammar.test.ts b/src/tests/grammar.test.ts index 30ad42b2..ba6aab6b 100644 --- a/src/tests/grammar.test.ts +++ b/src/tests/grammar.test.ts @@ -5,7 +5,7 @@ import { test } from 'vitest'; import * as assert from 'assert'; import { EncodedTokenAttributes, OptionalStandardTokenType, StandardTokenType } from '../encodedTokenAttributes'; -import { Registry } from '../main'; +import { Registry, parseRawGrammar } from '../main'; import { FontStyle } from '../theme'; import { getOniguruma } from './onigLibs'; import { FontInfo } from '../grammar/grammar'; @@ -195,4 +195,79 @@ test('Fonts are correctly set 2', async function () { } finally { registry.dispose(); } -}); \ No newline at end of file +}); +for (const id of ['metadata', 1, 999]) { + for (const location of ['pattern', 'repository', 'capture', 'injection', 'external']) { + test(`Raw rule id metadata does not affect ${location} tokenization (${id})`, async () => { + const raw = { + scopeName: 'source.test', + patterns: [{ begin: '"', end: '"', name: 'string', patterns: [{ include: '#id' }] }, { include: '#captured' }, { include: 'source.external' }], + repository: { + id: { match: '\\\\.', name: 'escape' }, + captured: { match: '(x)', name: 'word', captures: { '1': { name: 'capture', patterns: [{ match: 'x', name: 'inner' }] } } } + }, + injections: { 'L:source.test': { match: 'z', name: 'injected' } } + }; + const external = { scopeName: 'source.external', patterns: [{ match: 'y', name: 'external' }], repository: {} }; + const target = location === 'pattern' ? raw.patterns[0] + : location === 'repository' ? raw.repository.id + : location === 'capture' ? raw.repository.captured.captures['1'] + : location === 'injection' ? raw.injections['L:source.test'] : external.patterns[0]; + Object.assign(target, { id }); + const input = JSON.stringify(raw); + const externalInput = JSON.stringify(external); + const grammarInput = parseRawGrammar(input, 'test.json'); + const externalGrammarInput = parseRawGrammar(externalInput, 'external.json'); + for (let pass = 0; pass < 2; pass++) { + const registry = new Registry({ + onigLib: getOniguruma(), + loadGrammar: async scope => scope === 'source.test' ? grammarInput : externalGrammarInput + }); + try { + const grammar = await registry.loadGrammar('source.test'); + assert.ok(grammar); + const result = grammar.tokenizeLine('"\\n" x y z', null); + assert.deepStrictEqual(result.tokens, [ + { startIndex: 0, endIndex: 1, scopes: ['source.test', 'string'] }, + { startIndex: 1, endIndex: 3, scopes: ['source.test', 'string', 'escape'] }, + { startIndex: 3, endIndex: 4, scopes: ['source.test', 'string'] }, + { startIndex: 4, endIndex: 5, scopes: ['source.test'] }, + { startIndex: 5, endIndex: 6, scopes: ['source.test', 'word', 'capture', 'inner'] }, + { startIndex: 6, endIndex: 7, scopes: ['source.test'] }, + { startIndex: 7, endIndex: 8, scopes: ['source.test', 'external'] }, + { startIndex: 8, endIndex: 9, scopes: ['source.test'] }, + { startIndex: 9, endIndex: 10, scopes: ['source.test', 'injected'] } + ]); + assert.strictEqual(JSON.stringify(grammarInput), input); + assert.strictEqual(JSON.stringify(externalGrammarInput), externalInput); + } finally { + registry.dispose(); + } + } + }); + } +} + +for (const capture of ['beginCaptures', 'endCaptures', 'whileCaptures']) { + test(`Raw rule id metadata does not affect ${capture}`, async () => { + const raw = { + scopeName: 'source.test', repository: {}, + patterns: [{ + begin: '(x)', + ...(capture === 'whileCaptures' ? { while: '(y)' } : { end: '(y)' }), + name: 'block', + [capture]: { '1': { id: 'metadata', patterns: [{ match: '[xy]', name: 'captured' }] } } + }] + }; + const registry = new Registry({ onigLib: getOniguruma(), loadGrammar: async () => parseRawGrammar(JSON.stringify(raw), 'test.json') }); + try { + const grammar = await registry.loadGrammar('source.test'); + assert.ok(grammar); + const first = grammar.tokenizeLine('x', null); + const result = capture === 'beginCaptures' ? first : grammar.tokenizeLine('y', first.ruleStack); + assert.deepStrictEqual(result.tokens[0], { startIndex: 0, endIndex: 1, scopes: ['source.test', 'block', 'captured'] }); + } finally { + registry.dispose(); + } + }); +}