Ignore raw grammar metadata when assigning rule IDs

src/grammar/grammar.ts

@@ -409,6 +409,13 @@ export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib {
 
 function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): IRawGrammar {
 	grammar = clone(grammar);
+	// Raw grammars may contain metadata named id. Only compiled rules may use it.
+	clearRuleIds(grammar);
+	if (grammar.injections) {
+		for (const selector in grammar.injections) {
+			clearRuleIds(grammar.injections[selector]);
+		}
+	}
 
 	grammar.repository = grammar.repository || <any>{};
 	grammar.repository.$self = {
@@ -420,6 +427,25 @@ function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): I
 	return grammar;
 }
 
+function clearRuleIds(rule: IRawRule): void {
+	delete rule.id;
+	if (rule.patterns) {
+		for (const pattern of rule.patterns) {
+			clearRuleIds(pattern);
+		}
+	}
+	for (const key of ['repository', 'captures', 'beginCaptures', 'endCaptures', 'whileCaptures'] as const) {
+		const rules = rule[key];
+		if (rules) {
+			for (const name in rules) {
+				if (name !== '$vscodeTextmateLocation' && rules[name]) {
+					clearRuleIds(rules[name]);
+				}
+			}
+		}
+	}
+}
+
 export class AttributedScopeStack {
 	static fromExtension(namesScopeList: AttributedScopeStack | null, contentNameScopesList: AttributedScopeStackFrame[]): AttributedScopeStack | null {
 		let current = namesScopeList;

src/tests/grammar.test.ts

@@ -5,7 +5,7 @@
 import { test } from 'vitest';
 import * as assert from 'assert';
 import { EncodedTokenAttributes, OptionalStandardTokenType, StandardTokenType } from '../encodedTokenAttributes';
-import { Registry } from '../main';
+import { Registry, parseRawGrammar } from '../main';
 import { FontStyle } from '../theme';
 import { getOniguruma } from './onigLibs';
 import { FontInfo } from '../grammar/grammar';
@@ -195,4 +195,79 @@ test('Fonts are correctly set 2', async function () {
 	} finally {
 		registry.dispose();
 	}
-});
+});
+for (const id of ['metadata', 1, 999]) {
+	for (const location of ['pattern', 'repository', 'capture', 'injection', 'external']) {
+		test(`Raw rule id metadata does not affect ${location} tokenization (${id})`, async () => {
+			const raw = {
+				scopeName: 'source.test',
+				patterns: [{ begin: '"', end: '"', name: 'string', patterns: [{ include: '#id' }] }, { include: '#captured' }, { include: 'source.external' }],
+				repository: {
+					id: { match: '\\\\.', name: 'escape' },
+					captured: { match: '(x)', name: 'word', captures: { '1': { name: 'capture', patterns: [{ match: 'x', name: 'inner' }] } } }
+				},
+				injections: { 'L:source.test': { match: 'z', name: 'injected' } }
+			};
+			const external = { scopeName: 'source.external', patterns: [{ match: 'y', name: 'external' }], repository: {} };
+			const target = location === 'pattern' ? raw.patterns[0]
+				: location === 'repository' ? raw.repository.id
+					: location === 'capture' ? raw.repository.captured.captures['1']
+						: location === 'injection' ? raw.injections['L:source.test'] : external.patterns[0];
+			Object.assign(target, { id });
+			const input = JSON.stringify(raw);
+			const externalInput = JSON.stringify(external);
+			const grammarInput = parseRawGrammar(input, 'test.json');
+			const externalGrammarInput = parseRawGrammar(externalInput, 'external.json');
+			for (let pass = 0; pass < 2; pass++) {
+				const registry = new Registry({
+					onigLib: getOniguruma(),
+					loadGrammar: async scope => scope === 'source.test' ? grammarInput : externalGrammarInput
+				});
+				try {
+					const grammar = await registry.loadGrammar('source.test');
+					assert.ok(grammar);
+					const result = grammar.tokenizeLine('"\\n" x y z', null);
+					assert.deepStrictEqual(result.tokens, [
+						{ startIndex: 0, endIndex: 1, scopes: ['source.test', 'string'] },
+						{ startIndex: 1, endIndex: 3, scopes: ['source.test', 'string', 'escape'] },
+						{ startIndex: 3, endIndex: 4, scopes: ['source.test', 'string'] },
+						{ startIndex: 4, endIndex: 5, scopes: ['source.test'] },
+						{ startIndex: 5, endIndex: 6, scopes: ['source.test', 'word', 'capture', 'inner'] },
+						{ startIndex: 6, endIndex: 7, scopes: ['source.test'] },
+						{ startIndex: 7, endIndex: 8, scopes: ['source.test', 'external'] },
+						{ startIndex: 8, endIndex: 9, scopes: ['source.test'] },
+						{ startIndex: 9, endIndex: 10, scopes: ['source.test', 'injected'] }
+					]);
+					assert.strictEqual(JSON.stringify(grammarInput), input);
+					assert.strictEqual(JSON.stringify(externalGrammarInput), externalInput);
+				} finally {
+					registry.dispose();
+				}
+			}
+		});
+	}
+}
+
+for (const capture of ['beginCaptures', 'endCaptures', 'whileCaptures']) {
+	test(`Raw rule id metadata does not affect ${capture}`, async () => {
+		const raw = {
+			scopeName: 'source.test', repository: {},
+			patterns: [{
+				begin: '(x)',
+				...(capture === 'whileCaptures' ? { while: '(y)' } : { end: '(y)' }),
+				name: 'block',
+				[capture]: { '1': { id: 'metadata', patterns: [{ match: '[xy]', name: 'captured' }] } }
+			}]
+		};
+		const registry = new Registry({ onigLib: getOniguruma(), loadGrammar: async () => parseRawGrammar(JSON.stringify(raw), 'test.json') });
+		try {
+			const grammar = await registry.loadGrammar('source.test');
+			assert.ok(grammar);
+			const first = grammar.tokenizeLine('x', null);
+			const result = capture === 'beginCaptures' ? first : grammar.tokenizeLine('y', first.ruleStack);
+			assert.deepStrictEqual(result.tokens[0], { startIndex: 0, endIndex: 1, scopes: ['source.test', 'block', 'captured'] });
+		} finally {
+			registry.dispose();
+		}
+	});
+}