From b022f751c004fa200485216d143cff9295eab13b Mon Sep 17 00:00:00 2001 From: glx22 Date: Tue, 14 Jan 2025 21:53:55 +0100 Subject: [PATCH 1/2] Add: "include" keyword, allowing to include files without external tools --- nml/generic.py | 10 ++++++ nml/main.py | 7 +--- nml/tokens.py | 53 +++++++++++++++++++++++++--- regression/044_include.inc | 1 + regression/044_include.nml | 4 +++ regression/expected/044_include.grf | Bin 0 -> 79 bytes regression/expected/044_include.nfo | 19 ++++++++++ 7 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 regression/044_include.inc create mode 100644 regression/044_include.nml create mode 100644 regression/expected/044_include.grf create mode 100644 regression/expected/044_include.nfo diff --git a/nml/generic.py b/nml/generic.py index a39180bce..9724cbd12 100644 --- a/nml/generic.py +++ b/nml/generic.py @@ -13,6 +13,7 @@ with NML; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.""" +import codecs import os import sys import time @@ -616,3 +617,12 @@ def open_cache_file(sources, extension, mode): "Can't create cache file {}. Check permissions, or use --cache-dir or --no-cache.".format(path), ) raise + + +def read_file(input): + try: + script = input.read() + except UnicodeDecodeError as ex: + raise ScriptError(f"Input file is not utf-8 encoded: {ex}") + # Strip a possible BOM + return script.lstrip(str(codecs.BOM_UTF8, "utf-8")) diff --git a/nml/main.py b/nml/main.py index afd2435b2..3f44648d6 100644 --- a/nml/main.py +++ b/nml/main.py @@ -410,12 +410,7 @@ def nml( generic.print_progress("Reading {} ...".format(input_filename or "standard input")) - try: - script = inputfile.read() - except UnicodeDecodeError as ex: - raise generic.ScriptError("Input file is not utf-8 encoded: {}".format(ex)) - # Strip a possible BOM - script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) + script = generic.read_file(inputfile) if script.strip() == "": generic.print_error("Empty input file") diff --git a/nml/tokens.py b/nml/tokens.py index 32358980c..9e2f6af57 100644 --- a/nml/tokens.py +++ b/nml/tokens.py @@ -13,6 +13,7 @@ with NML; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.""" +import codecs import re import sys @@ -62,7 +63,7 @@ "recolour_sprite": "RECOLOUR_SPRITE", "engine_override": "ENGINE_OVERRIDE", "sort": "SORT_VEHICLES", - "const": "CONST" + "const": "CONST", } # fmt: on @@ -82,6 +83,8 @@ class NMLLexer: @type text: C{str} """ + states = (("include", "inclusive"),) + # Tokens tokens = list(reserved.values()) + [ "ID", @@ -158,6 +161,37 @@ class NMLLexer: t_COLON = r":" t_SEMICOLON = r";" + def t_include_INCLUDE(self, t): + r"\binclude\b" + + def expect(type): + t = self.lexer.token() + if t is None: + raise generic.ScriptError("Syntax error, unexpected end-of-file") + if t.type != type: + raise generic.ScriptError(f'Syntax error, unexpected token "{t.value}"', t.lineno) + return t + + self.lexer.push_state("INITIAL") + expect("LPAREN") + fname = expect("STRING_LITERAL").value.value + expect("RPAREN") + expect("SEMICOLON") + self.lexer.pop_state() + + self.states.append((self.lexer.lexdata, self.lexer.lexpos)) + self.includes.append(self.lexer.lineno) + for i in self.includes: + if generic.find_file(i.filename) == generic.find_file(fname): + raise generic.ScriptError(f"Include loop detected: {fname}", t.lineno) + with codecs.open(generic.find_file(fname), "r", "utf-8") as input: + script = generic.read_file(input) + self.setup(script, fname) + + token = self.lexer.token() + self.lexer.lexmatch = None # is deleted, so the slot should exist on exit + return token + def t_FLOAT(self, t): r"\d+\.\d+" t.value = expression.ConstantFloat(float(t.value), t.lineno) @@ -258,11 +292,24 @@ def t_error(self, t): ) sys.exit(1) + def t_include_eof(self, t): + if self.states: + lexdata, lexpos = self.states.pop() + self.lexer.input(lexdata) + self.lexer.lexpos = lexpos + self.lexer.lineno = self.includes.pop() + if self.lexer.lexpos != self.lexer.lexlen: + return self.lexer.token() + return None + def build(self): """ Initial construction of the scanner. """ self.lexer = lex.lex(module=self) + self.lexer.begin("include") + self.includes = [] + self.states = [] def setup(self, text, fname): """ @@ -274,8 +321,6 @@ def setup(self, text, fname): @param fname: Filename associated with the input text (main input file). @type fname: C{str} """ - self.includes = [] - self.text = text self.set_position(fname, 1) self.lexer.input(text) @@ -289,7 +334,7 @@ def increment_lines(self, count): self.set_position(self.lexer.lineno.filename, self.lexer.lineno.line_start + count) def find_column(self, t): - last_cr = self.text.rfind("\n", 0, t.lexpos) + last_cr = t.lexer.lexdata.rfind("\n", 0, t.lexpos) if last_cr < 0: last_cr = 0 return t.lexpos - last_cr diff --git a/regression/044_include.inc b/regression/044_include.inc new file mode 100644 index 000000000..bc5c7ed93 --- /dev/null +++ b/regression/044_include.inc @@ -0,0 +1 @@ +param[0] = param[0] + 1; diff --git a/regression/044_include.nml b/regression/044_include.nml new file mode 100644 index 000000000..2563e8b53 --- /dev/null +++ b/regression/044_include.nml @@ -0,0 +1,4 @@ +param[0] = 0; +include("044_include.inc"); +include ("044_include.inc" ); +include( "044_include.inc"); diff --git a/regression/expected/044_include.grf b/regression/expected/044_include.grf new file mode 100644 index 0000000000000000000000000000000000000000..4096e6d3db5bc7280f434baf34a997e64473f39f GIT binary patch literal 79 ocmZQza1U~8;^mU!vSk1QP9Xlz3&bD}jK|3EpOF9$tQ<%H0J~EPFaQ7m literal 0 HcmV?d00001 diff --git a/regression/expected/044_include.nfo b/regression/expected/044_include.nfo new file mode 100644 index 000000000..2b14f9bfb --- /dev/null +++ b/regression/expected/044_include.nfo @@ -0,0 +1,19 @@ +// Automatically generated by GRFCODEC. Do not modify! +// (Info version 32) +// Escapes: 2+ 2- 2< 2> 2u< 2u> 2/ 2% 2u/ 2u% 2* 2& 2| 2^ 2sto = 2s 2rst = 2r 2psto 2ror = 2rot 2cmp 2ucmp 2<< 2u>> 2>> +// Escapes: 71 70 7= 7! 7< 7> 7G 7g 7gG 7GG 7gg 7c 7C +// Escapes: D= = DR D+ = DF D- = DC Du* = DM D* = DnF Du<< = DnC D<< = DO D& D| Du/ D/ Du% D% +// Format: spritenum imagefile depth xpos ypos xsize ysize xrel yrel zoom flags + +// param[0] = 0 +0 * 9 0D 00 \D= FF 00 \dx00000000 + +// param[0] = (param[0] + 1) +1 * 9 0D 00 \D+ 00 FF \dx00000001 + +// param[0] = (param[0] + 1) +2 * 9 0D 00 \D+ 00 FF \dx00000001 + +// param[0] = (param[0] + 1) +3 * 9 0D 00 \D+ 00 FF \dx00000001 + From 729cb3bc65caf5cfb433e9aa8dbce939139777f7 Mon Sep 17 00:00:00 2001 From: glx22 Date: Tue, 13 Oct 2026 18:46:32 +0200 Subject: [PATCH 2/2] Add: Optional 'ID = "replacement' list when parsing included files --- nml/generic.py | 10 ++++++++-- nml/tokens.py | 15 +++++++++++---- regression/044_include.inc | 5 ++++- regression/044_include.nml | 10 +++++++--- regression/expected/044_include.grf | Bin 79 -> 177 bytes regression/expected/044_include.nfo | 25 +++++++++++++++++++++++-- 6 files changed, 53 insertions(+), 12 deletions(-) diff --git a/nml/generic.py b/nml/generic.py index 9724cbd12..d105e2400 100644 --- a/nml/generic.py +++ b/nml/generic.py @@ -15,6 +15,7 @@ import codecs import os +import re import sys import time @@ -619,10 +620,15 @@ def open_cache_file(sources, extension, mode): raise -def read_file(input): +def read_file(input, symbols={}): try: script = input.read() except UnicodeDecodeError as ex: raise ScriptError(f"Input file is not utf-8 encoded: {ex}") # Strip a possible BOM - return script.lstrip(str(codecs.BOM_UTF8, "utf-8")) + script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) + + def repl(m): + return symbols.get(*m.groups("")) + + return re.sub("{([a-zA-Z_][a-zA-Z0-9_]*)(?:=([^}]*?))?}", repl, script) diff --git a/nml/tokens.py b/nml/tokens.py index 9e2f6af57..67fd944d1 100644 --- a/nml/tokens.py +++ b/nml/tokens.py @@ -164,18 +164,22 @@ class NMLLexer: def t_include_INCLUDE(self, t): r"\binclude\b" - def expect(type): + def expect(*types): t = self.lexer.token() if t is None: raise generic.ScriptError("Syntax error, unexpected end-of-file") - if t.type != type: + if t.type not in types: raise generic.ScriptError(f'Syntax error, unexpected token "{t.value}"', t.lineno) return t self.lexer.push_state("INITIAL") expect("LPAREN") fname = expect("STRING_LITERAL").value.value - expect("RPAREN") + self.symbols.append(self.symbols[0].copy()) + while expect("COMMA", "RPAREN").type == "COMMA": + id = expect("ID").value.value + expect("EQ") + self.symbols[0][id] = re.sub(r"\\(.)", r"\1", expect("STRING_LITERAL").value.value) expect("SEMICOLON") self.lexer.pop_state() @@ -185,7 +189,7 @@ def expect(type): if generic.find_file(i.filename) == generic.find_file(fname): raise generic.ScriptError(f"Include loop detected: {fname}", t.lineno) with codecs.open(generic.find_file(fname), "r", "utf-8") as input: - script = generic.read_file(input) + script = generic.read_file(input, self.symbols[0]) self.setup(script, fname) token = self.lexer.token() @@ -222,6 +226,7 @@ def t_ID(self, t): def t_STRING_LITERAL(self, t): r'"([^"\\]|\\.)*"' t.value = expression.StringLiteral(t.value[1:-1], t.lineno) + self.increment_lines(t.value.value.count("\n")) return t # Ignored characters @@ -298,6 +303,7 @@ def t_include_eof(self, t): self.lexer.input(lexdata) self.lexer.lexpos = lexpos self.lexer.lineno = self.includes.pop() + self.symbols[0] = self.symbols.pop() if self.lexer.lexpos != self.lexer.lexlen: return self.lexer.token() return None @@ -310,6 +316,7 @@ def build(self): self.lexer.begin("include") self.includes = [] self.states = [] + self.symbols = [{}] def setup(self, text, fname): """ diff --git a/regression/044_include.inc b/regression/044_include.inc index bc5c7ed93..146c5bf88 100644 --- a/regression/044_include.inc +++ b/regression/044_include.inc @@ -1 +1,4 @@ -param[0] = param[0] + 1; +param[0] = param[0] + {VALUE}; +const _{NAME}xy = {VALUE}; +param[0] = param[0] + _{NAME}xy; +{extra=param[3] = 3;} diff --git a/regression/044_include.nml b/regression/044_include.nml index 2563e8b53..523bca60a 100644 --- a/regression/044_include.nml +++ b/regression/044_include.nml @@ -1,4 +1,8 @@ param[0] = 0; -include("044_include.inc"); -include ("044_include.inc" ); -include( "044_include.inc"); +include("044_include{empty=}.inc", VALUE = "1", + extra = " + param[1] = 1; + param[2] = 2; + "); +include("044_include.inc", VALUE = "2", NAME = "test"); +include("044_include.inc", VALUE = "3", NAME = "test2"); diff --git a/regression/expected/044_include.grf b/regression/expected/044_include.grf index 4096e6d3db5bc7280f434baf34a997e64473f39f..49a3b349414320782906f32d921da670cbcc3e3c 100644 GIT binary patch literal 177 zcmZQza1U~8;^mU!n$G|PoIw1a7l=U|7>|+RKO+tvP>KN|#{}dtp{T^gV+N{ZMp222 K2U7`D3