diff --git a/nml/generic.py b/nml/generic.py index a39180bc..d105e240 100644 --- a/nml/generic.py +++ b/nml/generic.py @@ -13,7 +13,9 @@ with NML; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.""" +import codecs import os +import re import sys import time @@ -616,3 +618,17 @@ def open_cache_file(sources, extension, mode): "Can't create cache file {}. Check permissions, or use --cache-dir or --no-cache.".format(path), ) raise + + +def read_file(input, symbols={}): + try: + script = input.read() + except UnicodeDecodeError as ex: + raise ScriptError(f"Input file is not utf-8 encoded: {ex}") + # Strip a possible BOM + script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) + + def repl(m): + return symbols.get(*m.groups("")) + + return re.sub("{([a-zA-Z_][a-zA-Z0-9_]*)(?:=([^}]*?))?}", repl, script) diff --git a/nml/main.py b/nml/main.py index afd2435b..3f44648d 100644 --- a/nml/main.py +++ b/nml/main.py @@ -410,12 +410,7 @@ def nml( generic.print_progress("Reading {} ...".format(input_filename or "standard input")) - try: - script = inputfile.read() - except UnicodeDecodeError as ex: - raise generic.ScriptError("Input file is not utf-8 encoded: {}".format(ex)) - # Strip a possible BOM - script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) + script = generic.read_file(inputfile) if script.strip() == "": generic.print_error("Empty input file") diff --git a/nml/tokens.py b/nml/tokens.py index 32358980..67fd944d 100644 --- a/nml/tokens.py +++ b/nml/tokens.py @@ -13,6 +13,7 @@ with NML; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.""" +import codecs import re import sys @@ -62,7 +63,7 @@ "recolour_sprite": "RECOLOUR_SPRITE", "engine_override": "ENGINE_OVERRIDE", "sort": "SORT_VEHICLES", - "const": "CONST" + "const": "CONST", } # fmt: on @@ -82,6 +83,8 @@ class NMLLexer: @type text: C{str} """ + states = (("include", "inclusive"),) + # Tokens tokens = list(reserved.values()) + [ "ID", @@ -158,6 +161,41 @@ class NMLLexer: t_COLON = r":" t_SEMICOLON = r";" + def t_include_INCLUDE(self, t): + r"\binclude\b" + + def expect(*types): + t = self.lexer.token() + if t is None: + raise generic.ScriptError("Syntax error, unexpected end-of-file") + if t.type not in types: + raise generic.ScriptError(f'Syntax error, unexpected token "{t.value}"', t.lineno) + return t + + self.lexer.push_state("INITIAL") + expect("LPAREN") + fname = expect("STRING_LITERAL").value.value + self.symbols.append(self.symbols[0].copy()) + while expect("COMMA", "RPAREN").type == "COMMA": + id = expect("ID").value.value + expect("EQ") + self.symbols[0][id] = re.sub(r"\\(.)", r"\1", expect("STRING_LITERAL").value.value) + expect("SEMICOLON") + self.lexer.pop_state() + + self.states.append((self.lexer.lexdata, self.lexer.lexpos)) + self.includes.append(self.lexer.lineno) + for i in self.includes: + if generic.find_file(i.filename) == generic.find_file(fname): + raise generic.ScriptError(f"Include loop detected: {fname}", t.lineno) + with codecs.open(generic.find_file(fname), "r", "utf-8") as input: + script = generic.read_file(input, self.symbols[0]) + self.setup(script, fname) + + token = self.lexer.token() + self.lexer.lexmatch = None # is deleted, so the slot should exist on exit + return token + def t_FLOAT(self, t): r"\d+\.\d+" t.value = expression.ConstantFloat(float(t.value), t.lineno) @@ -188,6 +226,7 @@ def t_ID(self, t): def t_STRING_LITERAL(self, t): r'"([^"\\]|\\.)*"' t.value = expression.StringLiteral(t.value[1:-1], t.lineno) + self.increment_lines(t.value.value.count("\n")) return t # Ignored characters @@ -258,11 +297,26 @@ def t_error(self, t): ) sys.exit(1) + def t_include_eof(self, t): + if self.states: + lexdata, lexpos = self.states.pop() + self.lexer.input(lexdata) + self.lexer.lexpos = lexpos + self.lexer.lineno = self.includes.pop() + self.symbols[0] = self.symbols.pop() + if self.lexer.lexpos != self.lexer.lexlen: + return self.lexer.token() + return None + def build(self): """ Initial construction of the scanner. """ self.lexer = lex.lex(module=self) + self.lexer.begin("include") + self.includes = [] + self.states = [] + self.symbols = [{}] def setup(self, text, fname): """ @@ -274,8 +328,6 @@ def setup(self, text, fname): @param fname: Filename associated with the input text (main input file). @type fname: C{str} """ - self.includes = [] - self.text = text self.set_position(fname, 1) self.lexer.input(text) @@ -289,7 +341,7 @@ def increment_lines(self, count): self.set_position(self.lexer.lineno.filename, self.lexer.lineno.line_start + count) def find_column(self, t): - last_cr = self.text.rfind("\n", 0, t.lexpos) + last_cr = t.lexer.lexdata.rfind("\n", 0, t.lexpos) if last_cr < 0: last_cr = 0 return t.lexpos - last_cr diff --git a/regression/044_include.inc b/regression/044_include.inc new file mode 100644 index 00000000..146c5bf8 --- /dev/null +++ b/regression/044_include.inc @@ -0,0 +1,4 @@ +param[0] = param[0] + {VALUE}; +const _{NAME}xy = {VALUE}; +param[0] = param[0] + _{NAME}xy; +{extra=param[3] = 3;} diff --git a/regression/044_include.nml b/regression/044_include.nml new file mode 100644 index 00000000..523bca60 --- /dev/null +++ b/regression/044_include.nml @@ -0,0 +1,8 @@ +param[0] = 0; +include("044_include{empty=}.inc", VALUE = "1", + extra = " + param[1] = 1; + param[2] = 2; + "); +include("044_include.inc", VALUE = "2", NAME = "test"); +include("044_include.inc", VALUE = "3", NAME = "test2"); diff --git a/regression/expected/044_include.grf b/regression/expected/044_include.grf new file mode 100644 index 00000000..49a3b349 Binary files /dev/null and b/regression/expected/044_include.grf differ diff --git a/regression/expected/044_include.nfo b/regression/expected/044_include.nfo new file mode 100644 index 00000000..b4fc60e1 --- /dev/null +++ b/regression/expected/044_include.nfo @@ -0,0 +1,40 @@ +// Automatically generated by GRFCODEC. Do not modify! +// (Info version 32) +// Escapes: 2+ 2- 2< 2> 2u< 2u> 2/ 2% 2u/ 2u% 2* 2& 2| 2^ 2sto = 2s 2rst = 2r 2psto 2ror = 2rot 2cmp 2ucmp 2<< 2u>> 2>> +// Escapes: 71 70 7= 7! 7< 7> 7G 7g 7gG 7GG 7gg 7c 7C +// Escapes: D= = DR D+ = DF D- = DC Du* = DM D* = DnF Du<< = DnC D<< = DO D& D| Du/ D/ Du% D% +// Format: spritenum imagefile depth xpos ypos xsize ysize xrel yrel zoom flags + +// param[0] = 0 +0 * 9 0D 00 \D= FF 00 \dx00000000 + +// param[0] = (param[0] + 1) +1 * 9 0D 00 \D+ 00 FF \dx00000001 + +// param[0] = (param[0] + 1) +2 * 9 0D 00 \D+ 00 FF \dx00000001 + +// param[1] = 1 +3 * 9 0D 01 \D= FF 00 \dx00000001 + +// param[2] = 2 +4 * 9 0D 02 \D= FF 00 \dx00000002 + +// param[0] = (param[0] + 2) +5 * 9 0D 00 \D+ 00 FF \dx00000002 + +// param[0] = (param[0] + 2) +6 * 9 0D 00 \D+ 00 FF \dx00000002 + +// param[3] = 3 +7 * 9 0D 03 \D= FF 00 \dx00000003 + +// param[0] = (param[0] + 3) +8 * 9 0D 00 \D+ 00 FF \dx00000003 + +// param[0] = (param[0] + 3) +9 * 9 0D 00 \D+ 00 FF \dx00000003 + +// param[3] = 3 +10 * 9 0D 03 \D= FF 00 \dx00000003 +