Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions nml/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
with NML; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA."""

import codecs
import os
import re
import sys
import time

Expand Down Expand Up @@ -616,3 +618,17 @@ def open_cache_file(sources, extension, mode):
"Can't create cache file {}. Check permissions, or use --cache-dir or --no-cache.".format(path),
)
raise


def read_file(input, symbols={}):
try:
script = input.read()
except UnicodeDecodeError as ex:
raise ScriptError(f"Input file is not utf-8 encoded: {ex}")
# Strip a possible BOM
script = script.lstrip(str(codecs.BOM_UTF8, "utf-8"))

def repl(m):
return symbols.get(*m.groups(""))

return re.sub("{([a-zA-Z_][a-zA-Z0-9_]*)(?:=([^}]*?))?}", repl, script)
7 changes: 1 addition & 6 deletions nml/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,12 +410,7 @@ def nml(

generic.print_progress("Reading {} ...".format(input_filename or "standard input"))

try:
script = inputfile.read()
except UnicodeDecodeError as ex:
raise generic.ScriptError("Input file is not utf-8 encoded: {}".format(ex))
# Strip a possible BOM
script = script.lstrip(str(codecs.BOM_UTF8, "utf-8"))
script = generic.read_file(inputfile)

if script.strip() == "":
generic.print_error("Empty input file")
Expand Down
60 changes: 56 additions & 4 deletions nml/tokens.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
with NML; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA."""

import codecs
import re
import sys

Expand Down Expand Up @@ -62,7 +63,7 @@
"recolour_sprite": "RECOLOUR_SPRITE",
"engine_override": "ENGINE_OVERRIDE",
"sort": "SORT_VEHICLES",
"const": "CONST"
"const": "CONST",
}
# fmt: on

Expand All @@ -82,6 +83,8 @@ class NMLLexer:
@type text: C{str}
"""

states = (("include", "inclusive"),)

# Tokens
tokens = list(reserved.values()) + [
"ID",
Expand Down Expand Up @@ -158,6 +161,41 @@ class NMLLexer:
t_COLON = r":"
t_SEMICOLON = r";"

def t_include_INCLUDE(self, t):
r"\binclude\b"

def expect(*types):
t = self.lexer.token()
if t is None:
raise generic.ScriptError("Syntax error, unexpected end-of-file")
if t.type not in types:
raise generic.ScriptError(f'Syntax error, unexpected token "{t.value}"', t.lineno)
return t

self.lexer.push_state("INITIAL")
expect("LPAREN")
fname = expect("STRING_LITERAL").value.value
self.symbols.append(self.symbols[0].copy())
while expect("COMMA", "RPAREN").type == "COMMA":
id = expect("ID").value.value
expect("EQ")
self.symbols[0][id] = re.sub(r"\\(.)", r"\1", expect("STRING_LITERAL").value.value)
expect("SEMICOLON")
self.lexer.pop_state()

self.states.append((self.lexer.lexdata, self.lexer.lexpos))
self.includes.append(self.lexer.lineno)
for i in self.includes:
if generic.find_file(i.filename) == generic.find_file(fname):
raise generic.ScriptError(f"Include loop detected: {fname}", t.lineno)
with codecs.open(generic.find_file(fname), "r", "utf-8") as input:
script = generic.read_file(input, self.symbols[0])
self.setup(script, fname)

token = self.lexer.token()
self.lexer.lexmatch = None # is deleted, so the slot should exist on exit
return token

def t_FLOAT(self, t):
r"\d+\.\d+"
t.value = expression.ConstantFloat(float(t.value), t.lineno)
Expand Down Expand Up @@ -188,6 +226,7 @@ def t_ID(self, t):
def t_STRING_LITERAL(self, t):
r'"([^"\\]|\\.)*"'
t.value = expression.StringLiteral(t.value[1:-1], t.lineno)
self.increment_lines(t.value.value.count("\n"))
return t

# Ignored characters
Expand Down Expand Up @@ -258,11 +297,26 @@ def t_error(self, t):
)
sys.exit(1)

def t_include_eof(self, t):
if self.states:
lexdata, lexpos = self.states.pop()
self.lexer.input(lexdata)
self.lexer.lexpos = lexpos
self.lexer.lineno = self.includes.pop()
self.symbols[0] = self.symbols.pop()
if self.lexer.lexpos != self.lexer.lexlen:
return self.lexer.token()
return None

def build(self):
"""
Initial construction of the scanner.
"""
self.lexer = lex.lex(module=self)
self.lexer.begin("include")
self.includes = []
self.states = []
self.symbols = [{}]

def setup(self, text, fname):
"""
Expand All @@ -274,8 +328,6 @@ def setup(self, text, fname):
@param fname: Filename associated with the input text (main input file).
@type fname: C{str}
"""
self.includes = []
self.text = text
self.set_position(fname, 1)
self.lexer.input(text)

Expand All @@ -289,7 +341,7 @@ def increment_lines(self, count):
self.set_position(self.lexer.lineno.filename, self.lexer.lineno.line_start + count)

def find_column(self, t):
last_cr = self.text.rfind("\n", 0, t.lexpos)
last_cr = t.lexer.lexdata.rfind("\n", 0, t.lexpos)
if last_cr < 0:
last_cr = 0
return t.lexpos - last_cr
4 changes: 4 additions & 0 deletions regression/044_include.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
param[0] = param[0] + {VALUE};
const _{NAME}xy = {VALUE};
param[0] = param[0] + _{NAME}xy;
{extra=param[3] = 3;}
8 changes: 8 additions & 0 deletions regression/044_include.nml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
param[0] = 0;
include("044_include{empty=}.inc", VALUE = "1",
extra = "
param[1] = 1;
param[2] = 2;
");
include("044_include.inc", VALUE = "2", NAME = "test");
include("044_include.inc", VALUE = "3", NAME = "test2");
Binary file added regression/expected/044_include.grf
Binary file not shown.
40 changes: 40 additions & 0 deletions regression/expected/044_include.nfo
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Automatically generated by GRFCODEC. Do not modify!
// (Info version 32)
// Escapes: 2+ 2- 2< 2> 2u< 2u> 2/ 2% 2u/ 2u% 2* 2& 2| 2^ 2sto = 2s 2rst = 2r 2psto 2ror = 2rot 2cmp 2ucmp 2<< 2u>> 2>>
// Escapes: 71 70 7= 7! 7< 7> 7G 7g 7gG 7GG 7gg 7c 7C
// Escapes: D= = DR D+ = DF D- = DC Du* = DM D* = DnF Du<< = DnC D<< = DO D& D| Du/ D/ Du% D%
// Format: spritenum imagefile depth xpos ypos xsize ysize xrel yrel zoom flags

// param[0] = 0
0 * 9 0D 00 \D= FF 00 \dx00000000

// param[0] = (param[0] + 1)
1 * 9 0D 00 \D+ 00 FF \dx00000001

// param[0] = (param[0] + 1)
2 * 9 0D 00 \D+ 00 FF \dx00000001

// param[1] = 1
3 * 9 0D 01 \D= FF 00 \dx00000001

// param[2] = 2
4 * 9 0D 02 \D= FF 00 \dx00000002

// param[0] = (param[0] + 2)
5 * 9 0D 00 \D+ 00 FF \dx00000002

// param[0] = (param[0] + 2)
6 * 9 0D 00 \D+ 00 FF \dx00000002

// param[3] = 3
7 * 9 0D 03 \D= FF 00 \dx00000003

// param[0] = (param[0] + 3)
8 * 9 0D 00 \D+ 00 FF \dx00000003

// param[0] = (param[0] + 3)
9 * 9 0D 00 \D+ 00 FF \dx00000003

// param[3] = 3
10 * 9 0D 03 \D= FF 00 \dx00000003

Loading