Skip to content

crash_test.py improvements #1997

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
slipher wants to merge 3 commits into
base: master
Choose a base branch
from
+33 −17
Open

crash_test.py improvements #1997

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cae11f9
tools/crash_test.py: add --no-breakpad option
slipher Jul 9, 2026
e085bff
Miscellaneous tools/crash_test.py adjustments
slipher Sep 5, 2025
d44eff2
crash_test.py: require function OR file names
slipher Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 33 additions & 17 deletions tools/crash_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,23 @@ def Begin(self):
def End(self):
print(f"==={self.status}: {self.name}===")

def Verify(self, cond, reason=None):
def Verify(self, cond, reason=None, fail=True):
if not cond:
if reason is not None:
print(f"FAILURE: {reason}")
self.status = "FAILED"
if GIVE_UP:
raise Exception("Giving up on first failure")
if fail:
if reason is not None:
print(f"FAILURE: {reason}")
self.status = "FAILED"
if GIVE_UP:
raise Exception("Giving up on first failure")
else:
print(f"WARNING: {reason}")

def Go(self):
self.Begin()
try:
self.Do()
except Exception as e:
traceback.print_exception(e)
traceback.print_exc()
self.status = "FAILED"
self.End()
return self.status == "PASSED"
Expand All @@ -73,12 +76,13 @@ def __init__(self, module, engine, tprefix, fault):

def Do(self):
vmtype = "0" if SYMBOL_ZIPS else "1"
print("Running daemon...")
print("Running Daemon...")
p = subprocess.run([self.engine,
"-noforward", # catch stupid Linux persistent sockets (if gamelogic test...)
"-homepath", self.dir,
"-set", "vm.sgame.type", vmtype,
"-set", "vm.cgame.type", vmtype,
"-set", "vm.timeout", "30", # in case of emulator

@illwieckz illwieckz Jul 29, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As a side note, in my branch implementing box64 support I also increase this in engine directly.
I don't see why this timeout should be short: the time is only waited when the game is slow to start, and if it's slow to start but not failing, we should not give up. Previously I faced the same timeout problem when running the game on a network file system over a very slow network where it was slow but not failing (that may have been what prompted the introduction of the cvar). When the system is not slow, the delay is never experienced, and when the system is slow, there is no other solution than waiting and it's fine.

@slipher slipher Jul 29, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How could a slow network cause that? The VM process can't do any I/O other than IPC messages.

@illwieckz illwieckz Jul 30, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That timeout also counts the time between execing the nacl_loader with the nexe and the first actual response from the running nexe on IPC, or something like that. So the timeout counts for all the nacl_runtime and nexe to be read (if slow file system…), then counts for the compatibility layer (box64, rosetta2…) to initialize itself or even do some magic processing on the binaries, etc.

"-set", "sv_fps", "1000",
"-set", "common.framerate.max", "0",
"-set", "client.errorPopup", "0",
Expand All @@ -95,16 +99,21 @@ def Do(self):
stderr=subprocess.PIPE, check=bool(self.tprefix))
dumps = os.listdir(PathJoin(self.dir, "crashdump"))
assert len(dumps) == 1, dumps
if BREAKPAD_DIR is None:
return
dump = PathJoin(self.dir, "crashdump", dumps[0])
sw_out = PathJoin(TEMP_DIR, self.name + "_stackwalk.log")
with open(sw_out, "a+") as sw_f:
print(f"Extracting stack trace to '{sw_out}'...")
sw_f.truncate()
subprocess.run(Virtualize([PathJoin(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]), check=True, stdout=sw_f, stderr=subprocess.STDOUT)
sw_f.truncate(0)
subprocess.run(Virtualize([PathJoin(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]),
check=True, stdout=sw_f, stderr=sw_f)
sw_f.seek(0)
sw = sw_f.read()
TRACE_FUNC = "InjectFaultCmd::Run"
self.Verify(TRACE_FUNC in sw, "function names not found in trace (did you build with symbols?)")
# Check both function names and filenames. On Linux it seems like only one of them works at a time??
self.Verify(srcnames := ("Command.cpp" in sw), "source file names not found in trace", fail=False)
self.Verify(funcnames := ("InjectFaultCmd::Run" in sw), "function names not found in trace", fail=False)
self.Verify(srcnames or funcnames, "neither file nor function names found in trace")

def Virtualize(cmdline):
bin, *args = cmdline
Expand Down Expand Up @@ -146,10 +155,11 @@ def Do(self):
else:
tprefix = ""
eng = module
assert os.path.isfile(PathJoin(GAME_DIR, "crash_server" + EXE))
engine = ModulePath(eng)
assert os.path.isfile(engine), engine

if not SYMBOL_ZIPS:
if BREAKPAD_DIR is not None and not SYMBOL_ZIPS:
target = ModulePath(module)
assert os.path.isfile(target), target
print(f"Symbolizing '{target}'...")
Expand All @@ -173,6 +183,7 @@ def ArgParser(usage=None):
" Otherwise, enter end-to-end mode: symbols are produced from the binaries and VM type defaults to 1 (NaCl from PWD). In this mode you will likely need to provide pak paths via --daemon-args.")
ap.add_argument("--game-dir", type=str, default=".", help="Path to Daemon (+ gamelogic) binaries")
ap.add_argument("--breakpad-dir", type=str, default=BREAKPAD_DIR, help=r"Path to Breakpad repo containing built dump_syms and stackwalk binaries. It may be a \\wsl.localhost\ path on Windows hosts in order to symbolize NaCl.")
ap.add_argument("--no-breakpad", action="store_true", help="Do not symbolize or stack-walk; only test for dump file existence")
ap.add_argument("--give-up", action="store_true", help="Stop after first test failure")
ap.add_argument("--nacl-arch", type=str, choices=["amd64", "i686", "armhf"], default="amd64") # TODO auto-detect?
ap.add_argument("module", nargs="*",
Expand All @@ -191,7 +202,7 @@ def ArgParser(usage=None):
help="Extra arguments for Daemon (e.g. -pakpath)")
pa = ap.parse_args(sys.argv[1:])
GAME_DIR = pa.game_dir
BREAKPAD_DIR = pa.breakpad_dir
BREAKPAD_DIR = None if pa.no_breakpad else pa.breakpad_dir
GIVE_UP = pa.give_up
DAEMON_USER_ARGS = pa.daemon_args
NACL_ARCH = pa.nacl_arch
Expand All @@ -207,9 +218,14 @@ def ArgParser(usage=None):

if SYMBOL_ZIPS:
print("Symbol zip(s) detected. Using release validation mode with pre-built symbols")
for z in SYMBOL_ZIPS:
with zipfile.ZipFile(PathJoin(GAME_DIR, z), 'r') as z:
z.extractall(SYMBOL_DIR)
if BREAKPAD_DIR:
for z in SYMBOL_ZIPS:
with zipfile.ZipFile(PathJoin(GAME_DIR, z), 'r') as z:
z.extractall(SYMBOL_DIR)
if sys.platform == "darwin":
assert os.path.isdir(os.path.join(GAME_DIR, "Unvanquished.app"))
DAEMON_USER_ARGS = ["-pakpath", os.path.join(GAME_DIR, "pkg")] + DAEMON_USER_ARGS
GAME_DIR = os.path.join(GAME_DIR, "Unvanquished.app/Contents/MacOS")
else:
print("No symbol zip detected. Using end-to-end Breakpad tooling test mode with dump_syms")

Expand Down
Loading