From dc5be577db61b9d712fa0f630d247bf17f1f023c Mon Sep 17 00:00:00 2001 From: Cole Fuerth Date: Thu, 24 Sep 2026 09:36:03 -0700 Subject: [PATCH] nix: only install the Unity stdio hooks in a Unity player LD_PRELOAD is inherited by every child process, so Doorstop is also loaded into the launcher scripts that start the game. When no UnityPlayer module is present, doorstop_ctor() falls back to hooking the main executable's PLT and installs fclose_hook and dup2_hook there. Both hooks deliberately suppress redirection of stdout/stderr, which is what a shell uses to implement command substitution and file redirection, so any /bin/sh in the launch chain silently breaks: $ LD_PRELOAD=libdoorstop.so DOORSTOP_ENABLED=1 \ DOORSTOP_TARGET_ASSEMBLY=... \ sh -c 'x="$(echo hello)"; echo "got=[$x]"' hello got=[] This breaks BepInEx on every Steam Linux Runtime game: the runtime's _v2-entry-point parses its arguments with `getopt_temp="$(getopt ...)"`, gets an empty string back and exits with "Error: A command to run is required", so the game never starts. Only install the stdio hooks when the PLT we opened actually belongs to a Unity player -- either because we hooked UnityPlayer directly, or because the main executable ships a Unity data folder next to it. Co-Authored-By: Claude Opus 5 (1M context) --- src/nix/entrypoint.c | 69 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/src/nix/entrypoint.c b/src/nix/entrypoint.c index 457e11d..308444a 100644 --- a/src/nix/entrypoint.c +++ b/src/nix/entrypoint.c @@ -64,7 +64,7 @@ char_t *default_boot_config_path = NULL; FILE *fopen64_hook(char *filename, char *mode) { char *actual_file_name = filename; - if (strcmp(filename, default_boot_config_path) == 0) { + if (filename && strcmp(filename, default_boot_config_path) == 0) { actual_file_name = config.boot_config_override; LOG("Overriding boot.config to %s", actual_file_name); } @@ -76,7 +76,7 @@ FILE *fopen64_hook(char *filename, char *mode) { FILE *fopen_hook(char *filename, char *mode) { char *actual_file_name = filename; - if (strcmp(filename, default_boot_config_path) == 0) { + if (filename && strcmp(filename, default_boot_config_path) == 0) { actual_file_name = config.boot_config_override; LOG("Overriding boot.config to %s", actual_file_name); } @@ -92,6 +92,50 @@ int dup2_hook(int od, int nd) { return dup2(od, nd); } +#if defined(__APPLE__) +#define main_module_is_unity_player() TRUE +#else +static bool_t data_folder_exists(const char_t *dir, const char_t *name) { + size_t size = strlen(dir) + strlen(name) + STR_LEN(TEXT("/_Data")); + char_t *data_dir = (char_t *)malloc(size * sizeof(char_t)); + if (!data_dir) + return FALSE; + + snprintf(data_dir, size, TEXT("%s/%s_Data"), dir, name); + bool_t result = folder_exists(data_dir); + + free(data_dir); + return result; +} + +// LD_PRELOAD reaches every child, so the stdio hooks above would land in the +// launcher shells too. A player is identified by its data folder; anything we +// cannot determine keeps the hooks, as before. (macOS bundles differ -- +// unchanged there.) +static bool_t main_module_is_unity_player(void) { + char_t *exe = program_path(); + if (!exe || !*exe) { + free(exe); + return TRUE; + } + + char_t *dir = get_folder_name(exe); + // get_file_name() strips at the last dot: Game.Client needs + // Game.Client_Data. + char_t *full_name = get_file_name(exe, TRUE); + char_t *base_name = get_file_name(exe, FALSE); + + bool_t result = data_folder_exists(dir, full_name) || + data_folder_exists(dir, base_name); + + free(base_name); + free(full_name); + free(dir); + free(exe); + return result; +} +#endif + __attribute__((constructor)) void doorstop_ctor() { init_logger(); load_config(); @@ -105,6 +149,8 @@ __attribute__((constructor)) void doorstop_ctor() { void *unity_player = plthook_handle_by_name("UnityPlayer"); + bool_t hook_unity_stdio = TRUE; + if (unity_player && PLTHOOK_OPEN_BY_HANDLE_OR_ADDRESS(&hook, unity_player) == 0) { LOG("Found UnityPlayer, hooking into it instead"); @@ -114,6 +160,12 @@ __attribute__((constructor)) void doorstop_ctor() { "%s\n", plthook_error()); return; + } else { + // UnityPlayer is mapped but unhookable -- still a player. + hook_unity_stdio = + unity_player != NULL || main_module_is_unity_player(); + if (!hook_unity_stdio) + LOG("Main module is not a Unity player; skipping stdio hooks"); } if (plthook_replace(hook, "dlsym", &dlsym_hook, NULL) != 0) @@ -145,13 +197,14 @@ __attribute__((constructor)) void doorstop_ctor() { } } - if (plthook_replace(hook, "fclose", &fclose_hook, NULL) != 0) - LOG("Failed to hook fclose, ignoring it. Error: %s", - plthook_error()); + if (hook_unity_stdio) { + if (plthook_replace(hook, "fclose", &fclose_hook, NULL) != 0) + LOG("Failed to hook fclose, ignoring it. Error: %s", + plthook_error()); - if (plthook_replace(hook, "dup2", &dup2_hook, NULL) != 0) - LOG("Failed to hook dup2, ignoring it. Error: %s", - plthook_error()); + if (plthook_replace(hook, "dup2", &dup2_hook, NULL) != 0) + LOG("Failed to hook dup2, ignoring it. Error: %s", plthook_error()); + } #if defined(__APPLE__) /*