diff --git a/code/asteroid/asteroid.cpp b/code/asteroid/asteroid.cpp index 9c59ee6940c..65977ef3341 100644 --- a/code/asteroid/asteroid.cpp +++ b/code/asteroid/asteroid.cpp @@ -40,6 +40,7 @@ #include "ship/shipfx.h" #include "ship/shiphit.h" #include "stats/scoring.h" +#include "utils/Random.h" #include "weapon/weapon.h" #include @@ -1900,8 +1901,8 @@ static void asteroid_maybe_break_up(object *pasteroid_obj) roids_to_create.push_back(split->asteroid_type); } - std::random_device rd; - std::shuffle(roids_to_create.begin(), roids_to_create.end(), std::mt19937(rd())); + std::mt19937 shuffle_rng(util::Random::shuffle_seed()); + std::shuffle(roids_to_create.begin(), roids_to_create.end(), shuffle_rng); size_t total_roids = roids_to_create.size(); for (size_t i = 0; i < total_roids; i++) { diff --git a/code/cmdline/cmdline.cpp b/code/cmdline/cmdline.cpp index 1351c72f1db..1c4566b87f6 100644 --- a/code/cmdline/cmdline.cpp +++ b/code/cmdline/cmdline.cpp @@ -523,6 +523,11 @@ cmdline_parm frame_profile_write_file("-profile_write_file", NULL, AT_NONE); // cmdline_parm no_unfocused_pause_arg("-no_unfocused_pause", NULL, AT_NONE); //Cmdline_no_unfocus_pause cmdline_parm retail_time_compression_range_arg("-orig_speedx_range", NULL, AT_NONE); //Cmdline_retail_time_compression_range cmdline_parm benchmark_mode_arg("-benchmark_mode", NULL, AT_NONE); //Cmdline_benchmark_mode +cmdline_parm headless_arg("-headless", NULL, AT_NONE); +cmdline_parm unthrottled_arg("-unthrottled", NULL, AT_NONE); +cmdline_parm fixed_dt_arg("-fixed-dt", "Authoritative simulation step in seconds", AT_FLOAT); +cmdline_parm max_sim_time_arg("-max-sim-time", "End after this many simulated seconds", AT_FLOAT); +cmdline_parm sim_result_arg("-sim-result", "Write the simulation result JSON to this path", AT_STRING); cmdline_parm pilot_arg("-pilot", nullptr, AT_STRING); //Cmdline_pilot cmdline_parm noninteractive_arg("-noninteractive", NULL, AT_NONE); //Cmdline_noninteractive cmdline_parm json_profiling("-json_profiling", NULL, AT_NONE); //Cmdline_json_profiling @@ -565,6 +570,12 @@ bool Cmdline_profile_write_file = false; bool Cmdline_no_unfocus_pause = false; bool Cmdline_retail_time_compression_range = false; bool Cmdline_benchmark_mode = false; +bool Cmdline_headless = false; +bool Cmdline_unthrottled = false; +float Cmdline_fixed_dt = 1.0f / 30.0f; +float Cmdline_max_sim_time = 0.0f; +char *Cmdline_sim_result = nullptr; +bool Cmdline_sim_time_limit_reached = false; const char *Cmdline_pilot = nullptr; bool Cmdline_noninteractive = false; bool Cmdline_json_profiling = false; @@ -2378,6 +2389,57 @@ bool SetCmdlineParams() Cmdline_benchmark_mode = true; } + if (headless_arg.found()) + { + Cmdline_headless = true; + Cmdline_unthrottled = true; + Cmdline_freespace_no_sound = 1; + Cmdline_freespace_no_music = 1; + Cmdline_nomovies = 1; + // nobody can dismiss an error or warning dialog, so report and exit instead (as -standalone does) + Cmdline_noninteractive = true; + } + + if (unthrottled_arg.found()) + { + Cmdline_unthrottled = true; + } + + if (fixed_dt_arg.found()) + { + const float dt = fixed_dt_arg.get_float(); + // a step below the fixed-point resolution (1/65536 s) would round to zero; written as a positive range + // test so that NaN is rejected too + if (!(dt >= 1.0f / 65536.0f && dt <= 0.25f)) + { + Error(LOCATION, "-fixed-dt must be finite and in the range [1/65536, 0.25] seconds."); + return false; + } + Cmdline_fixed_dt = dt; + } + + if (max_sim_time_arg.found()) + { + const float max_sim_time = max_sim_time_arg.get_float(); + // mission time is 16.16 fixed point, so it cannot count past 32767 seconds + if (!(max_sim_time > 0.0f && max_sim_time <= 32767.0f)) + { + Error(LOCATION, "-max-sim-time must be finite, greater than zero and at most 32767 seconds."); + return false; + } + Cmdline_max_sim_time = max_sim_time; + } + + if (sim_result_arg.found()) + { + Cmdline_sim_result = sim_result_arg.str(); + } + + if (Cmdline_unthrottled) + { + Cmdline_NoFPSCap = 1; + } + if (pilot_arg.found()) { Cmdline_pilot = pilot_arg.str(); diff --git a/code/cmdline/cmdline.h b/code/cmdline/cmdline.h index 43a698b6d6c..979f58ef480 100644 --- a/code/cmdline/cmdline.h +++ b/code/cmdline/cmdline.h @@ -147,6 +147,12 @@ extern bool Cmdline_profile_write_file; extern bool Cmdline_no_unfocus_pause; extern bool Cmdline_retail_time_compression_range; extern bool Cmdline_benchmark_mode; +extern bool Cmdline_headless; +extern bool Cmdline_unthrottled; +extern float Cmdline_fixed_dt; +extern float Cmdline_max_sim_time; +extern char *Cmdline_sim_result; +extern bool Cmdline_sim_time_limit_reached; extern const char *Cmdline_pilot; extern bool Cmdline_noninteractive; extern bool Cmdline_json_profiling; diff --git a/code/graphics/2d.cpp b/code/graphics/2d.cpp index 82cc870dda4..2ebdc970b1c 100644 --- a/code/graphics/2d.cpp +++ b/code/graphics/2d.cpp @@ -2076,7 +2076,7 @@ bool gr_init(std::unique_ptr&& graphicsOps, GraphicsAPI if (d_mode == GraphicsAPI::Default) { d_mode = gr_get_configured_render_api(); } - } else if ( !Is_standalone ) { + } else if ( !Is_standalone && !Cmdline_headless ) { // We cannot continue without this, quit, but try to help the user out first ptr = os_config_read_string(nullptr, NOX("VideocardFs2open"), nullptr); @@ -2228,7 +2228,7 @@ bool gr_init(std::unique_ptr&& graphicsOps, GraphicsAPI } // if we are in standalone mode then just use special defaults - if (Is_standalone) { + if (Is_standalone || Cmdline_headless) { mode = GraphicsAPI::Stub; width = 640; height = 480; @@ -2334,17 +2334,19 @@ bool gr_init(std::unique_ptr&& graphicsOps, GraphicsAPI gr_is_capable(gr_capability::CAPABILITY_PERSISTENT_BUFFER_MAPPING) ? "Enabled" : "Disabled")); mprintf(("Checking mod required rendering features...\n")); - for (gr_capability ext : Required_render_ext) { - if (!gr_is_capable(ext)) { - Error(LOCATION, "Feature %s required by mod not supported by system.\n", - gr_capability_to_human_readable_string(ext).c_str()); + if (!Cmdline_headless) { + for (gr_capability ext : Required_render_ext) { + if (!gr_is_capable(ext)) { + Error(LOCATION, "Feature %s required by mod not supported by system.\n", + gr_capability_to_human_readable_string(ext).c_str()); + } } } mprintf((" All required features are supported.\n")); bool missing_installation = false; if (!running_unittests && Web_cursor == nullptr) { - if (Is_standalone) { + if (Is_standalone || Cmdline_headless) { // Cursors don't work in standalone mode, just check if the animation exists. auto handle = bm_load_animation("cursorweb"); if (handle < 0) { @@ -2372,7 +2374,7 @@ bool gr_init(std::unique_ptr&& graphicsOps, GraphicsAPI gr_set_shader(NULL); - if (!Is_standalone) { + if (!Is_standalone && !Cmdline_headless) { if (Using_in_game_options) { // The value should have been loaded into the variable already so we can use that here gr_set_gamma(Gr_gamma); @@ -3296,7 +3298,7 @@ void gr_imgui_begin_frame() ImGui::NewFrame(); } -void gr_flip(bool execute_scripting) +void gr_process_frame_callbacks(bool execute_scripting) { // Execute external "On Frame" work items executor::OnFrameExecutor->process(); @@ -3311,6 +3313,11 @@ void gr_flip(bool execute_scripting) OnFrameHook->run(); } } +} + +void gr_flip(bool execute_scripting) +{ + gr_process_frame_callbacks(execute_scripting); model_process_cached_ui_render_instances(); @@ -3569,6 +3576,13 @@ static graphics::util::GPUMemoryHeap* get_gpu_heap(GpuHeap heap_type) { void gr_heap_allocate(GpuHeap heap_type, size_t size, void* data, size_t& offset_out, gr_buffer_handle& handle_out) { TRACE_SCOPE(tracing::GpuHeapAllocate); + if (gr_screen.mode == GraphicsAPI::Stub) { + // The stub renderer has no GPU heaps. Model geometry is not rendered in + // this mode, while authoritative model and collision data remain loaded. + offset_out = 0; + handle_out = gr_buffer_handle::invalid(); + return; + } auto gpuHeap = get_gpu_heap(heap_type); @@ -3579,6 +3593,9 @@ void gr_heap_allocate(GpuHeap heap_type, size_t size, void* data, size_t& offset void gr_heap_deallocate(GpuHeap heap_type, size_t data_offset) { TRACE_SCOPE(tracing::GpuHeapDeallocate); + if (gr_screen.mode == GraphicsAPI::Stub) { + return; + } auto gpuHeap = get_gpu_heap(heap_type); diff --git a/code/graphics/2d.h b/code/graphics/2d.h index 437bc85368d..494712771e8 100644 --- a/code/graphics/2d.h +++ b/code/graphics/2d.h @@ -1160,6 +1160,9 @@ bool gr_is_screenshot_requested(); //#define gr_flip GR_CALL(gr_screen.gf_flip) void gr_flip(bool execute_scripting = true); +// Run per-frame scripting work without performing presentation. Headless +// simulation calls this once after each authoritative gameplay frame. +void gr_process_frame_callbacks(bool execute_scripting = true); inline void gr_setup_frame() { gr_screen.gf_setup_frame(); diff --git a/code/io/timer.cpp b/code/io/timer.cpp index d3bec6523af..e07ef39d0c2 100644 --- a/code/io/timer.cpp +++ b/code/io/timer.cpp @@ -39,6 +39,10 @@ static bool Timestamp_sudo_paused = false; static uint64_t Timestamp_microseconds_at_mission_start = 0; +static bool Simulation_clock_enabled = false; +static uint64_t Simulation_clock_microseconds = 0; +static uint64_t Simulation_clock_fraction = 0; + static uint64_t timestamp_get_raw(bool start_frame = false); @@ -137,6 +141,26 @@ std::uint64_t timer_get_nanoseconds() return static_cast(time * Timer_to_nanoseconds); } +void timer_set_simulation_clock_enabled(bool enabled) +{ + Simulation_clock_enabled = enabled; + // Start well past zero, as a real session's clock is by the time a mission starts: TIMESTAMP values 0 and 1 + // mean "never" and "immediate", and cooldowns measured from a zero-initialised time assume a running clock. + Simulation_clock_microseconds = 3600 * MICROSECONDS_PER_SECOND; + Simulation_clock_fraction = 0; +} + +void timer_advance_simulation_clock(fix delta) +{ + if (!Simulation_clock_enabled || delta <= 0) { + return; + } + + const auto fixed_microseconds = static_cast(delta) * MICROSECONDS_PER_SECOND + Simulation_clock_fraction; + Simulation_clock_microseconds += fixed_microseconds / F1_0; + Simulation_clock_fraction = fixed_microseconds % F1_0; +} + static uint64_t timestamp_get_raw(bool start_frame) { static uint64_t timestamp_raw = 0; @@ -158,6 +182,10 @@ static uint64_t timestamp_get_raw(bool start_frame) static uint64_t timestamp_get_microseconds() { + if (Simulation_clock_enabled) { + return Simulation_clock_microseconds; + } + return Timestamp_time_compression_microseconds_offset + static_cast(timestamp_get_raw() * Timer_to_microseconds * Timestamp_time_compression_multiplier); } diff --git a/code/io/timer.h b/code/io/timer.h index 0e46c02f7ff..7ff23fa8640 100644 --- a/code/io/timer.h +++ b/code/io/timer.h @@ -109,6 +109,12 @@ extern std::uint64_t timer_get_nanoseconds(); extern int timer_get_seconds(); // seconds since program started... not accurate, but good for long // runtimes with second-based timeouts +// Optional authoritative mission clock for unthrottled experiment runs. Wall-clock +// timer APIs remain unchanged so profiling and UI deadlines keep real time. Like the +// real clock it only moves forward, across missions too. +void timer_set_simulation_clock_enabled(bool enabled); +void timer_advance_simulation_clock(fix delta); + //================================================================= //================================================================= diff --git a/code/parse/sexp.cpp b/code/parse/sexp.cpp index 2ac42ba0db2..b047c495565 100644 --- a/code/parse/sexp.cpp +++ b/code/parse/sexp.cpp @@ -15506,8 +15506,8 @@ void sexp_send_builtin_message(int n) { } if (shuffle) { - std::random_device rd; - std::shuffle(sender_names.begin(), sender_names.end(), std::mt19937(rd())); + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(sender_names.begin(), sender_names.end(), rng); } auto subject = have_subject ? subject_entry->shipp() : nullptr; diff --git a/code/popup/popup.cpp b/code/popup/popup.cpp index 68b61edb5ad..31ba06a7663 100644 --- a/code/popup/popup.cpp +++ b/code/popup/popup.cpp @@ -12,6 +12,7 @@ #include #include "anim/animplay.h" +#include "cmdline/cmdline.h" #include "freespace.h" #include "gamesequence/gamesequence.h" #include "gamesnd/gamesnd.h" @@ -1148,7 +1149,17 @@ int popup(int flags, int nchoices, ... ) vsnprintf(Popup_info.raw_text, sizeof(Popup_info.raw_text), format, args); va_end(args); Popup_info.raw_text[sizeof(Popup_info.raw_text)-1] = '\0'; - + + // A headless run has nobody to answer, so log the popup and give the abort answer, unless a script answers dialogs. + if (Cmdline_headless && !scripting::hooks::OnDialogFrame->isActive()) { + mprintf(("Headless run: skipped popup \"%s\"\n", Popup_info.raw_text)); + for (i = 0; i < nchoices; i++) { + vm_free(Popup_info.button_text[i]); + Popup_info.button_text[i] = nullptr; + } + return -1; + } + gamesnd_play_iface(InterfaceSounds::POPUP_APPEAR); // play sound when popup appears io::mouse::CursorManager::get()->pushStatus(); diff --git a/code/radar/radarsetup.cpp b/code/radar/radarsetup.cpp index 2bf2f2fa0a3..bc5144db359 100644 --- a/code/radar/radarsetup.cpp +++ b/code/radar/radarsetup.cpp @@ -10,6 +10,7 @@ #include "bmpman/bmpman.h" +#include "cmdline/cmdline.h" #include "debugconsole/console.h" #include "freespace.h" #include "gamesnd/gamesnd.h" @@ -174,6 +175,10 @@ void radar_stuff_blip_info(object *objp, int is_bright, color **blip_color, int void radar_plot_object( object *objp ) { + if (Cmdline_headless) { + return; + } + vec3d pos, tempv; float awacs_level, dist, max_radar_dist; vec3d world_pos = objp->pos; diff --git a/code/scripting/global_hooks.cpp b/code/scripting/global_hooks.cpp index 2a6ee23bf9f..e98912b26be 100644 --- a/code/scripting/global_hooks.cpp +++ b/code/scripting/global_hooks.cpp @@ -559,6 +559,24 @@ const std::shared_ptr> OnCameraSetUpHook = Hook<>::Factory("On Camera Set {"Camera", "camera", "The camera about to be used for rendering."}, }); +const std::shared_ptr> OnShipDamage = Hook::Factory( + "On Ship Damage", "Called after resolved damage is applied; lethal hits run before the destruction callback.", + { + {"Target", "ship", "The ship that took the damage."}, + {"Source", "object", "The weapon, beam, shockwave, or other object that caused the damage. Could be nil."}, + {"Attacker", "object", "The source ship or parent object, if available."}, + {"WeaponClass", "string", "The weapon class name, or NON_WEAPON if the source has no weapon class."}, + {"WeaponSubtype", "string", "The weapon subtype, or UNKNOWN if the source has no weapon class."}, + {"SourceKind", "string", "PROJECTILE, BEAM, SHOCKWAVE, COLLISION, or OTHER."}, + {"DamageType", "string", "The resolved damage type, or UNKNOWN if none is defined."}, + {"ShieldDamage", "number", "Actual total shield strength removed by this damage application."}, + {"HullDamage", "number", "Actual hull strength removed, clamped to the target's remaining hull."}, + {"Quadrant", "number", "Shield quadrant hit, or -1 when no single quadrant applies."}, + {"Ballistic", "boolean", "Whether the weapon class is marked ballistic."}, + {"BeamWeapon", "boolean", "Whether the weapon class is configured as a beam."}, + {"LethalHit", "boolean", "Whether this hit reduced positive hull strength to zero or below."}, + }); + // ========== FRED HOOKS ========== const std::shared_ptr> FredOnMissionLoad = Hook<>::Factory("FRED On Mission Load", diff --git a/code/scripting/global_hooks.h b/code/scripting/global_hooks.h index 27e3d94292a..1c38571c2bf 100644 --- a/code/scripting/global_hooks.h +++ b/code/scripting/global_hooks.h @@ -100,6 +100,8 @@ extern const std::shared_ptr> OnStateAboutToEndHook; extern const std::shared_ptr> OnStateEndHook; extern const std::shared_ptr> OnCameraSetUpHook; +extern const std::shared_ptr> OnShipDamage; + // FRED Hooks extern const std::shared_ptr> FredOnMissionLoad; extern const std::shared_ptr> FredOnMissionSpecsSave; diff --git a/code/scripting/hook_conditions.cpp b/code/scripting/hook_conditions.cpp index 9fb0c79d915..3a6b7ee6b1f 100644 --- a/code/scripting/hook_conditions.cpp +++ b/code/scripting/hook_conditions.cpp @@ -232,6 +232,10 @@ HOOK_CONDITIONS_START(ShipDeathConditions) HOOK_CONDITION_SHIPP(ShipDeathConditions, "", "that died.", dying_shipp); HOOK_CONDITIONS_END +HOOK_CONDITIONS_START(ShipDamageConditions) + HOOK_CONDITION_SHIPP(ShipDamageConditions, "Target ", "that took resolved damage.", target_shipp); +HOOK_CONDITIONS_END + HOOK_CONDITIONS_START(SubsystemDeathConditions) HOOK_CONDITION_SHIPP(SubsystemDeathConditions, "", "whose subsystem got destroyed.", affected_shipp); HOOK_CONDITIONS_END diff --git a/code/scripting/hook_conditions.h b/code/scripting/hook_conditions.h index 28d49fe7734..79d0a9b0199 100644 --- a/code/scripting/hook_conditions.h +++ b/code/scripting/hook_conditions.h @@ -63,6 +63,11 @@ struct ShipDeathConditions { const ship* dying_shipp; }; +struct ShipDamageConditions { + HOOK_DEFINE_CONDITIONS; + const ship* target_shipp; +}; + struct SubsystemDeathConditions { HOOK_DEFINE_CONDITIONS; const ship* affected_shipp; diff --git a/code/ship/afterburner.cpp b/code/ship/afterburner.cpp index 2860e61769f..a808a3bc89f 100644 --- a/code/ship/afterburner.cpp +++ b/code/ship/afterburner.cpp @@ -9,6 +9,7 @@ +#include "cmdline/cmdline.h" #include "freespace.h" #include "gamesnd/gamesnd.h" #include "hud/hudets.h" @@ -44,6 +45,13 @@ static int Player_afterburner_start_time; #define DISENGAGE_TIME 1500 // time in ms to play faded loop sound when afterburner disengages +// Cooldowns run on the real clock, which an unthrottled run's simulated time does not follow; there they use +// the mission clock instead, or they would last longer the faster the machine simulates. +static int afterburner_clock_ms() +{ + return Cmdline_unthrottled ? timestamp() : timer_get_milliseconds(); +} + /** * Call at the start of a mission @@ -88,7 +96,7 @@ void afterburners_start(object *objp) shipp->flags.set(Ship::Ship_Flags::Attempting_to_afterburn); - int now = timer_get_milliseconds(); + int now = afterburner_clock_ms(); if (now - shipp->afterburner_last_end_time < (int)(sip->afterburner_cooldown_time * 1000.0f)) { @@ -143,7 +151,7 @@ void afterburners_start(object *objp) Ship_info[shipp->ship_info_index].animations.getAll(model_get_instance(shipp->model_instance_num), animation::ModelAnimationTriggerType::Afterburner).start(animation::ModelAnimationDirection::FWD); if ( objp == Player_obj ) { - Player_afterburner_start_time = timer_get_milliseconds(); + Player_afterburner_start_time = afterburner_clock_ms(); Player_disengage_timer = 1; Player_afterburner_vol = AFTERBURNER_DEFAULT_VOL; @@ -330,7 +338,7 @@ void afterburners_stop(object *objp, int key_released) return; } - shipp->afterburner_last_end_time = timer_get_milliseconds(); + shipp->afterburner_last_end_time = afterburner_clock_ms(); if (scripting::hooks::OnAfterburnerEnd->isActive()) { scripting::hooks::OnAfterburnerEnd->run(scripting::hooks::ShipSourceConditions{ shipp }, diff --git a/code/ship/ship.cpp b/code/ship/ship.cpp index 21f148802b5..61efc191c57 100644 --- a/code/ship/ship.cpp +++ b/code/ship/ship.cpp @@ -11158,8 +11158,8 @@ static void ship_set_default_weapons(ship *shipp, ship_info *sip) for (int fp = 0; fp < pm->gun_banks[i].num_slots; fp++) { fpi.push_back(fp); } - std::random_device rd; - std::shuffle(fpi.begin(), fpi.end(), std::mt19937(rd())); + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(fpi.begin(), fpi.end(), rng); } swp->num_secondary_banks = sip->num_secondary_banks; @@ -11745,8 +11745,8 @@ static void ship_model_change(int n, int ship_type) for (int fp = 0; fp < pm->gun_banks[bank_i].num_slots; fp++) { fpi.push_back(fp); } - std::random_device rd; - std::shuffle(fpi.begin(), fpi.end(), std::mt19937(rd())); + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(fpi.begin(), fpi.end(), rng); } model_delete_instance(sp->model_instance_num); @@ -13496,8 +13496,8 @@ int ship_fire_primary(object * obj, int force, bool rollback_shot) for (int fp = 0; fp < num_slots; fp++) { fpi.push_back(fp); } - std::random_device rd; - std::shuffle(fpi.begin(), fpi.end(), std::mt19937(rd())); + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(fpi.begin(), fpi.end(), rng); } // The energy-consumption code executes even for ballistic primaries, because @@ -13783,26 +13783,26 @@ int ship_fire_primary(object * obj, int force, bool rollback_shot) switch (firing_pattern) { case FiringPattern::RANDOM_EXHAUSTIVE: { if (num_slots < (swp->primary_firepoint_next_to_fire_index[bank_to_fire] + point_count)) { - std::random_device rd; - std::shuffle(&swp->primary_firepoint_indices[bank_to_fire][0], &swp->primary_firepoint_indices[bank_to_fire][swp->primary_firepoint_next_to_fire_index[bank_to_fire]-1], std::mt19937(rd())); //NOLINT + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(&swp->primary_firepoint_indices[bank_to_fire][0], &swp->primary_firepoint_indices[bank_to_fire][swp->primary_firepoint_next_to_fire_index[bank_to_fire]-1], rng); //NOLINT } else if (swp->primary_firepoint_next_to_fire_index[bank_to_fire] < point_count) { - std::random_device rd; - std::shuffle(swp->primary_firepoint_indices[bank_to_fire].begin(), swp->primary_firepoint_indices[bank_to_fire].end(), std::mt19937(rd())); + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(swp->primary_firepoint_indices[bank_to_fire].begin(), swp->primary_firepoint_indices[bank_to_fire].end(), rng); } break; } case FiringPattern::RANDOM_NONREPEATING: { int shuffle_start = MIN(point_count, num_slots - point_count); - std::random_device rd; + std::mt19937 rng(util::Random::shuffle_seed()); auto middle_iterator = swp->primary_firepoint_indices[bank_to_fire].begin(); std::advance(middle_iterator, shuffle_start); - std::shuffle(middle_iterator, swp->primary_firepoint_indices[bank_to_fire].end(), std::mt19937(rd())); + std::shuffle(middle_iterator, swp->primary_firepoint_indices[bank_to_fire].end(), rng); std::rotate(swp->primary_firepoint_indices[bank_to_fire].begin(), middle_iterator, swp->primary_firepoint_indices[bank_to_fire].end()); break; } case FiringPattern::RANDOM_REPEATING: { - std::random_device rd; - std::shuffle(swp->primary_firepoint_indices[bank_to_fire].begin(), swp->primary_firepoint_indices[bank_to_fire].end(), std::mt19937(rd())); + std::mt19937 rng(util::Random::shuffle_seed()); + std::shuffle(swp->primary_firepoint_indices[bank_to_fire].begin(), swp->primary_firepoint_indices[bank_to_fire].end(), rng); break; } default: { diff --git a/code/ship/shiphit.cpp b/code/ship/shiphit.cpp index 7052850417e..a965a9cf8a1 100755 --- a/code/ship/shiphit.cpp +++ b/code/ship/shiphit.cpp @@ -2426,6 +2426,16 @@ static void ship_do_damage(object *ship_objp, object *other_obj, const vec3d *hi Assertion(ship_objp->instance >= 0 && ship_objp->type == OBJ_SHIP, "invalid ship target in ship_do_damage"); shipp = &Ships[ship_objp->instance]; + const bool damage_hook_active = scripting::hooks::OnShipDamage->isActive(); + bool damage_hook_emitted = false; + float hull_strength_before_damage = 0.0f; + float shield_strength_before_damage = 0.0f; + if (damage_hook_active) { + hull_strength_before_damage = ship_objp->hull_strength; + for (float shield_strength : ship_objp->shield_quadrant) { + shield_strength_before_damage += shield_strength; + } + } // maybe adjust damage done by shockwave for BIG|HUGE maybe_shockwave_damage_adjust(ship_objp, other_obj, &damage); @@ -2559,6 +2569,81 @@ static void ship_do_damage(object *ship_objp, object *other_obj, const vec3d *hi } int weapon_info_index = shiphit_get_damage_weapon(other_obj); + auto emit_damage_hook = [&]() { + if (!damage_hook_active || damage_hook_emitted) { + return; + } + + float shield_strength_after_damage = 0.0f; + for (float shield_strength : ship_objp->shield_quadrant) { + shield_strength_after_damage += shield_strength; + } + const float shield_damage = MAX(0.0f, shield_strength_before_damage - shield_strength_after_damage); + const float hull_damage = MAX(0.0f, MAX(0.0f, hull_strength_before_damage) - MAX(0.0f, ship_objp->hull_strength)); + if (shield_damage <= 0.0f && hull_damage <= 0.0f) { + return; + } + const bool lethal_hit = hull_strength_before_damage > 0.0f && ship_objp->hull_strength <= 0.0f; + + const weapon_info* wip = (weapon_info_index >= 0 && weapon_info_index < weapon_info_size()) + ? &Weapon_info[weapon_info_index] + : nullptr; + const char* weapon_class = wip != nullptr ? wip->name : "NON_WEAPON"; + const char* weapon_subtype = wip != nullptr && wip->subtype >= 0 && wip->subtype < Num_weapon_subtypes + ? Weapon_subtype_names[wip->subtype] + : "UNKNOWN"; + const bool ballistic = wip != nullptr && wip->wi_flags[Weapon::Info_Flags::Ballistic]; + const bool beam_weapon = wip != nullptr && wip->is_beam(); + + const char* source_kind = "OTHER"; + if (other_obj_is_weapon) { + source_kind = "PROJECTILE"; + } else if (other_obj_is_beam) { + source_kind = "BEAM"; + } else if (other_obj_is_shockwave) { + source_kind = "SHOCKWAVE"; + } else if (other_obj != nullptr && (other_obj->type == OBJ_SHIP || other_obj->type == OBJ_ASTEROID || other_obj->type == OBJ_DEBRIS)) { + source_kind = "COLLISION"; + } + + int resolved_damage_type = damage_type_idx; + if ((resolved_damage_type < 0 || resolved_damage_type >= static_cast(Damage_types.size())) && wip != nullptr) { + resolved_damage_type = wip->damage_type_idx; + } + const char* damage_type = resolved_damage_type >= 0 && resolved_damage_type < static_cast(Damage_types.size()) + ? Damage_types[resolved_damage_type].name + : "UNKNOWN"; + + object* attacker_objp = nullptr; + if (other_obj_is_beam) { + const int parent_objnum = beam_get_parent(other_obj); + if (parent_objnum >= 0 && parent_objnum < MAX_OBJECTS) { + attacker_objp = &Objects[parent_objnum]; + } + } else if (other_obj != nullptr && other_obj->type == OBJ_SHIP) { + attacker_objp = other_obj; + } else if (other_obj != nullptr && other_obj->parent >= 0 && other_obj->parent < MAX_OBJECTS && + Objects[other_obj->parent].signature == other_obj->parent_sig) { + attacker_objp = &Objects[other_obj->parent]; + } + + damage_hook_emitted = true; + scripting::hooks::OnShipDamage->run(scripting::hooks::ShipDamageConditions{ shipp }, + scripting::hook_param_list( + scripting::hook_param("Target", 'o', ship_objp), + scripting::hook_param("Source", 'o', other_obj), + scripting::hook_param("Attacker", 'o', attacker_objp, attacker_objp != nullptr), + scripting::hook_param("WeaponClass", 's', weapon_class), + scripting::hook_param("WeaponSubtype", 's', weapon_subtype), + scripting::hook_param("SourceKind", 's', source_kind), + scripting::hook_param("DamageType", 's', damage_type), + scripting::hook_param("ShieldDamage", 'f', shield_damage), + scripting::hook_param("HullDamage", 'f', hull_damage), + scripting::hook_param("Quadrant", 'i', quadrant), + scripting::hook_param("Ballistic", 'b', ballistic), + scripting::hook_param("BeamWeapon", 'b', beam_weapon), + scripting::hook_param("LethalHit", 'b', lethal_hit))); + }; // If we hit the shield, reduce it's strength and found // out how much damage is left over. @@ -2697,10 +2782,14 @@ static void ship_do_damage(object *ship_objp, object *other_obj, const vec3d *hi // diag_printf2("Simulated Hull for Ship %s hit, dropping from %.32f to %d.\n", shipp->ship_name, (int) ( ship_objp->sim_hull_strength * 100 ), (int) ( ( ship_objp->sim_hull_strength - damage ) * 100 ) ); ship_objp->sim_hull_strength -= damage; ship_objp->sim_hull_strength = MAX(0.0f, ship_objp->sim_hull_strength); + emit_damage_hook(); return; } } ship_objp->hull_strength -= damage; + if (ship_objp->hull_strength <= 0.0f) { + emit_damage_hook(); + } } // let damage gauge know that player ship just took damage @@ -2799,6 +2888,7 @@ static void ship_do_damage(object *ship_objp, object *other_obj, const vec3d *hi } } } + emit_damage_hook(); if (!global_damage) { maybe_play_conditional_impacts(impact_data, other_obj, ship_objp, true, submodel_num, hitpos, local_hitpos, hit_normal); diff --git a/code/utils/Random.cpp b/code/utils/Random.cpp index 2d944d281c6..9b9144f5b7c 100644 --- a/code/utils/Random.cpp +++ b/code/utils/Random.cpp @@ -8,6 +8,8 @@ #include "globalincs/pstypes.h" +extern bool Cmdline_reuse_rng_seed; + namespace util { thread_local std::mt19937 seeder {std::random_device()()}; @@ -54,6 +56,22 @@ void Random::seed(unsigned int val) SCP_rng.seed(val); } +void Random::seed_ranges(unsigned int val) +{ + Assert(val > 0); + seeder.seed(val); +} + +unsigned int Random::shuffle_seed() +{ + if (Cmdline_reuse_rng_seed) { + return static_cast(next()); + } + + std::random_device rd; + return rd(); +} + int Random::next() { return SCP_rng.next(); diff --git a/code/utils/Random.h b/code/utils/Random.h index 0ebb61655a6..03a32b534c6 100644 --- a/code/utils/Random.h +++ b/code/utils/Random.h @@ -10,6 +10,12 @@ class Random { // seed(1) will result in RNG using its default seed static void seed(unsigned int val); + // Seed independent RandomRange instances created after this call. + static void seed_ranges(unsigned int val); + + // Seed for a one-off generator handed to std::shuffle. With -seed it is drawn from the main + // RNG, so shuffles repeat from run to run; otherwise it comes from the OS, as before. + static unsigned int shuffle_seed(); // return a value in [0, MAX_VALUE] static int next(); // uniformly between 0 and MAX_VALUE diff --git a/freespace2/freespace.cpp b/freespace2/freespace.cpp index 46d0850fe21..c526ac474b2 100644 --- a/freespace2/freespace.cpp +++ b/freespace2/freespace.cpp @@ -63,6 +63,7 @@ #include "gamesnd/eventmusic.h" #include "gamesnd/gamesnd.h" #include "graphics/debug_sphere.h" +#include "graphics/2d.h" #include "graphics/font.h" #include "graphics/light.h" #include "graphics/matrix.h" @@ -210,6 +211,7 @@ #include #include +#include #include "imgui.h" @@ -353,6 +355,8 @@ int Show_area_effect = 0; object *Last_view_target = nullptr; int dogfight_blown = 0; +static uint64_t Sim_run_wall_start_us = 0; +static bool Sim_run_start_failed = false; int frame_int = -1; float frametimes[FRAME_FILTER]; @@ -1882,8 +1886,15 @@ void game_init() Game_current_mission_filename[0] = 0; - // seed the random number generator - Random::seed(static_cast(time(nullptr))); + // Seed both game RNG families from the command line when reproducible runs + // are requested. RandomRange instances must receive their seed before the + // tables are parsed during game initialization. + if (Cmdline_reuse_rng_seed) { + Random::seed(Cmdline_rng_seed); + Random::seed_ranges(Cmdline_rng_seed); + } else { + Random::seed(static_cast(time(nullptr))); + } ImGui::CreateContext(); @@ -1898,6 +1909,7 @@ void game_init() // Initialize the timer before the os timer_init(); + timer_set_simulation_clock_enabled(Cmdline_unthrottled); if (LoggingEnabled) { outwnd_init(); @@ -2006,11 +2018,11 @@ void game_init() // SOUND INIT START ///////////////////////////// - if ( !Is_standalone ) { + if ( !Is_standalone && !Cmdline_headless ) { snd_init(); } - if(!fsspeech_init()) { + if (!Cmdline_headless && !fsspeech_init()) { mprintf(("Failed to init speech\n")); } @@ -2019,7 +2031,7 @@ void game_init() ///////////////////////////// std::unique_ptr sdlGraphicsOperations; - if (!Is_standalone) { + if (!Is_standalone && !Cmdline_headless) { // Standalone mode doesn't require graphics operations sdlGraphicsOperations.reset(new SDLGraphicsOperations()); } @@ -2139,7 +2151,7 @@ void game_init() log_string(LOGFILE_EVENT_LOG,"FS2_Open Mission Log - Opened \n\n", 1); // standalone's don't use the joystick and it seems to sometimes cause them to not get shutdown properly - if(!Is_standalone){ + if(!Is_standalone && !Cmdline_headless){ io::joystick::init(); } @@ -2203,7 +2215,7 @@ void game_init() pilot_load_pic_list(); pilot_load_squad_pic_list(); - if (!Is_standalone) { + if (!Is_standalone && !Cmdline_headless) { // Load the default cursor and enable it io::mouse::Cursor* cursor = io::mouse::CursorManager::get()->loadCursor("cursor", true); if (cursor) { @@ -2248,7 +2260,7 @@ void game_init() // convert old pilot files (if they need it) convert_pilot_files(); - if ( !Is_standalone ) { + if ( !Is_standalone && !Cmdline_headless ) { #ifdef WITH_FFMPEG libs::ffmpeg::initialize(); #endif @@ -3764,22 +3776,19 @@ void game_simulation_frame() { TRACE_SCOPE(tracing::Simulation); - //Do camera stuff - //This is for the warpout cam - if ( Player->control_mode != PCM_NORMAL ) - Warp_camera.do_frame(flFrametime); + if (!Cmdline_headless) { + // Camera and photo-mode interpolation are presentation-only. + if (Player->control_mode != PCM_NORMAL) + Warp_camera.do_frame(flFrametime); - //Do ingame cutscenes stuff - if(!Time_compression_locked) - { - cam_do_frame(flFrametime); - } - else - { - cam_do_frame(flRealframetime); - } + if (!Time_compression_locked) { + cam_do_frame(flFrametime); + } else { + cam_do_frame(flRealframetime); + } - photo_mode_do_frame(flRealframetime); + photo_mode_do_frame(flRealframetime); + } // blow ships up in multiplayer dogfight if( MULTIPLAYER_MASTER && (Net_player != nullptr) && (Netgame.type_flags & NG_TYPE_DOGFIGHT) && (f2fl(Missiontime) >= 2.0f) && !dogfight_blown){ @@ -4363,14 +4372,14 @@ void game_frame(bool paused) } // Note: These are done even before the player enters, else buffers can overflow. - if (! (Game_mode & GM_STANDALONE_SERVER)){ + if (! (Game_mode & GM_STANDALONE_SERVER) && !Cmdline_headless){ radar_frame_init(); } shield_frame_init(); if ( !Pre_player_entry && actually_playing ) { - if (! (Game_mode & GM_STANDALONE_SERVER) ) { + if (! (Game_mode & GM_STANDALONE_SERVER) && !Cmdline_headless ) { if( (!popup_running_state()) && (!popupdead_is_active()) ){ game_process_keys(); @@ -4404,9 +4413,16 @@ void game_frame(bool paused) return; } + if (Cmdline_headless && !Pre_player_entry && !(Game_mode & GM_STANDALONE_SERVER)) { + // gr_flip normally dispatches these callbacks after simulation. Mods + // commonly run authoritative game logic (AI, economy, objectives) from + // OnFrame, so preserve that work while omitting presentation in headless runs. + gr_process_frame_callbacks(); + } + } - if (!Pre_player_entry) { + if (!Pre_player_entry && !Cmdline_headless) { if (! (Game_mode & GM_STANDALONE_SERVER)) { DEBUG_GET_TIME( clear_time1 ) if (!openxr_enabled()) { @@ -4561,6 +4577,42 @@ void game_set_frametime(int state) fix thistime; float frame_cap_diff; bool do_pre_player_skip = false; + if (Cmdline_unthrottled && state == GS_STATE_GAME_PLAY) { + // Keep the engine's normal authoritative frame/update path, but make its + // clock independent of wall time. Gameplay timestamps advance once after + // each simulation frame in game_do_frame(). + timer_start_frame(); + + const fix real_frametime = Pre_player_entry ? (F1_0 / 4) : fl2f(Cmdline_fixed_dt); + + if (Game_time_compression != Desired_time_compression) { + if (Time_compression_change_rate == 0) { + Game_time_compression = Desired_time_compression; + } else { + const bool ascending = Desired_time_compression > Game_time_compression; + Game_time_compression += fixmul(Time_compression_change_rate, real_frametime); + if ((ascending && Game_time_compression > Desired_time_compression) + || (!ascending && Game_time_compression < Desired_time_compression)) { + Game_time_compression = Desired_time_compression; + } + } + } + + Frametime = fixmul(real_frametime, Game_time_compression); + if (Frametime <= 0) { + Frametime = 1; + } + flRealframetime = f2fl(real_frametime); + flFrametime = f2fl(Frametime); + Last_time = timer_get_fixed_seconds(); + Last_frame_timestamp = _timestamp(); + + if (FrametimeOverall > (INT_MAX - Frametime)) { + FrametimeOverall = 0; + } + FrametimeOverall += Frametime; + return; + } // sync all timestamps across the entire frame timer_start_frame(); @@ -4709,6 +4761,15 @@ void game_do_frame(bool set_frametime) last_single_step = game_single_step; game_frame(); + + if (Cmdline_unthrottled) { + timer_advance_simulation_clock(Frametime); + } + + if (Cmdline_max_sim_time > 0.0f && timestamp_get_mission_time() >= fl2f(Cmdline_max_sim_time)) { + Cmdline_sim_time_limit_reached = true; + gameseq_post_event(GS_EVENT_QUIT_GAME); + } } void multi_maybe_do_frame() @@ -6003,9 +6064,10 @@ void game_enter_state( int old_state, int new_state ) mission_campaign_maybe_play_movie(CAMPAIGN_MOVIE_PRE_MISSION); // determine where to go next - if (mission_has_fiction()) { + // a headless run has nobody to read the fiction or command briefing + if (!Cmdline_headless && mission_has_fiction()) { gameseq_post_event(GS_EVENT_FICTION_VIEWER); - } else if (mission_has_cmd_brief()) { + } else if (!Cmdline_headless && mission_has_cmd_brief()) { gameseq_post_event(GS_EVENT_CMD_BRIEF); } else if (red_alert_mission()) { gameseq_post_event(GS_EVENT_RED_ALERT); @@ -6148,13 +6210,13 @@ void game_enter_state( int old_state, int new_state ) // Goober5000 - people may not have realized that pausing causes this state to be re-entered if ((old_state != GS_STATE_GAME_PAUSED) && (old_state != GS_STATE_MULTI_PAUSED) && (old_state != GS_STATE_MAIN_MENU)) { - if ( !Is_standalone ) + if ( !Is_standalone && !Cmdline_headless ) radar_mission_init(); //Set the current hud set_current_hud(); - if ( !Is_standalone ) { + if ( !Is_standalone && !Cmdline_headless ) { ship_init_cockpit_displays(Player_ship); } } @@ -6606,11 +6668,27 @@ void game_do_state(int state) case GS_STATE_BRIEFING: game_set_frametime(GS_STATE_BRIEFING); + if (Cmdline_headless && !(Game_mode & GM_MULTIPLAYER)) { + // Same as the engine does for missions flagged no-briefing: take the default loadout and enter. + const auto commit_status = commit_pressed(true); + if (commit_status != commit_pressed_status::SUCCESS) { + Warning(LOCATION, "Headless run could not commit the default loadout (status %d); ending the run.", + static_cast(commit_status)); + Sim_run_start_failed = true; + gameseq_post_event(GS_EVENT_QUIT_GAME); + } + break; + } brief_do_frame(flFrametime); break; case GS_STATE_DEBRIEF: game_set_frametime(GS_STATE_DEBRIEF); + if (Cmdline_headless && !(Game_mode & GM_MULTIPLAYER)) { + // The mission is over and nobody will dismiss the debriefing; the run ends with its result. + gameseq_post_event(GS_EVENT_QUIT_GAME); + break; + } debrief_do_frame(flFrametime); break; @@ -6689,6 +6767,11 @@ void game_do_state(int state) case GS_STATE_RED_ALERT: game_set_frametime(GS_STATE_RED_ALERT); + if (Cmdline_headless && !(Game_mode & GM_MULTIPLAYER)) { + // What the screen's Continue button does in single player: enter the mission. + gameseq_post_event(GS_EVENT_ENTER_GAME); + break; + } red_alert_do_frame(flFrametime); break; @@ -7020,7 +7103,7 @@ int game_main(int argc, char *argv[]) return 1; } - if (!Is_standalone && !headtracking::init()) + if (!Is_standalone && !Cmdline_headless && !headtracking::init()) { mprintf(("Headtracking is not enabled...\n")); } @@ -7053,11 +7136,12 @@ int game_main(int argc, char *argv[]) scripting::hooks::OnIntroAboutToPlay->run(); } - if (!Is_standalone && !skip_intro) { + if (!Is_standalone && !Cmdline_headless && !skip_intro) { movie::play("intro.mve"); } game_start_time(); + Sim_run_wall_start_us = timer_get_microseconds(); if (Is_standalone) { gameseq_post_event(GS_EVENT_STANDALONE_MAIN); @@ -7091,6 +7175,29 @@ int game_main(int argc, char *argv[]) // void game_shutdown(void) { + if (Cmdline_sim_result != nullptr) { + const auto sim_time = f2fl(timestamp_get_mission_time()); + const auto wall_time = Sim_run_wall_start_us > 0 + ? (timer_get_microseconds() - Sim_run_wall_start_us) / static_cast(MICROSECONDS_PER_SECOND) + : 0.0; + json_t* result = json_pack("{s:i,s:s,s:i,s:f,s:f,s:f,s:b,s:s}", + "schema", 1, + "mission", Game_current_mission_filename, + "seed", static_cast(Cmdline_rng_seed), + "fixed_dt", Cmdline_fixed_dt, + "max_sim_time", Cmdline_max_sim_time, + "simulated_seconds", sim_time, + "time_limit_reached", Cmdline_sim_time_limit_reached ? 1 : 0, + "termination", Cmdline_sim_time_limit_reached ? "time_limit" : Sim_run_start_failed ? "start_failed" : "mission_exit"); + if (result != nullptr) { + json_object_set_new(result, "wall_seconds", json_real(wall_time)); + if (json_dump_file(result, Cmdline_sim_result, JSON_INDENT(2) | JSON_SORT_KEYS) != 0) { + Warning(LOCATION, "Failed to write simulation result to '%s'.", Cmdline_sim_result); + } + json_decref(result); + } + } + events::EngineShutdown(); headtracking::shutdown(); @@ -7104,7 +7211,7 @@ void game_shutdown(void) #endif // don't ever flip a page on the standalone! - if(!(Game_mode & GM_STANDALONE_SERVER)){ + if(!(Game_mode & GM_STANDALONE_SERVER) && !Cmdline_headless){ gr_reset_clip(); gr_clear(); gr_flip(); @@ -7917,7 +8024,7 @@ void game_title_screen_display() { // anything that shares code between standalone and normal cannot make // gr_* calls in standalone mode because all gr_ calls are NULL pointers - if (Is_standalone) { + if (Is_standalone || Cmdline_headless) { return; } @@ -7996,7 +8103,7 @@ void game_title_screen_close() { // anything that shares code between standalone and normal cannot make // gr_* calls in standalone mode because all gr_ calls are NULL pointers - if (Is_standalone) { + if (Is_standalone || Cmdline_headless) { return; } diff --git a/tools/headless_batch/README.md b/tools/headless_batch/README.md new file mode 100644 index 00000000000..56c970ef57c --- /dev/null +++ b/tools/headless_batch/README.md @@ -0,0 +1,166 @@ +# Headless batch runner + +Run many FreeSpace Open missions as fast as the CPU allows, with no window, no rendering and no audio, then +collect one result per run. Each run is a separate engine process with its own workspace, seed and UDP port, +so runs share no in-process state and the batch scales with the number of cores. + +This directory has the runner (`run_batch.py`), its tests (`test_run_batch.py`) and a starting plan +(`example.plan.json`). The engine side of the feature is the set of `-headless` command-line options described +below. + +## The engine options + +| Option | Effect | +|---|---| +| `-headless` | No window, renderer, audio, speech, joystick, HUD/radar presentation or movies. Graphics runs on the stub backend. Implies `-unthrottled`, `-nomovies` and `-noninteractive` (warnings are only logged; an error or a Lua error ends the process instead of opening a dialog). | +| `-unthrottled` | Run frames as fast as possible. Mission time comes from a virtual clock that advances only by simulated time, which removes the main dependence of results on machine speed (see Reproducibility for what remains). Wall-clock timers (profiling, UI deadlines) keep real time. | +| `-fixed-dt ` | Length of one simulation step. Default 1/30; must be in [1/65536, 0.25]. It sets the step, it does not enlarge the engine's integration step. | +| `-max-sim-time ` | End the run once this much mission time has been simulated. Must be above 0 and at most 32767 (the range of the engine's fixed-point mission clock). | +| `-sim-result ` | When the engine shuts down, write a JSON file describing the run (below). | +| `-seed ` | Existing option. With this change it seeds the generators the simulation uses: the main generator, the generator that `RandomRange` instances use, and the ship firepoint, asteroid-split and random-message-sender shuffles. | + +`-sim-result` file: + +```json +{ "schema": 1, "mission": "my-mission", "seed": 1000, "fixed_dt": 0.0333333, + "max_sim_time": 120.0, "simulated_seconds": 120.03, "wall_seconds": 9.4, + "time_limit_reached": true, "termination": "time_limit" } +``` + +`termination` is `time_limit` when `-max-sim-time` ended the run, `start_failed` when the default loadout could not +be committed so the mission never started, and `mission_exit` when the mission or the game ended it first. + +## What runs and what does not + +The authoritative frame is unchanged: `game_do_frame()` calls `game_frame()` and `game_simulation_frame()`, so +object movement, ship and weapon processing, collisions, mission parsing and goals, and scripting all run. + +Script frame callbacks (`On Frame`) normally fire from `gr_flip()`, which headless mode never reaches. The +engine therefore runs them once per simulated frame through `gr_process_frame_callbacks()`, the same function the +rendered path uses. A mod that runs its game logic from `On Frame` behaves as usual. + +Screens that wait for a person are passed automatically. `-start_mission` normally stops at the fiction, command +briefing, red alert and briefing screens and at the debriefing. In a headless single-player run the engine skips +the fiction and command briefing. At the briefing it takes the default loadout through the same `commit_pressed()` +call it uses for missions flagged no-briefing; at the red alert screen it continues into the mission as that +screen's button does. When the mission ends it leaves the debriefing by quitting, so the result file is written. +If the default loadout cannot be committed (for example a required weapon is not on any ship) the run ends with a +warning in the engine log and `start_failed` in the result. Any other modal popup is logged and answered as if +cancelled, unless a script handles `On Dialog Frame`. + +Skipped: renderer setup and frame rendering, sound and speech, input devices, HUD and radar, camera and photo +mode interpolation, the intro movie, and real-time frame pacing. Particle, trail, shockwave, asteroid and nebula +updates still run, because nothing has shown them to be irrelevant to gameplay. Model and collision data still +load. + +## Quick start + +1. Make a **base workspace**: a portable FreeSpace Open folder holding the root `.vp` files, any mod folders the + mission needs, and a stored pilot (`data/players/.json`, or `.plr`, in the root `data` folder or in a mod). + The runner never writes to it. +2. Copy `example.plan.json`, set `engine`, `base_workspace`, `output_dir`, `mission` and `pilot`. +3. Check the plan, then run it: + + ``` + python tools/headless_batch/run_batch.py my.plan.json --dry-run # validate and print the commands + python tools/headless_batch/run_batch.py my.plan.json + ``` + +Before a long batch, run one short simulation (`"repeats": 1, "max_sim_time": 20`) and confirm the result file +reports `simulated_seconds` close to 20. A run that never writes a result ends as a timeout and is reported as a +failure with the reason. + +## Plan reference + +Unknown keys are rejected, so a misspelt option fails instead of being ignored. + +| Key | Required | Default | Meaning | +|---|---|---|---| +| `engine` | yes | | Path to the engine executable. | +| `base_workspace` | yes | | Portable workspace described above. | +| `output_dir` | yes | | Where runs and results go. Keep the path short on Windows, see Limits. | +| `mission` | yes | | Passed to `-start_mission`. A simple name (letters, digits, `_ . -`). | +| `pilot` | yes | | Stored pilot, passed to `-pilot`. | +| `max_sim_time` | yes | | Simulated seconds to run, passed to `-max-sim-time`. | +| `mods` | no | `[]` | Mod folder names under the base workspace, passed to `-mod` in this order. | +| `seeds` | no | | Explicit list of seeds, integers in 1..2147483647. One run per seed. | +| `seed_start`, `repeats` | no | 1000, 1 | Used when `seeds` is absent: seeds `seed_start` .. `seed_start + repeats - 1`. | +| `workers` | no | 1 | Engine processes running at once. | +| `fixed_dt` | no | 1/30 | Passed to `-fixed-dt`. | +| `timeout_seconds` | no | 600 | Wall-clock limit per run; a run over it is killed and counted as failed. | +| `base_port` | no | 39000 | Run *i* binds UDP port `base_port + i`. Every engine process binds one at startup, so ports must be unique. | +| `extra_args` | no | `[]` | Extra engine arguments appended to the command line. | + +## What a run does + +For run *i* with seed *s* the runner: + +1. creates `output_dir/runs/r000i/w`, hardlinks the root `.vp` files and every `.vp` inside the mod folders, + and copies the rest, so a run does not write through to the base workspace (the engine only reads `.vp` files; + a run that rewrote one in place would change the base too); +2. starts `engine -portable_mode -parse_cmdline_only -port -start_mission -pilot + -headless -fixed-dt
-max-sim-time -sim-result -noninteractive -nomovies -seed `, then + `-mod` if any, then `extra_args`, with the workspace as the working directory; +3. waits up to `timeout_seconds`, killing the whole process tree if it is exceeded; +4. reads the result file and records the run. + +A run is `ok` only if the engine exited with code 0 and wrote a readable result file whose `termination` is +`time_limit` or `mission_exit`. A nonzero exit, a missing or unreadable result, a mission that never started, or a +timeout is a failure and is recorded with the reason; a run whose result could not be read is not +reported as a success. On Windows the runner turns off crash and missing-file dialogs for its children so a +crashing engine exits instead of blocking an unattended batch (this changes no registry or machine setting). + +## Output + +``` +output_dir/ + results.jsonl one JSON record per finished run (index, seed, port, status, exit code, wall seconds, + the engine's result, and a list of problems) + summary.json runs, ok, failed, timed out, exit codes, terminations, simulated seconds, batch wall + seconds and simulated seconds per wall second + runs/r0000/ result.json (same record), stdout.txt, stderr.txt, engine-result.json, w/ (workspace) +``` + +`--resume` keeps finished (`ok`) runs whose seed still matches the plan and reruns the others (failed, +interrupted, or run with another seed) from a clean workspace; without it a non-empty +`runs/` folder is refused so earlier results are never overwritten. The exit code is 0 when every run is `ok`, +1 when any failed, and 2 for a plan or usage error. + +## Running the engine directly + +``` +fs2_open_x64.exe -portable_mode -parse_cmdline_only -mod mymod -start_mission my-mission -pilot Tester + -headless -fixed-dt 0.0333333333 -max-sim-time 120 -sim-result result.json -noninteractive -seed 1000 +``` + +## Reproducibility + +`-seed` removes the known engine-side sources of run-to-run variation, but it does not make runs exact replays. +A script that draws its own random numbers (for example Lua `math.random()`, which uses a separate generator) +must seed itself from the run seed. In our tests two short same-seed runs of one scenario were identical, while +two same-seed runs of a long scenario with many AI-driven ships diverged (24 versus 25 losses), and the cause was +not found (a real-clock afterburner cooldown, since fixed for `-unthrottled`, is a possible but unconfirmed +cause). Treat same-seed runs as correlated, not identical, and use several seeds for any comparison. The +engine has no fixed-step mode outside this feature, so rendered and headless runs of one mission are not +expected to match frame for frame either. + +## Limits and known issues + +- **Windows paths.** The engine cannot open files whose full path is 260 characters or longer. The runner checks + this before it starts: it computes the longest path it will stage under `output_dir` and refuses the plan, + naming the longest `output_dir` that fits. +- **Hardlinks.** `.vp` files are hardlinked, which needs `base_workspace` and `output_dir` on the same volume. + The runner checks this before it starts and refuses the plan otherwise. +- **Process-level parallelism only.** The engine has a lot of global state, so each simulation gets its own + process. More workers than physical cores did not help in our measurements. +- **Not a rendering test.** Anything that depends on the renderer, audio or input does not run. +- **Negative option values.** The engine's command-line parser reads a value that starts with `-` (for example + `-seed -1`) as a new option and shows an "Unrecognized command line parameter" message box, which blocks a + headless run. The runner cannot produce this (seeds must be above zero and `fixed_dt` is range-checked); keep to + positive values when you start the engine by hand. + +## Tests + +`python -m unittest tools/headless_batch/test_run_batch.py`. They cover plan validation, command construction, +workspace staging, the same-volume and path-length checks, and a stub engine for success, crash, missing result and +timeout. They need no game files. diff --git a/tools/headless_batch/example.plan.json b/tools/headless_batch/example.plan.json new file mode 100644 index 00000000000..103e30b1ee0 --- /dev/null +++ b/tools/headless_batch/example.plan.json @@ -0,0 +1,16 @@ +{ + "engine": "C:/path/to/fs2_open_x64.exe", + "base_workspace": "C:/path/to/portable-workspace", + "output_dir": "C:/path/to/batch-output", + "mods": ["mymod"], + "mission": "my-mission", + "pilot": "Tester", + "max_sim_time": 120, + "seed_start": 1000, + "repeats": 4, + "workers": 2, + "fixed_dt": 0.03333333333333333, + "timeout_seconds": 600, + "base_port": 39000, + "extra_args": [] +} diff --git a/tools/headless_batch/run_batch.py b/tools/headless_batch/run_batch.py new file mode 100644 index 00000000000..d938defd725 --- /dev/null +++ b/tools/headless_batch/run_batch.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +"""Run many headless FreeSpace Open simulations in parallel OS processes. + +Each run is one engine process in its own portable workspace, with its own seed +and UDP port, started with -headless, a fixed step and a simulated-time limit. +The engine's -sim-result sidecar is collected into results.jsonl and summary.json. +See README.md for the plan format and the engine options this relies on. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import ctypes +import json +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +NAME = re.compile(r"[A-Za-z0-9_.-]+") +RESERVED_NAMES = {".", ".."} +MAX_SEED = 2**31 - 1 +MAX_SIM_TIME = 32767.0 # the engine's mission clock is 16.16 fixed point +FIXED_DT_MIN = 1.0 / 65536 # a shorter step rounds to zero in the engine +FINISHED = {"time_limit", "mission_exit"} # engine terminations that mean the mission ran + +# Plan keys, with None meaning "required". Unknown keys are rejected so typos fail loudly. +PLAN_DEFAULTS: dict[str, Any] = { + "engine": None, + "base_workspace": None, + "output_dir": None, + "mission": None, + "pilot": None, + "max_sim_time": None, + "mods": [], + "seeds": None, + "seed_start": 1000, + "repeats": 1, + "workers": 1, + "fixed_dt": 1.0 / 30.0, + "timeout_seconds": 600.0, + "base_port": 39000, + "extra_args": [], +} + + +class PlanError(ValueError): + """The plan file or the workspace it names is not usable.""" + + +def _name(value: Any, what: str) -> str: + if not isinstance(value, str) or not NAME.fullmatch(value) or value in RESERVED_NAMES: + raise PlanError(f"{what} must be a simple name (letters, digits, _ . -), got {value!r}") + return value + + +def _path(value: Any, what: str) -> Path: + if not isinstance(value, str) or not value or "\0" in value: + raise PlanError(f"{what} must be a path string, got {value!r}") + return Path(value).resolve() + + +def _number(plan: dict[str, Any], key: str, low: float, high: float, integer: bool = False, + low_inclusive: bool = True) -> float | int: + """A finite number in the given range (a duration or step must be strictly above zero: low_inclusive=False).""" + value = plan[key] + if isinstance(value, bool) or not isinstance(value, (int, float)) or value != value or abs(value) == float("inf"): + raise PlanError(f"{key} must be a finite number, got {value!r}") + if integer and int(value) != value: + raise PlanError(f"{key} must be a whole number, got {value!r}") + above_low = value >= low if low_inclusive else value > low + if not (above_low and value <= high): + raise PlanError(f"{key} must be {'at least' if low_inclusive else 'above'} {low} and at most {high}, got {value!r}") + return int(value) if integer else float(value) + + +def load_plan(path: Path) -> dict[str, Any]: + """Read, validate and normalise a plan. Raises PlanError with a message that names the problem.""" + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PlanError(f"cannot read plan {path}: {exc}") from exc + if not isinstance(raw, dict): + raise PlanError("the plan must be a JSON object") + unknown = sorted(set(raw) - set(PLAN_DEFAULTS)) + if unknown: + raise PlanError(f"unknown plan keys: {', '.join(unknown)}") + plan = {**PLAN_DEFAULTS, **raw} + missing = [key for key in ("engine", "base_workspace", "output_dir", "mission", "pilot", "max_sim_time") + if plan[key] is None] + if missing: + raise PlanError(f"missing required plan keys: {', '.join(missing)}") + + plan["mission"] = _name(plan["mission"], "mission") + plan["pilot"] = _name(plan["pilot"], "pilot") + if not isinstance(plan["mods"], list): + raise PlanError("mods must be a list of directory names") + plan["mods"] = [_name(mod, "mod") for mod in plan["mods"]] + if not isinstance(plan["extra_args"], list) or not all(isinstance(a, str) for a in plan["extra_args"]): + raise PlanError("extra_args must be a list of strings") + + plan["fixed_dt"] = _number(plan, "fixed_dt", FIXED_DT_MIN, 0.25) + plan["max_sim_time"] = _number(plan, "max_sim_time", 0.0, MAX_SIM_TIME, low_inclusive=False) + plan["timeout_seconds"] = _number(plan, "timeout_seconds", 0.0, 1e7, low_inclusive=False) + plan["workers"] = _number(plan, "workers", 1, 256, integer=True) + plan["base_port"] = _number(plan, "base_port", 1024, 65000, integer=True) + + if plan["seeds"] is None: + start = int(_number(plan, "seed_start", 1, MAX_SEED, integer=True)) + repeats = int(_number(plan, "repeats", 1, 100000, integer=True)) + plan["seeds"] = [start + i for i in range(repeats)] + else: + seeds = plan["seeds"] + if (not isinstance(seeds, list) or not seeds + or not all(isinstance(s, int) and not isinstance(s, bool) and 1 <= s <= MAX_SEED for s in seeds)): + raise PlanError(f"seeds must be a non-empty list of integers in 1..{MAX_SEED}") + if plan["base_port"] + len(plan["seeds"]) > 65535: + raise PlanError("base_port plus the number of runs exceeds the port range") + + # Absolute, because the engine runs in its run folder: a relative -sim-result path would land there. + plan["engine"] = _path(plan["engine"], "engine") + plan["base_workspace"] = _path(plan["base_workspace"], "base_workspace") + plan["output_dir"] = _path(plan["output_dir"], "output_dir") + if not plan["engine"].is_file(): + raise PlanError(f"engine not found: {plan['engine']}") + check_workspace(plan["base_workspace"], plan["mods"], plan["pilot"]) + check_same_volume(plan["base_workspace"], plan["output_dir"]) + check_path_length(plan["base_workspace"], plan["mods"], plan["output_dir"]) + return plan + + +# The engine cannot open a file whose full path reaches this many characters (Windows MAX_PATH, terminator included). +WINDOWS_PATH_LIMIT = 260 +RUN_FOLDER_SUFFIX = os.sep + os.path.join("runs", "r0000", "w") # the longest run folder name the runner makes + + +def _volume_of(path: Path) -> int: + """Device id of the nearest existing ancestor (the folder may not exist yet).""" + probe = path.resolve() + while not probe.exists(): + probe = probe.parent + return probe.stat().st_dev + + +def check_same_volume(base: Path, output_dir: Path) -> None: + """.vp packs are hardlinked into each run, which only works inside one volume. Fail now, not on run 1.""" + if _volume_of(base) != _volume_of(output_dir): + raise PlanError(f"base_workspace and output_dir must be on the same volume so the .vp packs can be hardlinked " + f"({base} and {output_dir} are not)") + + +def longest_relative_path(base: Path, mods: list[str]) -> int: + """Length of the longest path, relative to a workspace root, among the files a run will have.""" + longest = 0 + for top in [base / "data", *(base / mod for mod in mods)]: + for folder, _dirs, files in os.walk(top): + relative = os.path.relpath(folder, base) + for name in files: + longest = max(longest, len(os.path.join(relative, name))) + return longest + + +def check_path_length(base: Path, mods: list[str], output_dir: Path, limit: int = WINDOWS_PATH_LIMIT) -> None: + """Windows only: refuse an output_dir so deep that the engine could not open some staged file.""" + if os.name != "nt": + return + worst = len(str(output_dir.resolve())) + len(RUN_FOLDER_SUFFIX) + 1 + longest_relative_path(base, mods) + if worst >= limit: + allowed = len(str(output_dir.resolve())) - (worst - limit + 1) + raise PlanError(f"output_dir is too long: the longest staged file path would be {worst} characters but the engine " + f"cannot open paths of {limit} or more. Use an output_dir of at most {allowed} characters " + f"(it is {len(str(output_dir.resolve()))} now)") + + +def check_workspace(base: Path, mods: list[str], pilot: str) -> None: + if not base.is_dir(): + raise PlanError(f"base_workspace not found: {base}") + if not list(base.glob("*.vp")): + raise PlanError(f"base_workspace has no root .vp files: {base}") + for mod in mods: + if not (base / mod).is_dir(): + raise PlanError(f"mod directory not found under base_workspace: {base / mod}") + # A stored pilot is mandatory: without one the engine stops at the pilot chooser. + places = [base / "data" / "players"] + [base / mod / "data" / "players" for mod in mods] + if not any((place / (pilot + ext)).is_file() for place in places for ext in (".json", ".plr")): + raise PlanError(f"no stored pilot {pilot!r} (data/players/{pilot}.json or .plr) in the base workspace or its mods") + + +def link_or_copy(source: str, destination: str) -> None: + """Hardlink .vp packs (immutable and large); copy everything else so a run can never write through to the base.""" + if source.lower().endswith(".vp"): + try: + os.link(source, destination) + return + except OSError as exc: + raise RuntimeError( + f"cannot hardlink {source}; put base_workspace and output_dir on the same volume") from exc + shutil.copy2(source, destination) + + +def _ignore_logs(_directory: str, names: list[str]) -> set[str]: + return {name for name in names if name.lower().endswith(".log")} + + +def prepare_workspace(base: Path, destination: Path, mods: list[str]) -> None: + destination.mkdir(parents=True, exist_ok=False) + for item in base.iterdir(): + if item.is_file() and item.suffix.lower() == ".vp": + link_or_copy(str(item), str(destination / item.name)) + if (base / "data").is_dir(): + shutil.copytree(base / "data", destination / "data", ignore=_ignore_logs, copy_function=link_or_copy) + else: + (destination / "data").mkdir() + for mod in mods: + shutil.copytree(base / mod, destination / mod, ignore=_ignore_logs, copy_function=link_or_copy) + + +def build_command(plan: dict[str, Any], seed: int, port: int, result_path: Path, launcher: tuple[str, ...] = ()) -> list[str]: + """The engine command line for one run. `launcher` is test plumbing (an interpreter argument before the flags).""" + command = [str(plan["engine"]), *launcher, + "-portable_mode", "-parse_cmdline_only", "-port", str(port), + "-start_mission", plan["mission"], "-pilot", plan["pilot"], + "-headless", "-fixed-dt", repr(plan["fixed_dt"]), "-max-sim-time", repr(plan["max_sim_time"]), + "-sim-result", str(result_path), "-noninteractive", "-nomovies", "-seed", str(seed)] + if plan["mods"]: + command += ["-mod", ",".join(plan["mods"])] + return command + plan["extra_args"] + + +def suppress_crash_dialogs() -> None: + """Windows only: stop a crashing engine from blocking an unattended batch on a dialog. + + Process-wide for this runner and inherited by its children; it changes no registry or machine setting. + """ + if os.name == "nt": + # SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX + ctypes.windll.kernel32.SetErrorMode(0x0001 | 0x0002 | 0x8000) + + +def kill_tree(process: subprocess.Popen[bytes]) -> None: + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], capture_output=True, check=False) + else: + try: + os.killpg(process.pid, 9) + except ProcessLookupError: + pass + process.wait() + + +def run_one(plan: dict[str, Any], index: int, seed: int, launcher: tuple[str, ...] = ()) -> dict[str, Any]: + """Stage a workspace, run one engine process to completion or timeout, and return its result record.""" + run_dir = plan["output_dir"] / "runs" / f"r{index:04d}" + workspace = run_dir / "w" + result_path = run_dir / "engine-result.json" + port = plan["base_port"] + index + record: dict[str, Any] = {"index": index, "seed": seed, "port": port, "status": "failed", + "exit_code": None, "timed_out": False, "wall_seconds": None, + "sim_result": None, "problems": []} + try: + prepare_workspace(plan["base_workspace"], workspace, plan["mods"]) + command = build_command(plan, seed, port, result_path, launcher) + started = time.monotonic() + with open(run_dir / "stdout.txt", "wb") as out, open(run_dir / "stderr.txt", "wb") as err: + process = subprocess.Popen(command, cwd=workspace, stdout=out, stderr=err, + start_new_session=(os.name != "nt")) + try: + process.wait(timeout=plan["timeout_seconds"]) + except subprocess.TimeoutExpired: + record["timed_out"] = True + kill_tree(process) + record["wall_seconds"] = round(time.monotonic() - started, 3) + record["exit_code"] = process.returncode + if result_path.is_file(): + try: + record["sim_result"] = json.loads(result_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + record["problems"].append(f"unreadable engine result: {exc}") + else: + termination = record["sim_result"].get("termination") + if termination not in FINISHED: + record["problems"].append(f"mission did not run to an end (termination {termination!r})") + else: + record["problems"].append("engine wrote no -sim-result file") + if record["timed_out"]: + record["problems"].append(f"killed after {plan['timeout_seconds']} s") + if record["exit_code"] not in (0, None) and not record["timed_out"]: + record["problems"].append(f"engine exit code {record['exit_code']}") + if not record["problems"]: + record["status"] = "ok" + except (OSError, RuntimeError) as exc: + record["problems"].append(str(exc)) + (run_dir / "result.json").write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8") + return record + + +def read_record(path: Path) -> dict[str, Any] | None: + """A run's result.json, or None if it is missing or unreadable (an interrupted write).""" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + +def summarize(records: list[dict[str, Any]], batch_wall: float) -> dict[str, Any]: + ok = [r for r in records if r["status"] == "ok"] + sim = sum(r["sim_result"].get("simulated_seconds", 0.0) for r in ok) + exits: dict[str, int] = {} + ends: dict[str, int] = {} + for r in records: + exits[str(r["exit_code"])] = exits.get(str(r["exit_code"]), 0) + 1 + if r["sim_result"]: + term = str(r["sim_result"].get("termination")) + ends[term] = ends.get(term, 0) + 1 + return {"runs": len(records), "ok": len(ok), "failed": len(records) - len(ok), + "timed_out": sum(1 for r in records if r["timed_out"]), + "exit_codes": exits, "terminations": ends, + "simulated_seconds_total": round(sim, 3), "batch_wall_seconds": round(batch_wall, 3), + "simulated_seconds_per_wall_second": round(sim / batch_wall, 3) if batch_wall > 0 else None} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument("plan", type=Path, help="plan JSON file (see README.md)") + parser.add_argument("--resume", action="store_true", help="keep finished runs in output_dir and run only the rest") + parser.add_argument("--dry-run", action="store_true", help="validate the plan and print the commands without running") + args = parser.parse_args(argv) + + try: + plan = load_plan(args.plan) + except PlanError as exc: + print(f"plan error: {exc}", file=sys.stderr) + return 2 + runs_dir = plan["output_dir"] / "runs" + + if args.dry_run: + for index, seed in enumerate(plan["seeds"]): + print(subprocess.list2cmdline(build_command(plan, seed, plan["base_port"] + index, runs_dir / f"r{index:04d}" / "engine-result.json"))) + return 0 + + done: dict[int, dict[str, Any]] = {} + if runs_dir.exists() and any(runs_dir.iterdir()): + if not args.resume: + print(f"plan error: {runs_dir} is not empty; use --resume or a new output_dir", file=sys.stderr) + return 2 + for index, seed in enumerate(plan["seeds"]): + run_dir = runs_dir / f"r{index:04d}" + if not run_dir.exists(): + continue + previous = read_record(run_dir / "result.json") + if previous and previous.get("status") == "ok" and previous.get("seed") == seed: + done[index] = previous + else: + # failed, interrupted before it wrote a record, or run with another seed: rerun from a clean workspace + shutil.rmtree(run_dir) + runs_dir.mkdir(parents=True, exist_ok=True) + + suppress_crash_dialogs() + pending = [(i, s) for i, s in enumerate(plan["seeds"]) if i not in done] + records = dict(done) + started = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=plan["workers"]) as pool: + futures = [pool.submit(run_one, plan, i, s) for i, s in pending] + for future in concurrent.futures.as_completed(futures): + record = future.result() + records[record["index"]] = record + with open(plan["output_dir"] / "results.jsonl", "a", encoding="utf-8") as sink: + sink.write(json.dumps(record, sort_keys=True) + "\n") + print(f"run {record['index']:>4} seed {record['seed']}: {record['status']}" + + (f" ({'; '.join(record['problems'])})" if record["problems"] else ""), flush=True) + ordered = [records[i] for i in sorted(records)] + summary = summarize(ordered, time.monotonic() - started) + (plan["output_dir"] / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 if summary["failed"] == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/headless_batch/test_run_batch.py b/tools/headless_batch/test_run_batch.py new file mode 100644 index 00000000000..c8d0b8e07cd --- /dev/null +++ b/tools/headless_batch/test_run_batch.py @@ -0,0 +1,221 @@ +"""Tests for run_batch.py. They need no FreeSpace Open install: a stub "engine" (a Python script run by the +current interpreter) stands in for the executable and writes the -sim-result sidecar the real engine writes. + + python -m unittest tools/headless_batch/test_run_batch.py +""" + +import functools +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import run_batch # noqa: E402 + +STUB_OK = r""" +import json, sys +args = sys.argv[1:] +path = args[args.index("-sim-result") + 1] +seed = int(args[args.index("-seed") + 1]) +json.dump({"schema": 1, "seed": seed, "simulated_seconds": 12.5, "termination": "time_limit"}, open(path, "w")) +""" +STUB_START_FAILED = STUB_OK.replace('"time_limit"', '"start_failed"') +STUB_SLEEP = "import time; time.sleep(60)" +STUB_CRASH = "import sys; sys.exit(3)" + + +class Fixture(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.base = self.root / "base" + (self.base / "data" / "players").mkdir(parents=True) + (self.base / "root_fs2.vp").write_bytes(b"vp") + (self.base / "data" / "players" / "Tester.json").write_text("{}") + (self.base / "data" / "debug.log").write_text("noise") + (self.base / "mymod" / "data").mkdir(parents=True) + (self.base / "mymod" / "data" / "mod.vp").write_bytes(b"vp") + (self.base / "mymod" / "data" / "table.tbm").write_text("x") + self.engine = Path(sys.executable) + + def tearDown(self): + self._tmp.cleanup() + + def plan_file(self, **overrides): + plan = {"engine": str(self.engine), "base_workspace": str(self.base), "output_dir": str(self.root / "out"), + "mission": "test-mission", "pilot": "Tester", "max_sim_time": 20, "mods": ["mymod"], + "seed_start": 500, "repeats": 3, "workers": 2, "base_port": 40000} + plan.update(overrides) + path = self.root / "plan.json" + path.write_text(json.dumps(plan)) + return path + + def stub(self, source): + path = self.root / "stub.py" + path.write_text(source) + return (str(path),) + + +class PlanTests(Fixture): + def test_defaults_and_seed_expansion(self): + plan = run_batch.load_plan(self.plan_file()) + self.assertEqual(plan["seeds"], [500, 501, 502]) + self.assertAlmostEqual(plan["fixed_dt"], 1 / 30) + self.assertEqual(plan["workers"], 2) + + def test_explicit_seeds_win_over_seed_start(self): + self.assertEqual(run_batch.load_plan(self.plan_file(seeds=[7, 9]))["seeds"], [7, 9]) + + def test_rejects_bad_plans_with_a_message_that_names_the_problem(self): + cases = { + "unknown plan keys": {"fixd_dt": 0.1}, + "fixed_dt must be at least": {"fixed_dt": 0}, + "fixed_dt must be at least 1.52587890625e-05 and at most 0.25": {"fixed_dt": 0.5}, + "max_sim_time": {"max_sim_time": -1}, + "max_sim_time must be above 0.0 and at most 32767": {"max_sim_time": 40000}, + "seeds must be": {"seeds": [0]}, + "simple name": {"mission": "../escape"}, + "mod directory not found": {"mods": ["nope"]}, + "no stored pilot": {"pilot": "Ghost"}, + "port range": {"base_port": 65000, "repeats": 1000}, + "engine must be a path string": {"engine": 5}, + "base_workspace must be a path string": {"base_workspace": ["x"]}, + "output_dir must be a path string": {"output_dir": "a\0b"}, + } + for fragment, overrides in cases.items(): + with self.subTest(fragment): + with self.assertRaises(run_batch.PlanError) as ctx: + run_batch.load_plan(self.plan_file(**overrides)) + self.assertIn(fragment, str(ctx.exception)) + + def test_missing_required_keys_are_listed(self): + path = self.root / "plan.json" + path.write_text(json.dumps({"engine": str(self.engine)})) + with self.assertRaises(run_batch.PlanError) as ctx: + run_batch.load_plan(path) + self.assertIn("base_workspace", str(ctx.exception)) + self.assertIn("pilot", str(ctx.exception)) + + def test_base_workspace_without_vp_files_is_rejected(self): + (self.base / "root_fs2.vp").unlink() + with self.assertRaises(run_batch.PlanError): + run_batch.load_plan(self.plan_file()) + + def test_output_dir_on_another_volume_is_rejected(self): + original = run_batch._volume_of + try: + run_batch._volume_of = lambda path: 1 if path == self.base.resolve() else 2 + with self.assertRaises(run_batch.PlanError) as ctx: + run_batch.load_plan(self.plan_file()) + self.assertIn("same volume", str(ctx.exception)) + finally: + run_batch._volume_of = original + + @unittest.skipUnless(sys.platform == "win32", "the path limit is a Windows limit") + def test_a_too_long_output_dir_is_rejected_with_the_allowed_length(self): + with self.assertRaises(run_batch.PlanError) as ctx: + run_batch.check_path_length(self.base, ["mymod"], self.root / "out", limit=60) + self.assertIn("too long", str(ctx.exception)) + self.assertIn("at most", str(ctx.exception)) + run_batch.check_path_length(self.base, ["mymod"], self.root / "out", limit=10000) # roomy limit: no error + + +class RunTests(Fixture): + def test_command_has_the_headless_flags_and_the_per_run_values(self): + plan = run_batch.load_plan(self.plan_file(extra_args=["-nosound"])) + command = run_batch.build_command(plan, 501, 40001, Path("r.json")) + self.assertEqual(command[0], str(self.engine)) + for flag in ("-headless", "-noninteractive", "-portable_mode", "-nomovies", "-nosound"): + self.assertIn(flag, command) + self.assertEqual(command[command.index("-seed") + 1], "501") + self.assertEqual(command[command.index("-port") + 1], "40001") + self.assertEqual(command[command.index("-mod") + 1], "mymod") + self.assertEqual(command[command.index("-max-sim-time") + 1], "20.0") + + def test_workspace_hardlinks_packs_copies_the_rest_and_skips_logs(self): + destination = self.root / "w" + run_batch.prepare_workspace(self.base, destination, ["mymod"]) + self.assertEqual((destination / "root_fs2.vp").stat().st_ino, (self.base / "root_fs2.vp").stat().st_ino) + self.assertEqual((destination / "mymod" / "data" / "mod.vp").stat().st_ino, + (self.base / "mymod" / "data" / "mod.vp").stat().st_ino) + (destination / "mymod" / "data" / "table.tbm").write_text("changed") + self.assertEqual((self.base / "mymod" / "data" / "table.tbm").read_text(), "x") # a run cannot write through + self.assertFalse((destination / "data" / "debug.log").exists()) + self.assertTrue((destination / "data" / "players" / "Tester.json").is_file()) + + def test_successful_run_collects_the_sidecar(self): + plan = run_batch.load_plan(self.plan_file()) + record = run_batch.run_one(plan, 1, 501, launcher=self.stub(STUB_OK)) + self.assertEqual(record["status"], "ok", record["problems"]) + self.assertEqual(record["sim_result"]["seed"], 501) + self.assertEqual(record["port"], 40001) + self.assertTrue((plan["output_dir"] / "runs" / "r0001" / "result.json").is_file()) + + def test_crash_and_missing_sidecar_are_failures_not_silent(self): + plan = run_batch.load_plan(self.plan_file()) + record = run_batch.run_one(plan, 0, 500, launcher=self.stub(STUB_CRASH)) + self.assertEqual(record["status"], "failed") + self.assertEqual(record["exit_code"], 3) + self.assertTrue(any("no -sim-result" in p for p in record["problems"])) + + def test_relative_output_dir_still_finds_the_engine_result(self): + # The engine runs in its run folder, so a relative -sim-result path would be written there instead. + old_cwd = os.getcwd() + os.chdir(self.root) + try: + plan = run_batch.load_plan(self.plan_file(output_dir="out")) + record = run_batch.run_one(plan, 0, 500, launcher=self.stub(STUB_OK)) + finally: + os.chdir(old_cwd) + self.assertEqual(record["status"], "ok", record["problems"]) + + def test_a_mission_that_never_started_is_a_failure(self): + plan = run_batch.load_plan(self.plan_file()) + record = run_batch.run_one(plan, 0, 500, launcher=self.stub(STUB_START_FAILED)) + self.assertEqual(record["status"], "failed") + self.assertTrue(any("start_failed" in p for p in record["problems"])) + + def test_resume_reruns_interrupted_unreadable_and_reseeded_runs_and_keeps_good_ones(self): + path = self.plan_file(seeds=[7, 8, 9, 10], repeats=None, seed_start=None) + runs = self.root / "out" / "runs" + (runs / "r0000" / "w").mkdir(parents=True) # interrupted before it wrote result.json + (runs / "r0001").mkdir() + (runs / "r0001" / "result.json").write_text('{"status": "ok", "se') # interrupted mid-write + (runs / "r0002").mkdir() + (runs / "r0002" / "result.json").write_text(json.dumps({"index": 2, "seed": 99, "status": "ok"})) # old plan + (runs / "r0003").mkdir() + kept = {"index": 3, "seed": 10, "status": "ok", "exit_code": 0, "timed_out": False, + "sim_result": {"simulated_seconds": 1.0, "termination": "time_limit"}, "problems": []} + (runs / "r0003" / "result.json").write_text(json.dumps(kept)) + original = run_batch.run_one + run_batch.run_one = functools.partial(original, launcher=self.stub(STUB_OK)) + try: + self.assertEqual(run_batch.main([str(path), "--resume"]), 0) + finally: + run_batch.run_one = original + for index, seed in enumerate([7, 8, 9]): + record = json.loads((runs / f"r{index:04d}" / "result.json").read_text()) + self.assertEqual((record["status"], record["seed"]), ("ok", seed)) + self.assertEqual(json.loads((runs / "r0003" / "result.json").read_text()), kept) # not rerun + + def test_timeout_kills_the_process(self): + plan = run_batch.load_plan(self.plan_file(timeout_seconds=1)) + record = run_batch.run_one(plan, 0, 500, launcher=self.stub(STUB_SLEEP)) + self.assertTrue(record["timed_out"]) + self.assertEqual(record["status"], "failed") + self.assertLess(record["wall_seconds"], 30) + + def test_summary_counts_and_rate(self): + ok = {"status": "ok", "exit_code": 0, "timed_out": False, "sim_result": {"simulated_seconds": 10.0, "termination": "time_limit"}} + bad = {"status": "failed", "exit_code": 3, "timed_out": False, "sim_result": None} + summary = run_batch.summarize([ok, ok, bad], batch_wall=5.0) + self.assertEqual((summary["runs"], summary["ok"], summary["failed"]), (3, 2, 1)) + self.assertEqual(summary["simulated_seconds_per_wall_second"], 4.0) + self.assertEqual(summary["terminations"], {"time_limit": 2}) + + +if __name__ == "__main__": + unittest.main()