From 53ba807a2dd4ba277864b42c19b0fef49897d2f8 Mon Sep 17 00:00:00 2001 From: Nicholas Harding <40708936+sir306@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:21:12 +1300 Subject: [PATCH 1/6] Derive missing agent heading and speed in one pass Files without rotation/speed members (every Juelich dataset, and Mobius HDF5 written positions-only) had heading and speed derived by scanning every timestep's sample list once per agent, then again to write the result back: roughly agents x samples. A 1.9 M-sample positions-only file took 12.0 s to derive. Walk the timesteps once instead, keeping a previous-sample pointer per agent. The rules are unchanged (look-ahead heading, hold while the step is under 0.1 cm, last frame carries the previous value, fewer than 2 samples skipped), and the derived values are bit-identical on the Juelich 030_c_56_h0 sample. The progress bar now advances by timesteps rather than agents, and FMobiusImportTimings gains DeriveSeconds. Timing, old -> new: 360 k samples 0.300 s -> 0.0064 s; 719 k samples 2.022 s -> 0.0121 s; 1.90 M samples 12.02 s -> 0.043 s. Tests: ProjectMobius.SimData.DerivedMotionParity compares against a verbatim copy of the old loops (real Juelich import plus edge cases); Mobius.Timing.DerivedMotionScaling is opt-in. AI-assisted with Claude Code; all changes reviewed, tested, and verified by a human maintainer in line with this repository's AI-tooling policy. --- .../MassAI/SubSystems/AgentDataSubsystem.cpp | 248 ++++---- .../MassAI/SubSystems/AgentDataSubsystem.h | 2 + .../Private/Tests/DerivedMotionTest.cpp | 569 ++++++++++++++++++ 3 files changed, 689 insertions(+), 130 deletions(-) create mode 100644 UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/DerivedMotionTest.cpp diff --git a/UnrealFolder/ProjectMobius/Source/ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp b/UnrealFolder/ProjectMobius/Source/ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp index 4aa7f4b52..f2d347afa 100644 --- a/UnrealFolder/ProjectMobius/Source/ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp +++ b/UnrealFolder/ProjectMobius/Source/ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp @@ -741,11 +741,95 @@ void FProcessAgentSimulationDataRunnable::RunSimulationDataGatheringLoop(bool bC TEXT("AgentDataSubsystem - RunSimulationDataGatheringLoop")); } } +namespace +{ + /** + * Hands every entity's movement samples to Visit in timestep order, each paired with that entity's + * NEXT sample (the look-ahead both derivations below need), in one pass over SimulationData. + * + * This replaced a per-entity scan that searched every timestep's whole sample array for the entity, + * then searched it again to write the result back: O(entities x total samples), which turned a + * positions-only load of a few thousand agents into minutes. This is O(total samples), and its + * bookkeeping is O(MaxAgents) rather than O(samples) - SimulationData can already run to gigabytes. + * + * Visit(EntityId, Sample, Next) is called once per sample, in timestep order per entity, with Next == + * nullptr for the entity's final sample. Which samples are visited is exactly what the old scan did, + * so derived values stay bit-identical (ProjectMobius.SimData.DerivedMotionParity holds the old scan + * as its oracle): + * - timesteps in ascending key order, whatever order the map iterates in; + * - only ids in [0, MaxAgents) - out-of-range ids stay in the map, untouched; + * - an entity listed twice in one timestep counts once: the FIRST sample is read and written; + * - an entity with fewer than 2 samples is never visited (there is no direction to derive). + */ + template + void ForEachEntitySampleWithNext(TMap>& SimulationData, const int32 MaxAgents, + const FThreadSafeBool& bShouldStop, const TWeakObjectPtr& OwnerSubsystem, + FVisitor&& Visit) + { + struct FEntityCursor + { + /** Latest sample seen, held back until its successor (or the end of the data) is known. */ + FSimMovementSample* Pending = nullptr; + /** Ordinal of the timestep Pending came from - detects a second sample in the same timestep. */ + int32 PendingStepOrdinal = INDEX_NONE; + /** True once Pending has a predecessor, i.e. the entity has at least 2 samples. */ + bool bHasPredecessor = false; + }; + + TArray Timesteps; + SimulationData.GetKeys(Timesteps); + Timesteps.Sort(); + + TArray Cursors; + Cursors.SetNum(FMath::Max(MaxAgents, 0)); + + for (int32 StepOrdinal = 0; StepOrdinal < Timesteps.Num(); ++StepOrdinal) + { + if (bShouldStop) return; + + for (FSimMovementSample& Sample : SimulationData.FindChecked(Timesteps[StepOrdinal])) + { + if (Sample.EntityID < 0 || Sample.EntityID >= MaxAgents) continue; + + FEntityCursor& Cursor = Cursors[Sample.EntityID]; + if (Cursor.PendingStepOrdinal == StepOrdinal) continue; + + if (Cursor.Pending) + { + Visit(Sample.EntityID, *Cursor.Pending, &Sample); + Cursor.bHasPredecessor = true; + } + Cursor.Pending = &Sample; + Cursor.PendingStepOrdinal = StepOrdinal; + } + + // Report progress to the subsystem for UI feedback (every 100 timesteps to avoid flooding) + if (StepOrdinal % 100 == 0) + { + float CurrentPercentage = static_cast(StepOrdinal) / static_cast(Timesteps.Num()); + if (UAgentDataSubsystem* Subsys = OwnerSubsystem.Get()) + { + Subsys->ProgressQueue.Enqueue(CurrentPercentage); + } + } + } + + // Each entity's final sample has no successor: Visit it last, so it carries the previous value + for (int32 EntityIdx = 0; EntityIdx < Cursors.Num(); ++EntityIdx) + { + if (Cursors[EntityIdx].bHasPredecessor) + { + Visit(EntityIdx, *Cursors[EntityIdx].Pending, nullptr); + } + } + } +} + /** * Calculates rotation from movement direction when the HDF5 source data lacks rotation information. * - * Algorithm: - * 1. For each entity, collect all position samples ordered by timestep + * Algorithm (one pass over all samples, see ForEachEntitySampleWithNext): + * 1. Walk each entity's position samples in timestep order * 2. For each timestep, calculate direction vector to the NEXT position (look-ahead approach) * 3. Convert direction to rotation angle in degrees using FMath::Atan2 * 4. For the last timestep, carry forward the previous calculated rotation @@ -764,50 +848,19 @@ void FProcessAgentSimulationDataRunnable::CalculateRotationFromMovement() UE_LOG(LogTemp, Log, TEXT("Calculating rotation from movement direction for %d entities"), MaxAgents); - // Process each entity independently - for (int32 EntityIdx = 0; EntityIdx < MaxAgents; ++EntityIdx) - { - if (bShouldStop) break; - - // Step 1: Collect all position samples for this entity across all timesteps - // We store (timestep index, position) pairs for later sorting - TArray> EntityPositions; + // Per entity: the rotation used for stationary steps and carried onto the last frame + TArray LastValidRotations; + LastValidRotations.Init(FRotator::ZeroRotator, FMath::Max(MaxAgents, 0)); - for (auto& Pair : *AgentMovementInfoData.SimulationData) + ForEachEntitySampleWithNext(*AgentMovementInfoData.SimulationData, MaxAgents, bShouldStop, OwnerSubsystem, + [&LastValidRotations](const int32 EntityIdx, FSimMovementSample& Sample, const FSimMovementSample* NextSample) { - for (const FSimMovementSample& Sample : Pair.Value) - { - if (Sample.EntityID == EntityIdx) - { - EntityPositions.Add(TPair(Pair.Key, Sample.Position)); - break; // Only one sample per entity per timestep, so we can exit early - } - } - } - - // Need at least 2 positions to calculate any rotation (need direction to next position) - if (EntityPositions.Num() < 2) continue; - - // Step 2: Sort positions by timestep to ensure chronological order - EntityPositions.Sort([](const TPair& A, const TPair& B) - { - return A.Key < B.Key; - }); - - // Step 3: Calculate rotation for each timestep using look-ahead to next position - // LastValidRotation is used for stationary entities or the last frame - FRotator LastValidRotation = FRotator::ZeroRotator; - - for (int32 i = 0; i < EntityPositions.Num(); ++i) - { - if (bShouldStop) break; - - FRotator NewRotation = LastValidRotation; // Default to previous rotation + FRotator& LastValidRotation = LastValidRotations[EntityIdx]; // Calculate direction from current position to NEXT position (look-ahead approach) - if (i < EntityPositions.Num() - 1) + if (NextSample) { - FVector Delta = EntityPositions[i + 1].Value - EntityPositions[i].Value; + FVector Delta = NextSample->Position - Sample.Position; // Only update rotation if there's meaningful movement (avoid jitter from tiny movements) // Threshold of 0.1 cm filters out noise while allowing detection of real movement @@ -821,35 +874,14 @@ void FProcessAgentSimulationDataRunnable::CalculateRotationFromMovement() // Apply -90 degree offset for mesh orientation correction // Unreal meshes typically face +X when yaw=0, but our coordinate system // expects facing direction to align with movement vector - NewRotation = FRotator(0.0f, YawDeg - 90.0f, 0.0f); - LastValidRotation = NewRotation; + LastValidRotation = FRotator(0.0f, YawDeg - 90.0f, 0.0f); } // If position hasn't changed significantly, keep LastValidRotation } // For the last frame, we keep LastValidRotation (carry forward previous rotation) - // Step 4: Update the rotation value in the actual simulation data - int32 Timestep = EntityPositions[i].Key; - for (FSimMovementSample& Sample : (*AgentMovementInfoData.SimulationData)[Timestep]) - { - if (Sample.EntityID == EntityIdx) - { - Sample.Rotation = NewRotation; - break; - } - } - } - - // Report progress to the subsystem for UI feedback (every 100 entities to avoid flooding) - if (EntityIdx % 100 == 0) - { - float CurrentPercentage = static_cast(EntityIdx) / static_cast(MaxAgents); - if (UAgentDataSubsystem* Subsys = OwnerSubsystem.Get()) - { - Subsys->ProgressQueue.Enqueue(CurrentPercentage); - } - } - } + Sample.Rotation = LastValidRotation; + }); UE_LOG(LogTemp, Log, TEXT("Finished calculating rotation from movement")); } @@ -857,8 +889,8 @@ void FProcessAgentSimulationDataRunnable::CalculateRotationFromMovement() /** * Calculates speed from position deltas when the HDF5 source data lacks speed information. * - * Algorithm: - * 1. For each entity, collect all position samples ordered by timestep + * Algorithm (one pass over all samples, see ForEachEntitySampleWithNext): + * 1. Walk each entity's position samples in timestep order * 2. Calculate speed using the formula: speed = distance / time * - Distance is the Euclidean distance between consecutive positions (in cm after unit conversion) * - Time is TimeBetweenSteps (the sampling interval in seconds) @@ -869,7 +901,7 @@ void FProcessAgentSimulationDataRunnable::CalculateRotationFromMovement() * this discrepancy occurs - possible causes include: * - TimeBetweenSteps not being set correctly before this function is called * - Position data not being properly converted to cm before storage - * - Entities that only appear in a single timestep (skipped due to Num() < 2 check) + * - Entities that only appear in a single timestep (skipped: they have no next position) * - Floating point precision issues with very small movements */ void FProcessAgentSimulationDataRunnable::CalculateSpeedFromMovement() @@ -882,80 +914,29 @@ void FProcessAgentSimulationDataRunnable::CalculateSpeedFromMovement() UE_LOG(LogTemp, Log, TEXT("Calculating speed from position deltas for %d entities"), MaxAgents); - // Process each entity independently - for (int32 EntityIdx = 0; EntityIdx < MaxAgents; ++EntityIdx) - { - if (bShouldStop) break; - - // Step 1: Collect all position samples for this entity across all timesteps - // We store (timestep index, position) pairs for later sorting - TArray> EntityPositions; + // Per entity: the speed carried onto the last frame + TArray LastValidSpeeds; + LastValidSpeeds.Init(0.0f, FMath::Max(MaxAgents, 0)); - for (auto& Pair : *AgentMovementInfoData.SimulationData) + ForEachEntitySampleWithNext(*AgentMovementInfoData.SimulationData, MaxAgents, bShouldStop, OwnerSubsystem, + [this, &LastValidSpeeds](const int32 EntityIdx, FSimMovementSample& Sample, const FSimMovementSample* NextSample) { - for (const FSimMovementSample& Sample : Pair.Value) - { - if (Sample.EntityID == EntityIdx) - { - EntityPositions.Add(TPair(Pair.Key, Sample.Position)); - break; // Only one sample per entity per timestep, so we can exit early - } - } - } - - // Need at least 2 positions to calculate any speed - if (EntityPositions.Num() < 2) continue; - - // Step 2: Sort positions by timestep to ensure chronological order - EntityPositions.Sort([](const TPair& A, const TPair& B) - { - return A.Key < B.Key; - }); - - // Step 3: Calculate speed for each timestep using look-ahead to next position - float LastValidSpeed = 0.0f; - - for (int32 i = 0; i < EntityPositions.Num(); ++i) - { - if (bShouldStop) break; - - float Speed = LastValidSpeed; // Default to previous speed (used for last frame) + float& LastValidSpeed = LastValidSpeeds[EntityIdx]; // Calculate speed from current position to next position - if (i < EntityPositions.Num() - 1) + if (NextSample) { - FVector Delta = EntityPositions[i + 1].Value - EntityPositions[i].Value; + FVector Delta = NextSample->Position - Sample.Position; float Distance = Delta.Size(); // Distance in cm (positions are converted to cm during loading) // Speed formula: speed (m/s) = distance (cm) / (time (s) * 100) // The * 100 converts cm to m - Speed = Distance / (TimeBetweenSteps * 100.0f); - LastValidSpeed = Speed; + LastValidSpeed = Distance / (TimeBetweenSteps * 100.0f); } // For the last frame, we keep LastValidSpeed (carry forward previous speed) - // Step 4: Update the speed value in the actual simulation data - int32 Timestep = EntityPositions[i].Key; - for (FSimMovementSample& Sample : (*AgentMovementInfoData.SimulationData)[Timestep]) - { - if (Sample.EntityID == EntityIdx) - { - Sample.Speed = Speed; - break; - } - } - } - - // Report progress to the subsystem for UI feedback (every 100 entities) - if (EntityIdx % 100 == 0) - { - float CurrentPercentage = static_cast(EntityIdx) / static_cast(MaxAgents); - if (UAgentDataSubsystem* Subsys = OwnerSubsystem.Get()) - { - Subsys->ProgressQueue.Enqueue(CurrentPercentage); - } - } - } + Sample.Speed = LastValidSpeed; + }); UE_LOG(LogTemp, Log, TEXT("Finished calculating speed from movement")); } @@ -1230,6 +1211,8 @@ uint32 FProcessAgentSimulationDataRunnable:: Run() return 0; } + const double DeriveStart = FPlatformTime::Seconds(); + // Calculate rotation from movement if rotation data is missing if (!AgentSimulationData.Metadata.bHasRotationData) { @@ -1255,6 +1238,11 @@ uint32 FProcessAgentSimulationDataRunnable:: Run() CalculateSpeedFromMovement(); } + if (!AgentSimulationData.Metadata.bHasRotationData || !AgentSimulationData.Metadata.bHasSpeedData) + { + ImportTimings.DeriveSeconds = FPlatformTime::Seconds() - DeriveStart; + } + if (bShouldStop) { return 0; diff --git a/UnrealFolder/ProjectMobius/Source/ProjectMobius/Public/MassAI/SubSystems/AgentDataSubsystem.h b/UnrealFolder/ProjectMobius/Source/ProjectMobius/Public/MassAI/SubSystems/AgentDataSubsystem.h index 260029612..bfaf6ac8e 100644 --- a/UnrealFolder/ProjectMobius/Source/ProjectMobius/Public/MassAI/SubSystems/AgentDataSubsystem.h +++ b/UnrealFolder/ProjectMobius/Source/ProjectMobius/Public/MassAI/SubSystems/AgentDataSubsystem.h @@ -250,6 +250,8 @@ struct FMobiusImportTimings double ParseSeconds = 0.0; /** RunSimulationDataGatheringLoop: dense per-timestep map build + unit conversions. */ double ConvertSeconds = 0.0; + /** CalculateRotationFromMovement + CalculateSpeedFromMovement (positions-only sources; 0 otherwise). */ + double DeriveSeconds = 0.0; /** CalcSmoothedStepMovementBrackets. */ double BracketSeconds = 0.0; /** WriteCacheForImport (A3; includes the reuse-check when it skips). */ diff --git a/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/DerivedMotionTest.cpp b/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/DerivedMotionTest.cpp new file mode 100644 index 000000000..e1d7fb9a6 --- /dev/null +++ b/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/DerivedMotionTest.cpp @@ -0,0 +1,569 @@ +// Copyright (c) 2025 ProjectMobius contributors. Licensed under MIT. +// +// DerivedMotionTest.cpp +// +// Guards FProcessAgentSimulationDataRunnable::CalculateRotationFromMovement() and +// CalculateSpeedFromMovement(), which derive heading and speed for sources that carry positions only +// (every Juelich file, and Mobius-HDF5 written positions-only). Both were rewritten from a per-entity +// scan, O(entities x total samples), to one O(total samples) pass; no derived value may move by a bit. +// +// The oracle is the old scan itself, copied below from AgentDataSubsystem.cpp at 7dc1b050. Only how it +// reaches the map, MaxAgents and TimeBetweenSteps changed; logging, progress and cancellation were +// dropped. Every sample's Rotation and Speed get a unique sentinel first, so a write the old scan never +// made - or one it made that the new pass skips - shows up as a mismatch, not as an equal value. +// +// ProjectMobius.SimData.DerivedMotionParity - real Juelich export + a hand-built edge-case map +// Mobius.Timing.DerivedMotionScaling - old vs new wall-clock at two sizes. Set +// MOBIUS_DERIVED_MOTION_FILE to a positions-only file +// to time a real import as well. +// +// Run: MobiusPerf\RunTests.ps1 -Filter "ProjectMobius.SimData.DerivedMotion+Mobius.Timing.DerivedMotion" +// +#if !UE_BUILD_SHIPPING + +#include "CoreMinimal.h" +#include "HAL/IConsoleManager.h" +#include "HAL/PlatformMisc.h" +#include "HAL/PlatformTime.h" +#include "Misc/AutomationTest.h" +#include "Misc/Paths.h" +#include "MassAI/SubSystems/AgentDataSubsystem.h" +#include "MassAI/Fragments/SharedFragments/SimulationFragment.h" + +namespace DerivedMotionOracle +{ + using FSimMap = TMap>; + + // FProcessAgentSimulationDataRunnable::CalculateRotationFromMovement() as of 7dc1b050. + void CalculateRotationFromMovement(FSimMap& SimulationData, const int32 MaxAgents) + { + for (int32 EntityIdx = 0; EntityIdx < MaxAgents; ++EntityIdx) + { + TArray> EntityPositions; + + for (auto& Pair : SimulationData) + { + for (const FSimMovementSample& Sample : Pair.Value) + { + if (Sample.EntityID == EntityIdx) + { + EntityPositions.Add(TPair(Pair.Key, Sample.Position)); + break; + } + } + } + + if (EntityPositions.Num() < 2) continue; + + EntityPositions.Sort([](const TPair& A, const TPair& B) + { + return A.Key < B.Key; + }); + + FRotator LastValidRotation = FRotator::ZeroRotator; + + for (int32 i = 0; i < EntityPositions.Num(); ++i) + { + FRotator NewRotation = LastValidRotation; + + if (i < EntityPositions.Num() - 1) + { + FVector Delta = EntityPositions[i + 1].Value - EntityPositions[i].Value; + + if (!Delta.IsNearlyZero(0.1f)) + { + float YawRad = FMath::Atan2(Delta.Y, Delta.X); + float YawDeg = FMath::RadiansToDegrees(YawRad); + + NewRotation = FRotator(0.0f, YawDeg - 90.0f, 0.0f); + LastValidRotation = NewRotation; + } + } + + int32 Timestep = EntityPositions[i].Key; + for (FSimMovementSample& Sample : SimulationData[Timestep]) + { + if (Sample.EntityID == EntityIdx) + { + Sample.Rotation = NewRotation; + break; + } + } + } + } + } + + // FProcessAgentSimulationDataRunnable::CalculateSpeedFromMovement() as of 7dc1b050. + void CalculateSpeedFromMovement(FSimMap& SimulationData, const int32 MaxAgents, const float TimeBetweenSteps) + { + for (int32 EntityIdx = 0; EntityIdx < MaxAgents; ++EntityIdx) + { + TArray> EntityPositions; + + for (auto& Pair : SimulationData) + { + for (const FSimMovementSample& Sample : Pair.Value) + { + if (Sample.EntityID == EntityIdx) + { + EntityPositions.Add(TPair(Pair.Key, Sample.Position)); + break; + } + } + } + + if (EntityPositions.Num() < 2) continue; + + EntityPositions.Sort([](const TPair& A, const TPair& B) + { + return A.Key < B.Key; + }); + + float LastValidSpeed = 0.0f; + + for (int32 i = 0; i < EntityPositions.Num(); ++i) + { + float Speed = LastValidSpeed; + + if (i < EntityPositions.Num() - 1) + { + FVector Delta = EntityPositions[i + 1].Value - EntityPositions[i].Value; + float Distance = Delta.Size(); + + Speed = Distance / (TimeBetweenSteps * 100.0f); + LastValidSpeed = Speed; + } + + int32 Timestep = EntityPositions[i].Key; + for (FSimMovementSample& Sample : SimulationData[Timestep]) + { + if (Sample.EntityID == EntityIdx) + { + Sample.Speed = Speed; + break; + } + } + } + } + } +} + +// Named, not anonymous: helper names like DoubleBits/MakeSample already exist in other test files' +// anonymous namespaces, which would collide if this module is ever unity-built. +namespace DerivedMotionTestUtil +{ + using DerivedMotionOracle::FSimMap; + + const TCHAR* JuelichFixtureRelative = TEXT("MobiusDataImporter/Hdf5TestData/JuelichTestCases/trajectories_hdf5/030_c_56_h0.h5"); + + uint64 DoubleBits(const double Value) + { + uint64 Bits = 0; + FMemory::Memcpy(&Bits, &Value, sizeof(Bits)); + return Bits; + } + + uint32 FloatBits(const float Value) + { + uint32 Bits = 0; + FMemory::Memcpy(&Bits, &Value, sizeof(Bits)); + return Bits; + } + + bool RotatorsBitEqual(const FRotator& A, const FRotator& B) + { + return DoubleBits(A.Pitch) == DoubleBits(B.Pitch) && DoubleBits(A.Yaw) == DoubleBits(B.Yaw) && DoubleBits(A.Roll) == DoubleBits(B.Roll); + } + + bool VectorsBitEqual(const FVector& A, const FVector& B) + { + return DoubleBits(A.X) == DoubleBits(B.X) && DoubleBits(A.Y) == DoubleBits(B.Y) && DoubleBits(A.Z) == DoubleBits(B.Z); + } + + /** A Speed no derivation can produce (negative) and that is unique per sample. */ + float SentinelSpeed(const int32 SampleOrdinal) + { + return -1.0f - static_cast(SampleOrdinal); + } + + /** Overwrite every Rotation/Speed with a per-sample sentinel, so "not written" is observable. */ + void StampSentinels(FSimMap& Map) + { + int32 SampleOrdinal = 0; + for (TPair>& Pair : Map) + { + for (FSimMovementSample& Sample : Pair.Value) + { + Sample.Rotation = FRotator(1000.0 + SampleOrdinal, 2000.0 + SampleOrdinal, 3000.0 + SampleOrdinal); + Sample.Speed = SentinelSpeed(SampleOrdinal); + ++SampleOrdinal; + } + } + } + + /** Runs the shipping derivation on Map through a bare runnable; Run() is never called. */ + void DeriveWithRunnable(const FString& AnyExistingFile, UAgentDataSubsystem* Owner, const TSharedPtr& Map, + const int32 MaxAgents, const float TimeBetweenSteps) + { + // The constructor only checks the file exists; nothing reads it unless Run() is called. + FProcessAgentSimulationDataRunnable Runnable(AnyExistingFile, Owner, /*bAutoStartThread*/ false); + Runnable.MaxAgents = MaxAgents; + Runnable.TimeBetweenSteps = TimeBetweenSteps; + Runnable.AgentSimulationData.Metadata.bHasRotationData = false; + Runnable.AgentSimulationData.Metadata.bHasSpeedData = false; + Runnable.AgentMovementInfoData.SimulationData = Map; + Runnable.CalculateRotationFromMovement(); + Runnable.CalculateSpeedFromMovement(); + } + + void DeriveWithOracle(FSimMap& Map, const int32 MaxAgents, const float TimeBetweenSteps) + { + DerivedMotionOracle::CalculateRotationFromMovement(Map, MaxAgents); + DerivedMotionOracle::CalculateSpeedFromMovement(Map, MaxAgents, TimeBetweenSteps); + } + + struct FParity + { + int32 Samples = 0; + /** Samples the oracle wrote (Speed no longer its sentinel). */ + int32 Derived = 0; + int32 Mismatches = 0; + FString FirstMismatch; + }; + + /** Bit-for-bit comparison of two derivations of the same sentinel-stamped source map. */ + FParity CompareDerivations(const FSimMap& Oracle, const FSimMap& Candidate) + { + FParity Result; + auto NoteMismatch = [&Result](const FString& What) + { + if (Result.Mismatches++ == 0) + { + Result.FirstMismatch = What; + } + }; + + if (Oracle.Num() != Candidate.Num()) + { + NoteMismatch(FString::Printf(TEXT("timestep count %d vs %d"), Oracle.Num(), Candidate.Num())); + return Result; + } + + int32 SampleOrdinal = 0; + for (const TPair>& Pair : Oracle) + { + const TArray* CandidateSamples = Candidate.Find(Pair.Key); + if (!CandidateSamples || CandidateSamples->Num() != Pair.Value.Num()) + { + NoteMismatch(FString::Printf(TEXT("timestep %d missing or resized"), Pair.Key)); + SampleOrdinal += Pair.Value.Num(); + continue; + } + + for (int32 Index = 0; Index < Pair.Value.Num(); ++Index, ++SampleOrdinal) + { + const FSimMovementSample& A = Pair.Value[Index]; + const FSimMovementSample& B = (*CandidateSamples)[Index]; + ++Result.Samples; + if (FloatBits(A.Speed) != FloatBits(SentinelSpeed(SampleOrdinal))) + { + ++Result.Derived; + } + + if (A.EntityID != B.EntityID || !VectorsBitEqual(A.Position, B.Position) + || !RotatorsBitEqual(A.Rotation, B.Rotation) || FloatBits(A.Speed) != FloatBits(B.Speed)) + { + NoteMismatch(FString::Printf(TEXT("timestep %d sample %d entity %d: oracle yaw %.9g speed %.9g, candidate yaw %.9g speed %.9g"), + Pair.Key, Index, A.EntityID, A.Rotation.Yaw, A.Speed, B.Rotation.Yaw, B.Speed)); + } + } + } + return Result; + } + + /** Oracle vs shipping derivation on copies of Source (which is sentinel-stamped first). */ + FParity RunParity(FSimMap Source, const int32 MaxAgents, const float TimeBetweenSteps, + const FString& AnyExistingFile, UAgentDataSubsystem* Owner, + double* OutOracleSeconds = nullptr, double* OutCandidateSeconds = nullptr) + { + StampSentinels(Source); + FSimMap OracleMap = Source; + TSharedPtr CandidateMap = MakeShared(MoveTemp(Source)); + + const double OracleStart = FPlatformTime::Seconds(); + DeriveWithOracle(OracleMap, MaxAgents, TimeBetweenSteps); + const double OracleSeconds = FPlatformTime::Seconds() - OracleStart; + + const double CandidateStart = FPlatformTime::Seconds(); + DeriveWithRunnable(AnyExistingFile, Owner, CandidateMap, MaxAgents, TimeBetweenSteps); + const double CandidateSeconds = FPlatformTime::Seconds() - CandidateStart; + + if (OutOracleSeconds) { *OutOracleSeconds = OracleSeconds; } + if (OutCandidateSeconds) { *OutCandidateSeconds = CandidateSeconds; } + return CompareDerivations(OracleMap, *CandidateMap); + } + + /** FNV-1a over every sample's derived fields in timestep order - comparable across builds. */ + uint64 DigestDerivedValues(const FSimMap& Map) + { + uint64 Hash = 0xcbf29ce484222325ull; + auto Mix = [&Hash](const uint64 Value) + { + for (int32 Byte = 0; Byte < 8; ++Byte) + { + Hash ^= (Value >> (Byte * 8)) & 0xffull; + Hash *= 0x100000001b3ull; + } + }; + + TArray Timesteps; + Map.GetKeys(Timesteps); + Timesteps.Sort(); + for (const int32 Timestep : Timesteps) + { + for (const FSimMovementSample& Sample : Map.FindChecked(Timestep)) + { + Mix(static_cast(Timestep)); + Mix(static_cast(static_cast(Sample.EntityID))); + Mix(DoubleBits(Sample.Rotation.Pitch)); + Mix(DoubleBits(Sample.Rotation.Yaw)); + Mix(DoubleBits(Sample.Rotation.Roll)); + Mix(FloatBits(Sample.Speed)); + } + } + return Hash; + } + + FSimMovementSample MakeDerivedMotionSample(const int32 EntityId, const double X, const double Y) + { + FSimMovementSample Sample; + Sample.EntityID = EntityId; + Sample.Position = FVector(X, Y, 0.0); + return Sample; + } + + /** Pins both .msc cache cvars off, so an import always parses and derives, and never writes a cache. */ + struct FScopedSimCacheOff + { + IConsoleVariable* WriteCVar = IConsoleManager::Get().FindConsoleVariable(TEXT("mobius.SimCache.WriteOnImport")); + IConsoleVariable* FastReloadCVar = IConsoleManager::Get().FindConsoleVariable(TEXT("mobius.SimCache.FastReload")); + int32 SavedWrite = WriteCVar ? WriteCVar->GetInt() : 1; + int32 SavedFastReload = FastReloadCVar ? FastReloadCVar->GetInt() : 1; + + FScopedSimCacheOff() + { + if (WriteCVar) { WriteCVar->Set(0, ECVF_SetByCode); } + if (FastReloadCVar) { FastReloadCVar->Set(0, ECVF_SetByCode); } + } + + ~FScopedSimCacheOff() + { + if (WriteCVar) { WriteCVar->Set(SavedWrite, ECVF_SetByCode); } + if (FastReloadCVar) { FastReloadCVar->Set(SavedFastReload, ECVF_SetByCode); } + } + }; + + /** + * Positions-only map shaped like an evacuation: agents enter and leave at staggered times, so + * timestep arrays vary in length and each agent sits at a different index in each one. + */ + FSimMap MakeSyntheticPositionsOnly(const int32 NumAgents, const int32 NumTimesteps) + { + FSimMap Map; + for (int32 Timestep = 0; Timestep < NumTimesteps; ++Timestep) + { + TArray& Samples = Map.Add(Timestep); + for (int32 Agent = 0; Agent < NumAgents; ++Agent) + { + const int32 Enter = (Agent * 37) % (NumTimesteps / 3); + const int32 Leave = Enter + NumTimesteps / 2 + (Agent * 53) % (NumTimesteps / 2); + if (Timestep < Enter || Timestep >= Leave) + { + continue; + } + const double Heading = Agent * 0.7; + const double Walked = 12.0 * (Timestep - Enter); + Samples.Add(MakeDerivedMotionSample(Agent, 50.0 * Agent + Walked * FMath::Cos(Heading), Walked * FMath::Sin(Heading))); + } + } + return Map; + } +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FDerivedMotionParityTest, + "ProjectMobius.SimData.DerivedMotionParity", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::ProductFilter) + +bool FDerivedMotionParityTest::RunTest(const FString& Parameters) +{ + using namespace DerivedMotionTestUtil; + + const FString JuelichPath = FPaths::ConvertRelativePathToFull(FPaths::Combine(FPaths::ProjectPluginsDir(), JuelichFixtureRelative)); + if (!TestTrue(FString::Printf(TEXT("Juelich fixture present (%s)"), *JuelichPath), FPaths::FileExists(JuelichPath))) + { + return false; + } + + UAgentDataSubsystem* Owner = NewObject(); + + // ---------- 1. Real Juelich export through the real import ---------- + { + FScopedSimCacheOff CacheOff; + FProcessAgentSimulationDataRunnable Import(JuelichPath, Owner, /*bAutoStartThread*/ false); + Import.Run(); + + // A cache hit would skip derivation altogether and make every comparison below trivially true. + TestFalse(TEXT("import parsed the source, not a cache"), Import.ImportTimings.bUsedFastReload); + TestEqual(TEXT("file format"), static_cast(Import.AgentFileFormat), static_cast(EMobiusAgentFileFormat::JuelichHdf5)); + TestTrue(TEXT("import ran the derivation"), Import.ImportTimings.DeriveSeconds > 0.0); + TestTrue(TEXT("Juelich has agents"), Import.MaxAgents > 0); + + const FSimMap& Imported = *Import.AgentMovementInfoData.SimulationData; + AddInfo(FString::Printf(TEXT("Juelich 030_c_56_h0: %d agents, %d timesteps, TimeBetweenSteps %.6f s, derive %.4f s; derived-value digest 0x%016llx"), + Import.MaxAgents, Imported.Num(), Import.TimeBetweenSteps, Import.ImportTimings.DeriveSeconds, DigestDerivedValues(Imported))); + + const FParity Parity = RunParity(Imported, Import.MaxAgents, Import.TimeBetweenSteps, JuelichPath, Owner); + TestTrue(TEXT("Juelich: oracle derived something"), Parity.Derived > 0); + TestEqual(FString::Printf(TEXT("Juelich: bit-identical to the old scan (%d samples, %d derived; first mismatch: %s)"), + Parity.Samples, Parity.Derived, *Parity.FirstMismatch), Parity.Mismatches, 0); + } + + // ---------- 2. Edge cases a clean export never hits ---------- + { + const int32 MaxAgents = 6; + const float TimeBetweenSteps = 0.1f; + + // Keys inserted out of order: both derivations must walk timesteps by key, not by insertion. + FSimMap Map; + Map.Add(3, TArray{ + MakeDerivedMotionSample(3, 400.0, 0.0), + MakeDerivedMotionSample(0, 30.0, 30.0), + MakeDerivedMotionSample(1, 100.0, 0.0), // first move of a stationary-from-start agent: +X, yaw -90 + MakeDerivedMotionSample(6, 1.0, 1.0), // id == MaxAgents: never derived + }); + Map.Add(0, TArray{ + MakeDerivedMotionSample(0, 0.0, 0.0), + MakeDerivedMotionSample(1, 0.0, 0.0), + MakeDerivedMotionSample(3, 0.0, 0.0), + MakeDerivedMotionSample(4, 0.0, 0.0), + MakeDerivedMotionSample(-1, 5.0, 5.0), // negative id: never derived + MakeDerivedMotionSample(7, 2.0, 2.0), // id > MaxAgents: never derived + }); + Map.Add(4, TArray()); // empty timestep + Map.Add(2, TArray{ + MakeDerivedMotionSample(2, 9.0, 9.0), // the only sample of entity 2: never derived + MakeDerivedMotionSample(0, 20.0, 20.0), + MakeDerivedMotionSample(1, 0.0, 0.0), + }); + Map.Add(1, TArray{ + MakeDerivedMotionSample(1, 0.0, 0.0), + MakeDerivedMotionSample(3, 100.0, 0.0), + MakeDerivedMotionSample(0, 10.0, 10.0), + MakeDerivedMotionSample(3, -500.0, -500.0), // second sample of entity 3 in one timestep: only the first counts + MakeDerivedMotionSample(6, 3.0, 3.0), + }); + Map.Add(5, TArray{ + MakeDerivedMotionSample(1, 100.0, 0.05), // sub-0.1 cm step along +Y: heading holds at -90, not 0 + MakeDerivedMotionSample(4, 0.0, 300.0), // entity 4 skipped timesteps 1-4 + MakeDerivedMotionSample(0, 60.0, 30.0), // last move +X: yaw -90, carried onto this final frame + }); + + const FParity Parity = RunParity(Map, MaxAgents, TimeBetweenSteps, JuelichPath, Owner); + TestEqual(FString::Printf(TEXT("edge cases: bit-identical to the old scan (%d samples, %d derived; first mismatch: %s)"), + Parity.Samples, Parity.Derived, *Parity.FirstMismatch), Parity.Mismatches, 0); + // 0: 5 samples, 1: 5, 3: 3 (duplicate excluded), 4: 2. Entities 2, 5 and the out-of-range ids: none. + TestEqual(TEXT("edge cases: exactly the expected samples were derived"), Parity.Derived, 15); + + // The parity check above proves "same as before"; these pin down what "before" was. + TSharedPtr Derived = MakeShared(Map); + StampSentinels(*Derived); + DeriveWithRunnable(JuelichPath, Owner, Derived, MaxAgents, TimeBetweenSteps); + const FSimMovementSample& StationaryStart = (*Derived)[0][1]; + TestTrue(TEXT("stationary-from-start agent holds yaw 0 (not the importer's -90)"), RotatorsBitEqual(StationaryStart.Rotation, FRotator::ZeroRotator)); + TestEqual(TEXT("stationary-from-start agent speed 0"), StationaryStart.Speed, 0.0f); + const FSimMovementSample& Duplicate = (*Derived)[1][3]; + TestTrue(TEXT("duplicate sample left untouched"), Duplicate.Speed < 0.0f); + TestTrue(TEXT("single-sample entity left untouched"), (*Derived)[2][0].Speed < 0.0f); + TestTrue(TEXT("negative id left untouched"), (*Derived)[0][4].Speed < 0.0f); + TestTrue(TEXT("id > MaxAgents left untouched"), (*Derived)[0][5].Speed < 0.0f); + TestTrue(TEXT("id == MaxAgents left untouched"), (*Derived)[3][3].Speed < 0.0f && (*Derived)[1][4].Speed < 0.0f); + TestEqual(TEXT("look-ahead speed = |next - this| cm / (dt * 100)"), (*Derived)[0][0].Speed, + static_cast(FVector(10.0, 10.0, 0.0).Size()) / (TimeBetweenSteps * 100.0f)); + TestEqual(TEXT("last frame carries the previous heading"), (*Derived)[5][2].Rotation.Yaw, -90.0); + TestTrue(TEXT("last frame carries the previous heading (bitwise)"), RotatorsBitEqual((*Derived)[5][2].Rotation, (*Derived)[3][1].Rotation)); + TestEqual(TEXT("sub-threshold step holds the heading"), (*Derived)[3][2].Rotation.Yaw, -90.0); + TestTrue(TEXT("sub-threshold step holds the heading (bitwise)"), RotatorsBitEqual((*Derived)[3][2].Rotation, (*Derived)[2][2].Rotation)); + } + + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FDerivedMotionScalingTimingTest, + "Mobius.Timing.DerivedMotionScaling", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::ProductFilter) + +bool FDerivedMotionScalingTimingTest::RunTest(const FString& Parameters) +{ + using namespace DerivedMotionTestUtil; + + const FString JuelichPath = FPaths::ConvertRelativePathToFull(FPaths::Combine(FPaths::ProjectPluginsDir(), JuelichFixtureRelative)); + UAgentDataSubsystem* Owner = NewObject(); + const float TimeBetweenSteps = 0.1f; + + // Doubling the agents doubles the samples: the old scan should grow ~4x, the new pass ~2x. + const int32 NumTimesteps = 1000; + const int32 AgentCounts[] = { 500, 1000 }; + double OracleSeconds[2] = {}; + double CandidateSeconds[2] = {}; + for (int32 Size = 0; Size < 2; ++Size) + { + const FSimMap Map = MakeSyntheticPositionsOnly(AgentCounts[Size], NumTimesteps); + const FParity Parity = RunParity(Map, AgentCounts[Size], TimeBetweenSteps, JuelichPath, Owner, &OracleSeconds[Size], &CandidateSeconds[Size]); + TestEqual(FString::Printf(TEXT("%d agents: bit-identical (first mismatch: %s)"), AgentCounts[Size], *Parity.FirstMismatch), Parity.Mismatches, 0); + AddInfo(FString::Printf(TEXT("%d agents x %d timesteps (%d samples): old scan %.3f s, new pass %.4f s, %.0fx"), + AgentCounts[Size], NumTimesteps, Parity.Samples, OracleSeconds[Size], CandidateSeconds[Size], + CandidateSeconds[Size] > 0.0 ? OracleSeconds[Size] / CandidateSeconds[Size] : 0.0)); + } + AddInfo(FString::Printf(TEXT("2x agents -> old scan %.1fx slower, new pass %.1fx slower"), + OracleSeconds[0] > 0.0 ? OracleSeconds[1] / OracleSeconds[0] : 0.0, + CandidateSeconds[0] > 0.0 ? CandidateSeconds[1] / CandidateSeconds[0] : 0.0)); + + // Loose gate, as in MobiusTimingTests: observed is orders of magnitude, so background load can't flake it. + TestTrue(FString::Printf(TEXT("new pass at least 10x the old scan at %d agents (%.4f s vs %.3f s)"), AgentCounts[1], CandidateSeconds[1], OracleSeconds[1]), + CandidateSeconds[1] * 10.0 <= OracleSeconds[1]); + + // Optional: a real positions-only file, through the real import first. + const FString FilePath = FPlatformMisc::GetEnvironmentVariable(TEXT("MOBIUS_DERIVED_MOTION_FILE")); + if (FilePath.IsEmpty()) + { + AddInfo(TEXT("MOBIUS_DERIVED_MOTION_FILE not set - real-file timing skipped")); + return true; + } + if (!TestTrue(FString::Printf(TEXT("MOBIUS_DERIVED_MOTION_FILE exists (%s)"), *FilePath), FPaths::FileExists(FilePath))) + { + return false; + } + + FScopedSimCacheOff CacheOff; + FProcessAgentSimulationDataRunnable Import(FilePath, Owner, /*bAutoStartThread*/ false); + Import.Run(); + TestFalse(TEXT("file import parsed the source, not a cache"), Import.ImportTimings.bUsedFastReload); + TestFalse(TEXT("file is positions-only (no rotation)"), Import.AgentSimulationData.Metadata.bHasRotationData); + TestFalse(TEXT("file is positions-only (no speed)"), Import.AgentSimulationData.Metadata.bHasSpeedData); + + const FMobiusImportTimings& Timings = Import.ImportTimings; + const FSimMap& Imported = *Import.AgentMovementInfoData.SimulationData; + double FileOracleSeconds = 0.0; + double FileCandidateSeconds = 0.0; + const FParity Parity = RunParity(Imported, Import.MaxAgents, Import.TimeBetweenSteps, FilePath, Owner, &FileOracleSeconds, &FileCandidateSeconds); + TestEqual(FString::Printf(TEXT("file: bit-identical (first mismatch: %s)"), *Parity.FirstMismatch), Parity.Mismatches, 0); + AddInfo(FString::Printf(TEXT("%s: %d agents, %d timesteps, %d samples. Import %.2f s (parse %.2f, convert %.2f, derive %.3f, brackets %.2f). Derivation on the imported map: old scan %.2f s, new pass %.3f s, %.0fx"), + *FPaths::GetCleanFilename(FilePath), Import.MaxAgents, Imported.Num(), Parity.Samples, + Timings.TotalSeconds, Timings.ParseSeconds, Timings.ConvertSeconds, Timings.DeriveSeconds, Timings.BracketSeconds, + FileOracleSeconds, FileCandidateSeconds, FileCandidateSeconds > 0.0 ? FileOracleSeconds / FileCandidateSeconds : 0.0)); + return true; +} + +#endif // !UE_BUILD_SHIPPING From a3a08badcdc6ddf9eab198d664a119aedf0bdf25 Mon Sep 17 00:00:00 2001 From: Nicholas Harding <40708936+sir306@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:21:23 +1300 Subject: [PATCH 2/6] Stop the log window and FrameGrabber slowing long sessions With the log window open, every incoming line rebuilt the whole backlog (up to 1000 lines) into one string and re-laid out a single word-wrapped text block, and a flush delivered its lines one at a time. Once the backlog was full, each frame that received a line cost ~30 ms; a 3,600-line heatmap burst ~98 ms. The log file itself is only ever appended to, never read back. - Log window: a virtualized SListView with one row per line, so only visible rows are laid out and appending is O(1); old lines are trimmed in batches (1000 stays the cap). It follows the newest line unless the user scrolls up. Close/destroy ordering is unchanged. - Feedback log cache: a ring buffer instead of RemoveAt(0); GetCachedLogLines() still returns oldest-first. - FrameGrabber: log a failure only when its reason changes and retry every 0.5 s instead of every frame. A 0x0 viewport (headless runs) had produced over a million identical log lines. Screenshot capture still initialises immediately and the resize re-init is unchanged. Measured with the window open and a full backlog: one line per frame 30.2 ms -> ~0.3 ms; 250-line flush 34.7 ms -> ~2 ms; 3,600-line flush 97.7 ms -> ~3 ms. FrameGrabber failure lines in a headless run: 1,805 -> 1. Tests: Mobius.RenderedTiming.LogWindow.* (needs a renderer, kept out of Mobius.Timing.) and in-game tests for the FrameGrabber spam and cache order. AI-assisted with Claude Code; all changes reviewed, tested, and verified by a human maintainer in line with this repository's AI-tooling policy. --- .../MobiusUserFeedbackSubsystem.cpp | 35 +- .../Private/Util/FrameGrabberHelper.cpp | 90 ++- .../Subsystems/MobiusUserFeedbackSubsystem.h | 8 + .../Public/Util/FrameGrabberHelper.h | 20 + .../Private/Logging/LogWindow.cpp | 193 ++++-- .../MobiusWidgets/Public/Logging/LogWindow.h | 82 ++- .../Private/Tests/LogWindowPerfTest.cpp | 594 ++++++++++++++++++ .../Private/Tests/MobiusLoggerInGameTest.cpp | 240 +++++++ 8 files changed, 1180 insertions(+), 82 deletions(-) create mode 100644 UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/LogWindowPerfTest.cpp create mode 100644 UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/MobiusLoggerInGameTest.cpp diff --git a/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Subsystems/MobiusUserFeedbackSubsystem.cpp b/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Subsystems/MobiusUserFeedbackSubsystem.cpp index 9a2658a08..19eaeb455 100644 --- a/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Subsystems/MobiusUserFeedbackSubsystem.cpp +++ b/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Subsystems/MobiusUserFeedbackSubsystem.cpp @@ -35,6 +35,7 @@ void UMobiusUserFeedbackSubsystem::Deinitialize() PendingErrors.Reset(); CachedLogLines.Reset(); + CachedLogHead = 0; RecentPromptTimes.Reset(); Super::Deinitialize(); @@ -226,7 +227,24 @@ TArray UMobiusUserFeedbackSubsystem::DrainPendingErrors() TArray UMobiusUserFeedbackSubsystem::GetCachedLogLines() const { - return CachedLogLines; + // Oldest first, which is what the log window's SetLines expects. The head only moves once the ring + // is full, so a zero head means the buffer is still in plain append order. + if (CachedLogHead == 0) + { + return CachedLogLines; + } + + TArray Ordered; + Ordered.Reserve(CachedLogLines.Num()); + for (int32 Index = CachedLogHead; Index < CachedLogLines.Num(); ++Index) + { + Ordered.Add(CachedLogLines[Index]); + } + for (int32 Index = 0; Index < CachedLogHead; ++Index) + { + Ordered.Add(CachedLogLines[Index]); + } + return Ordered; } FOnMobiusErrorReported& UMobiusUserFeedbackSubsystem::OnErrorReported() @@ -310,11 +328,18 @@ void UMobiusUserFeedbackSubsystem::ReportErrorInternal(const FMobiusErrorMessage void UMobiusUserFeedbackSubsystem::HandleLogLine(const FString& Line) { - CachedLogLines.Add(Line); - if (CachedLogLines.Num() > MaxCachedLogLines) + if (MaxCachedLogLines > 0) { - const int32 Overflow = CachedLogLines.Num() - MaxCachedLogLines; - CachedLogLines.RemoveAt(0, Overflow, EAllowShrinking::No); + if (CachedLogLines.Num() < MaxCachedLogLines) + { + CachedLogLines.Add(Line); + } + else + { + // Full: overwrite the oldest slot in place (reuses its allocation) and advance the head. + CachedLogLines[CachedLogHead] = Line; + CachedLogHead = (CachedLogHead + 1) % MaxCachedLogLines; + } } LogLineDelegate.Broadcast(Line); diff --git a/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Util/FrameGrabberHelper.cpp b/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Util/FrameGrabberHelper.cpp index 258aa37fc..0c7cbff89 100644 --- a/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Util/FrameGrabberHelper.cpp +++ b/UnrealFolder/ProjectMobius/Source/MobiusCore/Private/Util/FrameGrabberHelper.cpp @@ -172,20 +172,20 @@ bool UFrameGrabberHelper::TryInitialize() if (!GEngine) { - MobiusLog(TEXT("[FrameGrabber] TryInitialize failed: GEngine is null")); + NoteInitFailure(TEXT("GEngine is null")); return false; } if (!GEngine->GameViewport) { - MobiusLog(TEXT("[FrameGrabber] TryInitialize failed: GameViewport is null")); + NoteInitFailure(TEXT("GameViewport is null")); return false; } TSharedPtr GameViewportWidget = GEngine->GameViewport->GetGameViewportWidget(); if (!GameViewportWidget.IsValid()) { - MobiusLog(TEXT("[FrameGrabber] TryInitialize failed: GameViewportWidget not constructed yet")); + NoteInitFailure(TEXT("GameViewportWidget not constructed yet")); return false; } @@ -193,7 +193,7 @@ bool UFrameGrabberHelper::TryInitialize() TSharedPtr SlateViewportInterface = GameViewportWidget->GetViewportInterface().Pin(); if (!SlateViewportInterface.IsValid()) { - MobiusLog(TEXT("[FrameGrabber] TryInitialize failed: SlateViewportInterface not wired up yet")); + NoteInitFailure(TEXT("SlateViewportInterface not wired up yet")); return false; } @@ -201,13 +201,16 @@ bool UFrameGrabberHelper::TryInitialize() TSharedPtr SceneViewport = StaticCastSharedPtr(SlateViewportInterface); if (!SceneViewport.IsValid()) { - MobiusLog(TEXT("[FrameGrabber] TryInitialize failed: Unexpected viewport type (not FSceneViewport)")); - MobiusReportError( - FText::FromString(TEXT("Viewport Error")), - FText::FromString(TEXT("Could not access scene viewport for screenshot capture.")), - FText::FromString(TEXT("FrameGrabberHelper::TryInitialize")), - EMobiusErrorSeverity::Warning, - false); // Don't show popup for this, it may resolve on retry + // Escalate once per reason, not per retry: ReportError UE_LOGs and re-enqueues into the custom log too. + if (NoteInitFailure(TEXT("Unexpected viewport type (not FSceneViewport)"))) + { + MobiusReportError( + FText::FromString(TEXT("Viewport Error")), + FText::FromString(TEXT("Could not access scene viewport for screenshot capture.")), + FText::FromString(TEXT("FrameGrabberHelper::TryInitialize")), + EMobiusErrorSeverity::Warning, + false); // Don't show popup for this, it may resolve on retry + } return false; } @@ -217,8 +220,7 @@ bool UFrameGrabberHelper::TryInitialize() // If either dimension is 0, we can't capture yet if (ViewportSize.X <= 0 || ViewportSize.Y <= 0) { - MobiusLog(FString::Printf(TEXT("[FrameGrabber] TryInitialize failed: Invalid viewport size %dx%d"), - ViewportSize.X, ViewportSize.Y)); + NoteInitFailure(FString::Printf(TEXT("Invalid viewport size %dx%d"), ViewportSize.X, ViewportSize.Y)); return false; } @@ -227,23 +229,51 @@ bool UFrameGrabberHelper::TryInitialize() // Clamp/validate target too (avoid passing nonsense to frame grabber) if (TargetSize.X <= 0 || TargetSize.Y <= 0) { - MobiusLog(FString::Printf(TEXT("[FrameGrabber] TryInitialize failed: Invalid target size %dx%d"), - TargetSize.X, TargetSize.Y)); - MobiusReportError( - FText::FromString(TEXT("Configuration Error")), - FText::FromString(FString::Printf(TEXT("Invalid screenshot target size: %dx%d"), TargetSize.X, TargetSize.Y)), - FText::FromString(TEXT("FrameGrabberHelper::TryInitialize")), - EMobiusErrorSeverity::Error, - true); + if (NoteInitFailure(FString::Printf(TEXT("Invalid target size %dx%d"), TargetSize.X, TargetSize.Y))) + { + MobiusReportError( + FText::FromString(TEXT("Configuration Error")), + FText::FromString(FString::Printf(TEXT("Invalid screenshot target size: %dx%d"), TargetSize.X, TargetSize.Y)), + FText::FromString(TEXT("FrameGrabberHelper::TryInitialize")), + EMobiusErrorSeverity::Error, + true); + } return false; } FrameGrabber = MakeUnique(SceneViewport.ToSharedRef(), ViewportSize); FrameGrabber->StartCapturingFrames(); - MobiusLog(FString::Printf(TEXT("[FrameGrabber] Initialized successfully. ViewportSize: %dx%d, TargetSize: %dx%d"), - ViewportSize.X, ViewportSize.Y, TargetSize.X, TargetSize.Y)); + const FString QuietRetries = SuppressedInitFailures > 0 + ? FString::Printf(TEXT(" (after %d unlogged repeats of: %s)"), SuppressedInitFailures, *LastInitFailureReason) + : FString(); + MobiusLog(FString::Printf(TEXT("[FrameGrabber] Initialized successfully. ViewportSize: %dx%d, TargetSize: %dx%d%s"), + ViewportSize.X, ViewportSize.Y, TargetSize.X, TargetSize.Y, *QuietRetries)); + LastInitFailureReason.Reset(); + SuppressedInitFailures = 0; + + return true; +} + +bool UFrameGrabberHelper::NoteInitFailure(const FString& Reason) +{ + // Tick retries until the viewport is usable, and in editor / -game -nullrhi / headless automation runs + // it stays 0x0 for the whole session. Logging every attempt put one identical line per frame into + // MobiusCustomLog.txt - up to ~3,000 a second, 1.07 M of its 1.26 M lines - and each one went through + // the log window too. A reason is logged once, when it changes; repeats are only counted. + if (Reason == LastInitFailureReason) + { + ++SuppressedInitFailures; + return false; + } + const FString PreviousRepeats = SuppressedInitFailures > 0 + ? FString::Printf(TEXT(" - previous reason '%s' repeated %d more times"), *LastInitFailureReason, SuppressedInitFailures) + : FString(); + MobiusLog(FString::Printf(TEXT("[FrameGrabber] TryInitialize failed: %s (retrying quietly; logged again only if the reason changes)%s"), + *Reason, *PreviousRepeats)); + LastInitFailureReason = Reason; + SuppressedInitFailures = 0; return true; } #endif // !PLATFORM_MAC @@ -384,7 +414,9 @@ void UFrameGrabberHelper::TriggerCapture(const FString& InFileName) MobiusLog(TEXT("[FrameGrabber] FrameGrabber not initialized, attempting initialization...")); if (!TryInitialize()) { - MobiusLog(TEXT("[FrameGrabber] Error: Could not initialize FrameGrabber - viewport not ready.")); + // TryInitialize may have stayed quiet (the same reason Tick already logged), so name it here. + MobiusLog(FString::Printf(TEXT("[FrameGrabber] Error: Could not initialize FrameGrabber - viewport not ready (%s)."), + *LastInitFailureReason)); MobiusReportError( FText::FromString(TEXT("Initialization Failed")), FText::FromString(TEXT("Could not initialize screenshot capture. The viewport may not be ready yet.")), @@ -548,10 +580,16 @@ void UFrameGrabberHelper::Tick(float DeltaTime) // The OnScreenshotCaptured callback handles everything #else // Windows/other: Poll FFrameGrabber - // Try lazy initialization if not yet initialized + // Try lazy initialization if not yet initialized. Backed off while it keeps failing: under -nullrhi / + // headless the viewport is 0x0 for the whole session, and a retry every frame bought nothing. + // TriggerCapture still initializes on demand without waiting for this. if (!FrameGrabber.IsValid()) { - TryInitialize(); + InitRetryCooldown -= DeltaTime; + if (InitRetryCooldown <= 0.0f && !TryInitialize()) + { + InitRetryCooldown = InitRetryIntervalSeconds; + } } if (!bIsCapturing || !FrameGrabber.IsValid()) diff --git a/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Subsystems/MobiusUserFeedbackSubsystem.h b/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Subsystems/MobiusUserFeedbackSubsystem.h index cbbc781ec..a1537a45c 100644 --- a/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Subsystems/MobiusUserFeedbackSubsystem.h +++ b/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Subsystems/MobiusUserFeedbackSubsystem.h @@ -92,7 +92,15 @@ class MOBIUSCORE_API UMobiusUserFeedbackSubsystem : public UGameInstanceSubsyste bool bLogWindowOpen = false; TArray PendingErrors; + + /** + * Ring buffer of the newest MaxCachedLogLines lines, so HandleLogLine is O(1) instead of shifting the + * whole cache down on every line. Plain append order until it fills; from then on CachedLogHead is the + * slot of the OLDEST line, i.e. the next one to be overwritten. GetCachedLogLines unrolls it. + */ TArray CachedLogLines; + int32 CachedLogHead = 0; + TMap RecentPromptTimes; FDelegateHandle LogLineHandle; diff --git a/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Util/FrameGrabberHelper.h b/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Util/FrameGrabberHelper.h index 4c7ca0fbd..cdec2fd18 100644 --- a/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Util/FrameGrabberHelper.h +++ b/UnrealFolder/ProjectMobius/Source/MobiusCore/Public/Util/FrameGrabberHelper.h @@ -48,7 +48,27 @@ class MOBIUSCORE_API UFrameGrabberHelper : public UObject /** Attempt to initialize the frame grabber from the game engine's scene viewport */ bool TryInitialize(); + /** + * Log a TryInitialize failure only when its reason differs from the last one logged; repeats are + * counted instead. Returns true if this call logged, so the caller escalates (ReportError) once per + * reason rather than once per retry. + */ + bool NoteInitFailure(const FString& Reason); + TUniquePtr FrameGrabber; + + /** Reason of the last TryInitialize failure that was logged. Cleared by a successful init, so a + * failure after a later (resize) re-init logs again. */ + FString LastInitFailureReason; + + /** Identical failures swallowed since LastInitFailureReason was logged. */ + int32 SuppressedInitFailures = 0; + + /** Seconds until Tick may retry TryInitialize. Tick-side only: TriggerCapture never waits on it. */ + float InitRetryCooldown = 0.0f; + + /** Tick's retry period while the viewport is unusable (0x0 for a whole -nullrhi / headless session). */ + static constexpr float InitRetryIntervalSeconds = 0.5f; #endif #if PLATFORM_MAC diff --git a/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Private/Logging/LogWindow.cpp b/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Private/Logging/LogWindow.cpp index 1c2e713f1..3f6769365 100644 --- a/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Private/Logging/LogWindow.cpp +++ b/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Private/Logging/LogWindow.cpp @@ -13,9 +13,10 @@ #include "Engine/GameInstance.h" #include "Widgets/Input/SButton.h" #include "Widgets/Layout/SBorder.h" -#include "Widgets/Layout/SScrollBox.h" #include "Widgets/SBoxPanel.h" #include "Widgets/Text/STextBlock.h" +#include "Widgets/Views/SListView.h" +#include "Widgets/Views/STableRow.h" namespace { @@ -83,26 +84,25 @@ void SLogWindowWidget::AppendLine(const FString& Line) return; } - LogLines.Add(Line); - if (LogLines.Num() > MaxLines) - { - const int32 Overflow = LogLines.Num() - MaxLines; - LogLines.RemoveAt(0, Overflow, EAllowShrinking::No); - } + // O(1) per line whatever the backlog. The logger's flush calls this once per queued line, so a + // heatmap load arrives as thousands of calls in one frame; all of them together cost one list + // regeneration (visible rows only) on the next paint. + LogItems.Add(MakeShared(Line)); + const int32 Removed = LogItems.Num() > MaxLines ? TrimBacklog() : 0; - RebuildLogText(); + RefreshLogList(Removed, /*bForceToEnd=*/false); } void SLogWindowWidget::SetLines(const TArray& Lines) { - LogLines = Lines; - if (LogLines.Num() > MaxLines) + const int32 First = FMath::Max(0, Lines.Num() - MaxLines); + LogItems.Reset(Lines.Num() - First); + for (int32 Index = First; Index < Lines.Num(); ++Index) { - const int32 Overflow = LogLines.Num() - MaxLines; - LogLines.RemoveAt(0, Overflow, EAllowShrinking::No); + LogItems.Add(MakeShared(Lines[Index])); } - RebuildLogText(); + RefreshLogList(/*RemovedFromFront=*/0, /*bForceToEnd=*/true); } void SLogWindowWidget::ShowLogWindow() @@ -148,13 +148,22 @@ void SLogWindowWidget::ReleaseWindowState() // Unconditional: the binding outlives the window pointer, and OpenLogWindow re-binds on reopen. UnbindThemeChanged(); - // ScrollBox / LogTextBlock / CloseButton live inside the destroyed window's tree and are rebuilt by - // OpenLogWindow. LogLines and LogText are NOT cleared: they are the viewer's contents, and - // OpenLogWindow seeds the new STextBlock from LogText, so a reopen shows the same backlog. + // LogListView holds a raw pointer to LogItems. Detach it before letting go: on the OnWindowClosed + // route this widget - and LogItems with it - is destroyed inside the callback while the window's tree + // is still alive, so a list left pointing at LogItems would outlive the array it reads. + if (LogListView.IsValid()) + { + LogListView->ClearItemsSource(); + } + + // LogListView / CloseButton live inside the destroyed window's tree and are rebuilt by + // OpenLogWindow. LogItems is NOT cleared: it is the viewer's contents, and OpenLogWindow builds the + // new list over it, so a reopen shows the same backlog. LogWindowPtr.Reset(); - ScrollBox.Reset(); - LogTextBlock.Reset(); + LogListView.Reset(); CloseButton.Reset(); + LogSettleTimer.Reset(); + LogSettleFramesLeft = 0; } void SLogWindowWidget::SetEnabled(bool bEnabled) @@ -195,9 +204,27 @@ void SLogWindowWidget::OpenLogWindow() { LogFont = FSlateFontInfo(Inter, 10, FName("Regular")); } - const FTextBlockStyle LogTextStyle = FTextBlockStyle(FCoreStyle::Get().GetWidgetStyle("NormalText")) + LogTextStyle = FTextBlockStyle(FCoreStyle::Get().GetWidgetStyle("NormalText")) .SetFont(LogFont); + // Rows and list draw nothing of their own - no hover, selection or alternating-row brushes, no list + // background - so the themed RibbonBg border shows through exactly as it did behind the old single + // text block. + const FSlateNoResource NoBrush; + LogRowStyle = FTableRowStyle(FCoreStyle::Get().GetWidgetStyle("TableView.Row")) + .SetSelectorFocusedBrush(NoBrush) + .SetActiveHoveredBrush(NoBrush) + .SetActiveBrush(NoBrush) + .SetInactiveHoveredBrush(NoBrush) + .SetInactiveBrush(NoBrush) + .SetEvenRowBackgroundHoveredBrush(NoBrush) + .SetEvenRowBackgroundBrush(NoBrush) + .SetOddRowBackgroundHoveredBrush(NoBrush) + .SetOddRowBackgroundBrush(NoBrush) + .SetActiveHighlightedBrush(NoBrush) + .SetInactiveHighlightedBrush(NoBrush); + LogListViewStyle = FTableViewStyle().SetBackgroundBrush(NoBrush); + // Themed Close button (stored as a member: SButton caches raw pointers into the style's brushes). // Background brushes are stamped for the theme current at construct; the label colour follows live. CloseButtonStyle = MobiusWindowButtonStyle::MakeWindowButtonStyle(FindThemeSubsystemForLogWindow()); @@ -216,19 +243,12 @@ void SLogWindowWidget::OpenLogWindow() + SVerticalBox::Slot() .FillHeight(1.0f) [ - SAssignNew(ScrollBox, SScrollBox) - + SScrollBox::Slot() - [ - SAssignNew(LogTextBlock, STextBlock) - .Text(FText::FromString(LogText)) - .AutoWrapText(true) - .TextStyle(&LogTextStyle) - // Mono log text follows the theme (readable on either surface) via a live poll. - .ColorAndOpacity_Lambda([]() - { - return FSlateColor(PollLogWindowColor(EMobiusPaletteRole::InputMonoText, MobiusThemePalette::Color(EMobiusPaletteRole::InputMonoText, /*bLight=*/true))); - }) - ] + SAssignNew(LogListView, SListView>) + .ListItemsSource(&LogItems) + .OnGenerateRow(this, &SLogWindowWidget::GenerateLogRow) + .OnListViewScrolled(this, &SLogWindowWidget::HandleLogListScrolled) + .SelectionMode(ESelectionMode::None) + .ListViewStyle(&LogListViewStyle) ] + SVerticalBox::Slot() .AutoHeight() @@ -253,6 +273,9 @@ void SLogWindowWidget::OpenLogWindow() ] ]; + // A fresh list starts at the top; open on the newest line, as the old text block's ScrollToEnd did. + RefreshLogList(/*RemovedFromFront=*/0, /*bForceToEnd=*/true); + // Live re-theme for the Close button, event-driven off OnThemeChangedNative. The bind normally succeeds // right here; if it does not, fall back to a 2 Hz bootstrap timer that stops the moment it binds. // @@ -298,26 +321,108 @@ FReply SLogWindowWidget::HandleCloseClicked() return FReply::Handled(); } -void SLogWindowWidget::RebuildLogText() +TSharedRef SLogWindowWidget::GenerateLogRow(TSharedPtr Line, const TSharedRef& OwnerTable) +{ + return SNew(STableRow>, OwnerTable) + .Style(&LogRowStyle) + .ShowSelection(false) + [ + SNew(STextBlock) + .Text(FText::FromString(Line.IsValid() ? *Line : FString())) + .AutoWrapText(true) + .TextStyle(&LogTextStyle) + // Mono log text follows the theme (readable on either surface) via a live poll. + .ColorAndOpacity_Lambda([]() + { + return FSlateColor(PollLogWindowColor(EMobiusPaletteRole::InputMonoText, MobiusThemePalette::Color(EMobiusPaletteRole::InputMonoText, /*bLight=*/true))); + }) + ]; +} + +int32 SLogWindowWidget::TrimBacklog() { - LogText.Reset(); - for (int32 Index = 0; Index < LogLines.Num(); ++Index) + const int32 Keep = FMath::Max(0, MaxLines - LogTrimBatch); + const int32 Removed = FMath::Max(0, LogItems.Num() - Keep); + if (Removed > 0) { - LogText.Append(LogLines[Index]); - if (Index + 1 < LogLines.Num()) - { - LogText.Append(TEXT("\n")); - } + LogItems.RemoveAt(0, Removed, EAllowShrinking::No); + } + return Removed; +} + +void SLogWindowWidget::RefreshLogList(int32 RemovedFromFront, bool bForceToEnd) +{ + // Closed: nothing to refresh. OpenLogWindow builds a fresh list over LogItems. + if (!LogListView.IsValid()) + { + return; } - if (LogTextBlock.IsValid()) + if (bForceToEnd) { - LogTextBlock->SetText(FText::FromString(LogText)); + bFollowLogEnd = true; } + else if (!bFollowLogEnd && !LogListView->IsUserScrolling() + && LogListView->GetScrollDistanceRemaining().Y <= KINDA_SMALL_NUMBER) + { + // The reader scrolled back down to the end (scrollbar state from the list's last layout): follow again. + bFollowLogEnd = true; + } + + LogListView->RequestListRefresh(); + + if (bFollowLogEnd) + { + PinLogListToEnd(); + ArmLogListSettle(); + } + else if (RemovedFromFront > 0) + { + // The scroll offset is an item index: shift it with the trim so the lines being read stay put. + TGuardValue OwnScroll(bScrollingLogListProgrammatically, true); + LogListView->SetScrollOffset(FMath::Max(0.0f, LogListView->GetScrollOffset() - static_cast(RemovedFromFront))); + } +} + +void SLogWindowWidget::PinLogListToEnd() +{ + if (LogListView.IsValid()) + { + TGuardValue OwnScroll(bScrollingLogListProgrammatically, true); + LogListView->ScrollToBottom(); + } +} - if (ScrollBox.IsValid()) +void SLogWindowWidget::ArmLogListSettle() +{ + // Three frames: the one that generates the new rows (measured unwrapped), the one that re-measures + // them wrapped, and one spare. A burst re-arms rather than stacking timers. + LogSettleFramesLeft = 3; + if (!LogSettleTimer.IsValid() && LogListView.IsValid()) + { + LogSettleTimer = LogListView->RegisterActiveTimer(0.0f, FWidgetActiveTimerDelegate::CreateSP(this, &SLogWindowWidget::SettleLogListEnd)); + } +} + +EActiveTimerReturnType SLogWindowWidget::SettleLogListEnd(double InCurrentTime, float InDeltaTime) +{ + // Runs in the list's Paint before its Tick, so the re-pin is laid out in this same frame. + if (!bFollowLogEnd || !LogListView.IsValid() || LogSettleFramesLeft <= 0) + { + LogSettleFramesLeft = 0; + return EActiveTimerReturnType::Stop; + } + + --LogSettleFramesLeft; + PinLogListToEnd(); + return LogSettleFramesLeft > 0 ? EActiveTimerReturnType::Continue : EActiveTimerReturnType::Stop; +} + +void SLogWindowWidget::HandleLogListScrolled(double InScrollOffset) +{ + if (!bScrollingLogListProgrammatically && FSlateApplication::IsInitialized() && FSlateApplication::Get().IsProcessingInput()) { - ScrollBox->ScrollToEnd(); + bFollowLogEnd = false; } } diff --git a/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Public/Logging/LogWindow.h b/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Public/Logging/LogWindow.h index 0ddc1b98b..4fea528fa 100644 --- a/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Public/Logging/LogWindow.h +++ b/UnrealFolder/ProjectMobius/Source/MobiusWidgets/Public/Logging/LogWindow.h @@ -6,11 +6,13 @@ #include "Widgets/SCompoundWidget.h" #include "Styling/SlateTypes.h" +class FActiveTimerHandle; +class ITableRow; class SButton; class SMoveableWindow; -class SScrollBox; -class STextBlock; +class STableViewBase; class UUIThemeSubsystem; +template class SListView; enum class EMobiusUITheme : uint8; DECLARE_DELEGATE(FOnLogWindowClosed); @@ -49,7 +51,41 @@ class MOBIUSWIDGETS_API SLogWindowWidget final : public SCompoundWidget private: void OpenLogWindow(); FReply HandleCloseClicked(); - void RebuildLogText(); + + /** Row factory for LogListView: one STextBlock per line, built only for rows on screen. */ + TSharedRef GenerateLogRow(TSharedPtr Line, const TSharedRef& OwnerTable); + + /** Drop the oldest lines down to MaxLines - LogTrimBatch. Returns how many were removed. */ + int32 TrimBacklog(); + + /** + * Tell the list its items changed. Cheap and idempotent - the list regenerates once, on its next + * paint, for the visible rows only, however many lines arrived in between. Follows the end of the + * log unless the reader has scrolled away from it (bForceToEnd overrides that); a reader who has + * scrolled up keeps their place even when RemovedFromFront lines were trimmed above them. + */ + void RefreshLogList(int32 RemovedFromFront, bool bForceToEnd); + + /** ScrollToBottom, flagged as ours so HandleLogListScrolled does not read it as the reader's. */ + void PinLogListToEnd(); + + /** Start (or extend) the SettleLogListEnd timer on LogListView. */ + void ArmLogListSettle(); + + /** + * Re-pins the end for a few frames after a change. A new row's text is measured before it has a wrap + * width (AutoWrapText learns it from the row's first paint), so the frame that adds wrapping lines + * fits too many rows and the newest ones slide below the fold once they wrap. Registered on + * LogListView, which is painted - this widget never is. + */ + EActiveTimerReturnType SettleLogListEnd(double InCurrentTime, float InDeltaTime); + + /** + * Bound to the list's OnListViewScrolled. Only the reader's own scroll input (wheel, thumb drag, + * keys) stops the follow: our own scrolls are flagged, and the list's end-of-list clamp fires this + * from inside its Tick, i.e. during paint, never while Slate is processing input. + */ + void HandleLogListScrolled(double InScrollOffset); /** * Re-theme the Close button for the current theme. Event-driven: bound to @@ -93,13 +129,45 @@ class MOBIUSWIDGETS_API SLogWindowWidget final : public SCompoundWidget FOnLogWindowClosed OnLogWindowClosed; TSharedPtr LogWindowPtr; - TSharedPtr ScrollBox; - TSharedPtr LogTextBlock; - FString LogText; - TArray LogLines; + + /** + * Virtualized: one row per line, laid out only while on screen. This used to be a single STextBlock + * holding the whole backlog, rebuilt and re-laid out on EVERY appended line - with a full 1000-line + * backlog that is ~90 KB of text per line, and the logger delivers its queue one line at a time. + */ + TSharedPtr>> LogListView; + + /** The viewer's contents, oldest first; LogListView's items source. Survives a close/reopen. */ + TArray> LogItems; + + /** Hard cap on LogItems. */ int32 MaxLines = 1000; + + /** + * Trim granularity: overflowing MaxLines drops this many extra lines at once, so the front-of-array + * memmove is paid once per batch instead of once per appended line. + */ + int32 LogTrimBatch = 128; + + /** True while the view follows the newest line. Cleared only by the reader scrolling; restored when + * they scroll back to the end, and by every (re)open. */ + bool bFollowLogEnd = true; + + /** Set around our own scroll calls, so HandleLogListScrolled can tell them from the reader's. */ + bool bScrollingLogListProgrammatically = false; + + /** Frames SettleLogListEnd still has to re-pin, and its timer (owned by LogListView, dies with it). */ + int32 LogSettleFramesLeft = 0; + TWeakPtr LogSettleTimer; + bool bIsEnabled = true; FWindowStyle LogWindowStyle; + + /** Members, not locals: OnGenerateRow builds rows long after OpenLogWindow returns, and STableRow / + * SListView keep a pointer to the style they were given. */ + FTextBlockStyle LogTextStyle; + FTableRowStyle LogRowStyle; + FTableViewStyle LogListViewStyle; /** Themed Close-button style (member so SButton's cached brush pointers stay valid). */ FButtonStyle CloseButtonStyle; diff --git a/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/LogWindowPerfTest.cpp b/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/LogWindowPerfTest.cpp new file mode 100644 index 000000000..9d7bcf6b6 --- /dev/null +++ b/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/LogWindowPerfTest.cpp @@ -0,0 +1,594 @@ +// Copyright (c) 2026 ProjectMobius contributors. Licensed under MIT. +// +// LogWindowPerfTest.cpp +// +// Frame cost of the Mobius log window (SLogWindowWidget) once its backlog is full - the "Mobius gets +// slow after 30+ minutes with the log window open" report. Against a 1000-line backlog it measures: +// - idle : a redraw with nothing new (the window's standing per-frame cost) +// - trickle : one line per frame (the low-FPS watchdog and other steady producers) +// - burst N : N lines in one logger flush, then the next two frames. +// UMobiusCustomLoggerSubsystem::FlushToDisk broadcasts every queued line separately, +// 4x a second, so a heatmap load (~3,600 CreateMeshSection lines) lands as one burst. +// Each scenario times AppendLine (what the logger flush pays, synchronously) and +// FSlateApplication::ForceRedrawWindow on the log window alone - prepass, widget tick and paint, which +// is where text layout and list-row generation actually happen. Timing only AppendLine, or only the +// redraw, would hide whichever half a given implementation defers. +// +// Uses only the widget's public API and finds the window by title, so this one file measures any +// implementation of the window, before and after a change. Every scenario also checks the view is +// still pinned to the newest line - a cheap frame that stopped showing the end would not be a win. +// +// FollowsEndUnlessScrolledUp is the behavioural half: a reader who scrolls up (real wheel events routed +// through FSlateApplication, so they count as input) is not yanked back by new lines or a backlog trim, +// and following resumes once they scroll back down to the end. +// +// Needs a real renderer. Under -nullrhi Slate skips widget drawing entirely +// (GSlateSkipWidgetDrawingInHeadlessMode), so the redraw would time nothing; the test fails rather than +// report that. Run: +// MobiusPerf\RunTests.ps1 -Filter "Mobius.RenderedTiming.LogWindow." -Rendered +// Named "Mobius.RenderedTiming." - deliberately NOT containing "ProjectMobius." or "Mobius.Timing." - so +// neither the default correctness run nor -IncludeTiming picks it up: both run under -nullrhi, where these +// tests can only fail. Rows are appended to +// /MobiusPerfTimings/logwindow.csv; no absolute number is asserted. +// +#if !UE_BUILD_SHIPPING + +#include "CoreMinimal.h" +#include "Framework/Application/SlateApplication.h" +#include "HAL/FileManager.h" +#include "HAL/PlatformProcess.h" +#include "HAL/PlatformTime.h" +#include "Logging/LogWindow.h" +#include "Misc/App.h" +#include "Misc/AutomationTest.h" +#include "Misc/DateTime.h" +#include "Misc/FileHelper.h" +#include "Misc/Paths.h" +#include "Input/Events.h" +#include "Layout/WidgetPath.h" +#include "Rendering/SlateRenderer.h" +#include "Widgets/SWindow.h" +#include "Widgets/Text/STextBlock.h" + +namespace MobiusLogWindowTiming +{ + const FString LogWindowTitle = TEXT("Mobius Log"); + constexpr int32 BacklogLines = 1000; + + /** + * Two real line shapes from MobiusCustomLog.txt, alternated so the mean (~92 chars) sits near the + * ~89-char mean of the 1.26 M-line log on the dev machine. The index makes every line distinct. + */ + FString MakeLine(int32 Index) + { + if (Index % 2 == 0) + { + return FString::Printf(TEXT("[2026-10-22 11:23:45.%03d] [Heatmap floor=0] CreateMeshSection section=%d verts=1056 tris=1984 in 0.05 ms"), + Index % 1000, Index); + } + return FString::Printf(TEXT("[2026-10-22 11:23:45.%03d] Low FPS detected: 27.4 FPS (threshold 30.0) line %d"), + Index % 1000, Index); + } + + /** The "Mobius Log" window that was not already up before the widget was constructed. */ + TSharedPtr FindNewLogWindow(const TSet& Before) + { + for (const TSharedRef& Window : FSlateApplication::Get().GetTopLevelWindows()) + { + if (!Before.Contains(&Window.Get()) && Window->GetTitle().ToString().Equals(LogWindowTitle)) + { + return Window; + } + } + return nullptr; + } + + /** Drain the render thread. Through the Slate renderer (FSlateRHIRenderer::FlushCommands is + * FlushRenderingCommands) so this module needs no RenderCore link dependency. */ + void FlushSlateRendering() + { + if (FSlateRenderer* Renderer = FSlateApplication::Get().GetRenderer()) + { + Renderer->FlushCommands(); + } + } + + /** + * FSlateApplicationBase::UpdateAnyActiveTimersArePending is protected; a real Slate frame calls it + * once before drawing, and an active timer only runs in a paint that follows it. ForceRedrawWindow + * alone would therefore never run the window's timers. Re-exposed here - not re-implemented - so a + * redraw below behaves like a real frame. + */ + struct FActiveTimerPump : FSlateApplicationBase + { + using FSlateApplicationBase::UpdateAnyActiveTimersArePending; + }; + + void PumpActiveTimers() + { + FSlateApplicationBase& App = FSlateApplication::Get(); + (App.*(&FActiveTimerPump::UpdateAnyActiveTimersArePending))(); + } + + /** Game-thread cost of one frame of this window: active-timer pump + prepass + tick + paint. + * Render-thread work is flushed outside the timer so it neither counts nor piles up across + * iterations (a full draw-buffer pool would otherwise make the next draw block inside the timer). */ + double RedrawMs(const TSharedRef& Window) + { + FlushSlateRendering(); + const double Start = FPlatformTime::Seconds(); + PumpActiveTimers(); + FSlateApplication::Get().ForceRedrawWindow(Window); + const double Ms = (FPlatformTime::Seconds() - Start) * 1000.0; + FlushSlateRendering(); + return Ms; + } + + int32 CountTextBlocks(const TSharedRef& Root) + { + int32 Count = Root->GetType() == TEXT("STextBlock") ? 1 : 0; + FChildren* Children = Root->GetChildren(); + for (int32 Index = 0; Children && Index < Children->Num(); ++Index) + { + Count += CountTextBlocks(Children->GetChildAt(Index)); + } + return Count; + } + + /** First widget, depth-first, whose type name starts with Prefix ("SListView<...>" is templated). */ + TSharedPtr FindWidgetOfType(const TSharedRef& Root, const TCHAR* Prefix) + { + if (Root->GetType().ToString().StartsWith(Prefix)) + { + return Root; + } + FChildren* Children = Root->GetChildren(); + for (int32 Index = 0; Children && Index < Children->Num(); ++Index) + { + if (const TSharedPtr Found = FindWidgetOfType(Children->GetChildAt(Index), Prefix)) + { + return Found; + } + } + return nullptr; + } + + /** A painted STextBlock ending with Line. EndsWith, not Equals, so it matches both a single + * whole-backlog text block and a one-row-per-line list. */ + TSharedPtr FindPaintedTextEndingWith(const TSharedRef& Root, const FString& Line) + { + if (Root->GetType() == TEXT("STextBlock") + && StaticCastSharedRef(Root)->GetText().ToString().EndsWith(Line) + && Root->GetPaintSpaceGeometry().GetLocalSize().Y > 0.0) + { + return Root; + } + FChildren* Children = Root->GetChildren(); + for (int32 Index = 0; Children && Index < Children->Num(); ++Index) + { + if (const TSharedPtr Found = FindPaintedTextEndingWith(Children->GetChildAt(Index), Line)) + { + return Found; + } + } + return nullptr; + } + + double AbsoluteTop(const FGeometry& Geometry) + { + return Geometry.LocalToAbsolute(FVector2D::ZeroVector).Y; + } + + double AbsoluteBottom(const FGeometry& Geometry) + { + return Geometry.LocalToAbsolute(FVector2D(0.0, Geometry.GetLocalSize().Y)).Y; + } + + /** The scrolling viewport: the list (one row per line) or the scroll box (one whole-backlog text). */ + TSharedPtr FindLogViewport(const TSharedRef& Window) + { + const TSharedPtr List = FindWidgetOfType(Window, TEXT("SListView")); + return List.IsValid() ? List : FindWidgetOfType(Window, TEXT("SScrollBox")); + } + + /** + * Line is painted with its bottom on the viewport's bottom edge: the view is pinned to it, not merely + * holding a row for it somewhere below the fold. (Holding-but-not-showing is exactly what a freshly + * generated wrapping row does if nothing re-pins once it learns its wrap width.) + */ + bool IsLineAtViewportBottom(const TSharedRef& Window, const FString& Line, FString& OutDetail) + { + const TSharedPtr Viewport = FindLogViewport(Window); + if (!Viewport.IsValid()) + { + OutDetail = TEXT("no list or scroll box in the log window"); + return false; + } + const TSharedPtr Text = FindPaintedTextEndingWith(Viewport.ToSharedRef(), Line); + if (!Text.IsValid()) + { + OutDetail = TEXT("the line has no painted text block (not generated, or never drawn)"); + return false; + } + const double ViewportBottom = AbsoluteBottom(Viewport->GetPaintSpaceGeometry()); + const double TextBottom = AbsoluteBottom(Text->GetPaintSpaceGeometry()); + OutDetail = FString::Printf(TEXT("line bottom %.1f px, viewport bottom %.1f px"), TextBottom, ViewportBottom); + return FMath::Abs(TextBottom - ViewportBottom) <= 3.0; + } + + /** Texts of the painted text blocks lying wholly inside the viewport, top to bottom. */ + void CollectVisibleLines(const TSharedRef& Root, double ViewportTop, double ViewportBottom, TArray& OutLines) + { + if (Root->GetType() == TEXT("STextBlock")) + { + const FGeometry& Geometry = Root->GetPaintSpaceGeometry(); + if (Geometry.GetLocalSize().Y > 0.0 + && AbsoluteTop(Geometry) >= ViewportTop - 0.5 && AbsoluteBottom(Geometry) <= ViewportBottom + 0.5) + { + OutLines.Add(StaticCastSharedRef(Root)->GetText().ToString()); + } + return; + } + FChildren* Children = Root->GetChildren(); + for (int32 Index = 0; Children && Index < Children->Num(); ++Index) + { + CollectVisibleLines(Children->GetChildAt(Index), ViewportTop, ViewportBottom, OutLines); + } + } + + /** One wheel notch (or a big flick) routed to Target the way FSlateApplication routes real input - + * inside its input-processing scope, which is what the window keys "the reader scrolled" off. */ + bool RouteWheel(const TSharedRef& Target, float WheelDelta) + { + FWidgetPath FoundPath; + if (!FSlateApplication::Get().FindPathToWidget(Target, FoundPath)) + { + return false; + } + // FindPathToWidget leaves the per-widget virtual pointer positions empty, and pointer routing + // indexes them, so rebuild the path through the constructor that fills them (unset = no 3D widget). + TArray WidgetsAndPointers; + for (int32 Index = 0; Index < FoundPath.Widgets.Num(); ++Index) + { + WidgetsAndPointers.Add(FWidgetAndPointer(FoundPath.Widgets[Index])); + } + const FWidgetPath Path(WidgetsAndPointers); + + const FVector2D Centre = Target->GetPaintSpaceGeometry().GetAbsolutePositionAtCoordinates(FVector2D(0.5, 0.5)); + const FPointerEvent Wheel(/*PointerIndex*/ 0, Centre, Centre, TSet(), FKey(), WheelDelta, FModifierKeysState()); + return FSlateApplication::Get().RouteMouseWheelOrGestureEvent(Path, Wheel, nullptr).IsEventHandled(); + } + + double Median(TArray Values) + { + if (Values.Num() == 0) + { + return 0.0; + } + Values.Sort(); + const int32 Mid = Values.Num() / 2; + return (Values.Num() % 2 == 1) ? Values[Mid] : 0.5 * (Values[Mid - 1] + Values[Mid]); + } + + double MaxOf(const TArray& Values) + { + double Result = 0.0; + for (const double Value : Values) + { + Result = FMath::Max(Result, Value); + } + return Result; + } + + struct FScenarioResult + { + FString Scenario; + int32 LinesPerIteration = 0; + int32 Iterations = 0; + double AppendMedianMs = 0.0; + double Frame1MedianMs = 0.0; + double Frame2MedianMs = 0.0; + double TotalMedianMs = 0.0; + double TotalMaxMs = 0.0; + }; + + /** Append one row per scenario to the trend CSV (header written on first use). */ + void AppendCsvRow(const FScenarioResult& Result, int32 TextBlocks) + { + const FString Dir = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("MobiusPerfTimings")); + const FString CsvPath = FPaths::Combine(Dir, TEXT("logwindow.csv")); + FString Row; + if (!IFileManager::Get().FileExists(*CsvPath)) + { + IFileManager::Get().MakeDirectory(*Dir, /*Tree*/ true); + Row += TEXT("timestamp_utc,machine,scenario,lines_per_iteration,iterations,append_median_ms,frame1_median_ms,frame2_median_ms,total_median_ms,total_max_ms,text_blocks_in_window\n"); + } + Row += FString::Printf(TEXT("%s,%s,%s,%d,%d,%.4f,%.4f,%.4f,%.4f,%.4f,%d\n"), + *FDateTime::UtcNow().ToIso8601(), FPlatformProcess::ComputerName(), *Result.Scenario, + Result.LinesPerIteration, Result.Iterations, Result.AppendMedianMs, Result.Frame1MedianMs, + Result.Frame2MedianMs, Result.TotalMedianMs, Result.TotalMaxMs, TextBlocks); + FFileHelper::SaveStringToFile(Row, *CsvPath, FFileHelper::EEncodingOptions::ForceUTF8WithoutBOM, + &IFileManager::Get(), FILEWRITE_Append); + } +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FMobiusTimingLogWindowFullBacklogTest, + "Mobius.RenderedTiming.LogWindow.FullBacklogFrameCost", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::ClientContext | EAutomationTestFlags::ProductFilter) + +bool FMobiusTimingLogWindowFullBacklogTest::RunTest(const FString& Parameters) +{ + using namespace MobiusLogWindowTiming; + + if (!FSlateApplication::IsInitialized() || !FApp::CanEverRender()) + { + AddError(TEXT("Needs Slate and a real renderer: under -nullrhi Slate skips widget drawing, so the ") + TEXT("redraw half of every scenario would measure nothing. Run with RunTests.ps1 -Rendered.")); + return false; + } + + TSet WindowsBefore; + for (const TSharedRef& Window : FSlateApplication::Get().GetTopLevelWindows()) + { + WindowsBefore.Add(&Window.Get()); + } + + TArray Results; + int32 TextBlocksAtEnd = 0; + int32 NextLine = 0; + + { + const TSharedRef Widget = SNew(SLogWindowWidget); + const TSharedPtr Window = FindNewLogWindow(WindowsBefore); + if (!Window.IsValid()) + { + AddError(TEXT("Constructing SLogWindowWidget did not open a 'Mobius Log' window.")); + return false; + } + const TSharedRef WindowRef = Window.ToSharedRef(); + + // The backlog the owner's sessions reach after ~30 minutes: the cap, full. + TArray Backlog; + Backlog.Reserve(BacklogLines); + for (; NextLine < BacklogLines; ++NextLine) + { + Backlog.Add(MakeLine(NextLine)); + } + Widget->SetLines(Backlog); + + // Warm-up: the first paint establishes the wrap width, the next re-wraps against it, and any + // scroll-to-end request settles. + for (int32 Warm = 0; Warm < 3; ++Warm) + { + RedrawMs(WindowRef); + } + FString SeedDetail; + if (!IsLineAtViewportBottom(WindowRef, Backlog.Last(), SeedDetail)) + { + AddError(FString::Printf(TEXT("After seeding and three redraws the newest backlog line is not at the ") + TEXT("bottom of the view (%s) - the window is not drawing, or not following the end, so nothing ") + TEXT("below would be a measurement."), *SeedDetail)); + Widget->CloseLogWindow(); + return false; + } + + // --- idle: nothing new ---------------------------------------------------------------------- + { + TArray Frames; + for (int32 Iteration = 0; Iteration < 30; ++Iteration) + { + Frames.Add(RedrawMs(WindowRef)); + } + FScenarioResult& Result = Results.AddDefaulted_GetRef(); + Result.Scenario = TEXT("idle"); + Result.Iterations = Frames.Num(); + Result.Frame1MedianMs = Median(Frames); + Result.TotalMedianMs = Result.Frame1MedianMs; + Result.TotalMaxMs = MaxOf(Frames); + } + + // --- trickle: one line per frame ------------------------------------------------------------ + { + TArray Appends; + TArray Frames; + TArray Totals; + for (int32 Iteration = 0; Iteration < 120; ++Iteration) + { + const FString Line = MakeLine(NextLine++); + const double Start = FPlatformTime::Seconds(); + Widget->AppendLine(Line); + const double AppendMs = (FPlatformTime::Seconds() - Start) * 1000.0; + const double FrameMs = RedrawMs(WindowRef); + Appends.Add(AppendMs); + Frames.Add(FrameMs); + Totals.Add(AppendMs + FrameMs); + } + RedrawMs(WindowRef); + FString Detail; + TestTrue(TEXT("trickle: the view stays pinned to the newest line"), + IsLineAtViewportBottom(WindowRef, MakeLine(NextLine - 1), Detail)); + AddInfo(FString::Printf(TEXT("trickle end: %s"), *Detail)); + + FScenarioResult& Result = Results.AddDefaulted_GetRef(); + Result.Scenario = TEXT("trickle"); + Result.LinesPerIteration = 1; + Result.Iterations = Totals.Num(); + Result.AppendMedianMs = Median(Appends); + Result.Frame1MedianMs = Median(Frames); + Result.TotalMedianMs = Median(Totals); + Result.TotalMaxMs = MaxOf(Totals); + } + + // --- bursts: one logger flush of N lines, then two frames --------------------------------- + struct FBurst { int32 Lines; int32 Iterations; }; + const FBurst Bursts[] = { { 250, 5 }, { 3600, 3 } }; + for (const FBurst& Burst : Bursts) + { + TArray Appends; + TArray Frame1s; + TArray Frame2s; + TArray Totals; + for (int32 Iteration = 0; Iteration < Burst.Iterations; ++Iteration) + { + TArray Lines; + Lines.Reserve(Burst.Lines); + for (int32 Index = 0; Index < Burst.Lines; ++Index) + { + Lines.Add(MakeLine(NextLine++)); + } + + const double Start = FPlatformTime::Seconds(); + for (const FString& Line : Lines) + { + Widget->AppendLine(Line); + } + const double AppendMs = (FPlatformTime::Seconds() - Start) * 1000.0; + const double Frame1Ms = RedrawMs(WindowRef); + const double Frame2Ms = RedrawMs(WindowRef); + + FString Detail; + const bool bPinned = IsLineAtViewportBottom(WindowRef, Lines.Last(), Detail); + TestTrue(FString::Printf(TEXT("burst %d: the view is pinned to the newest line after two frames (%s)"), Burst.Lines, *Detail), + bPinned); + + Appends.Add(AppendMs); + Frame1s.Add(Frame1Ms); + Frame2s.Add(Frame2Ms); + Totals.Add(AppendMs + Frame1Ms + Frame2Ms); + } + + FScenarioResult& Result = Results.AddDefaulted_GetRef(); + Result.Scenario = FString::Printf(TEXT("burst_%d"), Burst.Lines); + Result.LinesPerIteration = Burst.Lines; + Result.Iterations = Totals.Num(); + Result.AppendMedianMs = Median(Appends); + Result.Frame1MedianMs = Median(Frame1s); + Result.Frame2MedianMs = Median(Frame2s); + Result.TotalMedianMs = Median(Totals); + Result.TotalMaxMs = MaxOf(Totals); + } + + // Fingerprint of the implementation under test: a single whole-backlog text block reads ~3 + // (body + Close label + title); a virtualized list reads roughly its visible row count. + TextBlocksAtEnd = CountTextBlocks(WindowRef); + + Widget->CloseLogWindow(); + } + + for (const FScenarioResult& Result : Results) + { + AddInfo(FString::Printf(TEXT("%-10s lines/iter=%4d iters=%3d | append %8.3f ms | frame1 %8.3f ms | frame2 %8.3f ms | total median %8.3f ms, max %8.3f ms"), + *Result.Scenario, Result.LinesPerIteration, Result.Iterations, Result.AppendMedianMs, + Result.Frame1MedianMs, Result.Frame2MedianMs, Result.TotalMedianMs, Result.TotalMaxMs)); + AppendCsvRow(Result, TextBlocksAtEnd); + } + AddInfo(FString::Printf(TEXT("text blocks in the log window at the end: %d"), TextBlocksAtEnd)); + + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FMobiusLogWindowFollowsEndUnlessScrolledUpTest, + "Mobius.RenderedTiming.LogWindow.FollowsEndUnlessScrolledUp", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::ClientContext | EAutomationTestFlags::ProductFilter) + +bool FMobiusLogWindowFollowsEndUnlessScrolledUpTest::RunTest(const FString& Parameters) +{ + using namespace MobiusLogWindowTiming; + + if (!FSlateApplication::IsInitialized() || !FApp::CanEverRender()) + { + AddError(TEXT("Needs Slate and a real renderer (geometry only exists once painted). Run with RunTests.ps1 -Rendered.")); + return false; + } + + TSet WindowsBefore; + for (const TSharedRef& Window : FSlateApplication::Get().GetTopLevelWindows()) + { + WindowsBefore.Add(&Window.Get()); + } + + { + const TSharedRef Widget = SNew(SLogWindowWidget); + const TSharedPtr Window = FindNewLogWindow(WindowsBefore); + if (!Window.IsValid()) + { + AddError(TEXT("Constructing SLogWindowWidget did not open a 'Mobius Log' window.")); + return false; + } + const TSharedRef WindowRef = Window.ToSharedRef(); + const auto Redraw = [&WindowRef](int32 Frames) + { + for (int32 Frame = 0; Frame < Frames; ++Frame) + { + RedrawMs(WindowRef); + } + }; + + int32 NextLine = 0; + TArray Backlog; + for (; NextLine < BacklogLines; ++NextLine) + { + Backlog.Add(MakeLine(NextLine)); + } + Widget->SetLines(Backlog); + Redraw(4); + + FString Detail; + TestTrue(TEXT("opens pinned to the newest line"), IsLineAtViewportBottom(WindowRef, Backlog.Last(), Detail)); + AddInfo(FString::Printf(TEXT("open: %s"), *Detail)); + + const TSharedPtr List = FindWidgetOfType(WindowRef, TEXT("SListView")); + if (!List.IsValid()) + { + AddError(TEXT("The log window has no SListView - this test drives the virtualized list.")); + Widget->CloseLogWindow(); + return false; + } + + // --- the reader scrolls up a few notches ---------------------------------------------------- + for (int32 Notch = 0; Notch < 5; ++Notch) + { + TestTrue(TEXT("wheel-up is handled by the list"), RouteWheel(List.ToSharedRef(), 1.0f)); + Redraw(2); + } + const double ViewportTop = AbsoluteTop(List->GetPaintSpaceGeometry()); + const double ViewportBottom = AbsoluteBottom(List->GetPaintSpaceGeometry()); + TArray VisibleBefore; + CollectVisibleLines(List.ToSharedRef(), ViewportTop, ViewportBottom, VisibleBefore); + TestTrue(TEXT("scrolled up: some lines are on screen"), VisibleBefore.Num() > 0); + TestFalse(TEXT("scrolled up: the newest line is no longer at the bottom"), IsLineAtViewportBottom(WindowRef, Backlog.Last(), Detail)); + + // --- new lines arrive - enough to overflow the cap and trim the front ------------------------ + FString Newest; + for (int32 Index = 0; Index < 200; ++Index) + { + Newest = MakeLine(NextLine++); + Widget->AppendLine(Newest); + } + Redraw(5); + TestFalse(TEXT("a scrolled-up reader is not yanked to the newest line"), IsLineAtViewportBottom(WindowRef, Newest, Detail)); + TArray VisibleAfter; + CollectVisibleLines(List.ToSharedRef(), ViewportTop, ViewportBottom, VisibleAfter); + if (VisibleBefore.Num() > 0) + { + TestTrue(TEXT("the lines being read stay on screen through new lines and a front trim"), + VisibleAfter.Contains(VisibleBefore[VisibleBefore.Num() / 2])); + } + + // --- the reader scrolls back down to the end: following resumes ----------------------------- + TestTrue(TEXT("wheel-down is handled by the list"), RouteWheel(List.ToSharedRef(), -1000.0f)); + Redraw(3); + Newest = MakeLine(NextLine++); + Widget->AppendLine(Newest); + Redraw(4); + TestTrue(TEXT("after scrolling back to the end, new lines are followed again"), IsLineAtViewportBottom(WindowRef, Newest, Detail)); + AddInfo(FString::Printf(TEXT("resumed: %s"), *Detail)); + + Widget->CloseLogWindow(); + } + return true; +} + +#endif // !UE_BUILD_SHIPPING diff --git a/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/MobiusLoggerInGameTest.cpp b/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/MobiusLoggerInGameTest.cpp new file mode 100644 index 000000000..5228d7afe --- /dev/null +++ b/UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/MobiusLoggerInGameTest.cpp @@ -0,0 +1,240 @@ +// Copyright (c) 2026 ProjectMobius contributors. Licensed under MIT. +// +// MobiusLoggerInGameTest.cpp +// +// In-game gates for the custom logger path +// (UMobiusCustomLoggerSubsystem -> UMobiusUserFeedbackSubsystem -> log window). +// +// - FrameGrabberRetryIsQuiet. UFrameGrabberHelper::Tick retries TryInitialize until the game viewport +// is usable. Under -game -nullrhi - every headless in-game automation run - the viewport stays 0x0 for +// the whole session, and the helper used to log "[FrameGrabber] TryInitialize failed: Invalid +// viewport size 0x0" on every frame: 1,070,058 of the 1,259,848 lines in the dev machine's +// MobiusCustomLog.txt, at up to ~3,000 lines/s. This counts those lines over a fixed idle window and +// fails above a small bound. Where the viewport IS valid the count is ~0 with or without the fix, so +// only a -nullrhi run discriminates; the test reports which viewport it saw. +// +// - FeedbackCacheKeepsNewestOldestFirst. UMobiusUserFeedbackSubsystem caches the newest lines to seed +// a (re)opened log window. It is a ring buffer; this pushes 1.5x its capacity of marker lines through +// the REAL logger flush and checks GetCachedLogLines() returns exactly the newest ones, oldest first - +// the wrap-around seam is where a ring buffer gets the order wrong. +// +// Both force logging on for their duration (it is a persisted user toggle and may be off) and restore it. +// ClientContext only: they need a game world and its GameInstance. Run: +// UnrealEditor-Cmd -game -nullrhi -unattended -ExecCmds="Automation RunTests Mobius.InGame.Logger.; Quit" +// Named "Mobius.InGame." so the default correctness filter never runs them. +// +#if !UE_BUILD_SHIPPING + +#include "CoreMinimal.h" +#include "Engine/Engine.h" +#include "Engine/GameInstance.h" +#include "Engine/GameViewportClient.h" +#include "Engine/World.h" +#include "HAL/PlatformTime.h" +#include "Misc/AutomationTest.h" +#include "Misc/Guid.h" +#include "Subsystems/MobiusCustomLoggerSubsystem.h" +#include "Subsystems/MobiusUserFeedbackSubsystem.h" +#include "Tests/AutomationCommon.h" +#include "UnrealClient.h" + +namespace MobiusLoggerInGameTest +{ + /** Idle window for the FrameGrabber count. At the ~120 fps a -nullrhi run reaches, the pre-fix code + * logs well over a thousand lines in this time. */ + constexpr float IdleSeconds = 10.0f; + + /** Generous: the fixed helper logs a failure reason once, when it CHANGES - normally before this + * test even starts, so 0 lines here. */ + constexpr int32 MaxFrameGrabberFailureLines = 3; + + /** UMobiusUserFeedbackSubsystem::MaxCachedLogLines (private). If that changes, so must this. */ + constexpr int32 FeedbackCacheCapacity = 1000; + + UWorld* FindGameWorld() + { + for (const FWorldContext& Context : GEngine->GetWorldContexts()) + { + if ((Context.WorldType == EWorldType::Game || Context.WorldType == EWorldType::PIE) && Context.World()) + { + return Context.World(); + } + } + return nullptr; + } + + FIntPoint GameViewportSize() + { + if (GEngine && GEngine->GameViewport && GEngine->GameViewport->Viewport) + { + return GEngine->GameViewport->Viewport->GetSizeXY(); + } + return FIntPoint::ZeroValue; + } + + struct FFrameGrabberSpamState + { + int32 FrameGrabberFailureLines = 0; + int32 AllLines = 0; + FDelegateHandle LogLineHandle; + bool bLoggingWasEnabled = true; + double StartSeconds = 0.0; + }; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FMobiusLoggerFrameGrabberRetryIsQuietTest, + "Mobius.InGame.Logger.FrameGrabberRetryIsQuiet", + EAutomationTestFlags::ClientContext | EAutomationTestFlags::ProductFilter) + +bool FMobiusLoggerFrameGrabberRetryIsQuietTest::RunTest(const FString& Parameters) +{ + using namespace MobiusLoggerInGameTest; + + if (!FindGameWorld()) + { + AddError(TEXT("no game world - run under UnrealEditor-Cmd -game")); + return false; + } + UMobiusCustomLoggerSubsystem* Logger = GEngine->GetEngineSubsystem(); + if (!Logger) + { + AddError(TEXT("UMobiusCustomLoggerSubsystem is missing")); + return false; + } + + const TSharedRef State = MakeShared(); + // SetLoggingEnabled UE_LOGs a warning, so only touch it when logging is actually off. + State->bLoggingWasEnabled = Logger->IsLoggingEnabled(); + if (!State->bLoggingWasEnabled) + { + Logger->SetLoggingEnabled(true); + } + + // Deliver everything queued so far first, so only lines produced inside the window are counted. + Logger->FlushQueuedMessages(); + State->LogLineHandle = Logger->OnLogLine().AddLambda([State](const FString& Line) + { + ++State->AllLines; + if (Line.Contains(TEXT("[FrameGrabber] TryInitialize failed"))) + { + ++State->FrameGrabberFailureLines; + } + }); + State->StartSeconds = FPlatformTime::Seconds(); + + ADD_LATENT_AUTOMATION_COMMAND(FEngineWaitLatentCommand(IdleSeconds)); + ADD_LATENT_AUTOMATION_COMMAND(FFunctionLatentCommand([this, State]() + { + UMobiusCustomLoggerSubsystem* LoggerNow = GEngine->GetEngineSubsystem(); + if (!LoggerNow) + { + AddError(TEXT("UMobiusCustomLoggerSubsystem went away during the idle window")); + return true; + } + LoggerNow->FlushQueuedMessages(); + LoggerNow->OnLogLine().Remove(State->LogLineHandle); + const double Elapsed = FPlatformTime::Seconds() - State->StartSeconds; + const FIntPoint Viewport = GameViewportSize(); + + AddInfo(FString::Printf(TEXT("%d '[FrameGrabber] TryInitialize failed' lines in %.1f s (%.1f/s), %d logger lines in total; game viewport %dx%d%s"), + State->FrameGrabberFailureLines, Elapsed, Elapsed > 0.0 ? State->FrameGrabberFailureLines / Elapsed : 0.0, + State->AllLines, Viewport.X, Viewport.Y, + (Viewport.X > 0 && Viewport.Y > 0) ? TEXT(" (valid viewport: this run cannot reproduce the 0x0 retry spam)") : TEXT(""))); + + TestTrue(FString::Printf(TEXT("FrameGrabber logged at most %d TryInitialize failures in %.0f s idle"), MaxFrameGrabberFailureLines, IdleSeconds), + State->FrameGrabberFailureLines <= MaxFrameGrabberFailureLines); + + if (!State->bLoggingWasEnabled) + { + LoggerNow->SetLoggingEnabled(false); + } + return true; + })); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FMobiusLoggerFeedbackCacheOrderTest, + "Mobius.InGame.Logger.FeedbackCacheKeepsNewestOldestFirst", + EAutomationTestFlags::ClientContext | EAutomationTestFlags::ProductFilter) + +bool FMobiusLoggerFeedbackCacheOrderTest::RunTest(const FString& Parameters) +{ + using namespace MobiusLoggerInGameTest; + + UWorld* World = FindGameWorld(); + UGameInstance* GameInstance = World ? World->GetGameInstance() : nullptr; + UMobiusUserFeedbackSubsystem* Feedback = GameInstance ? GameInstance->GetSubsystem() : nullptr; + UMobiusCustomLoggerSubsystem* Logger = GEngine->GetEngineSubsystem(); + if (!Feedback || !Logger) + { + AddError(TEXT("needs a game world with UMobiusUserFeedbackSubsystem and the custom logger - run under UnrealEditor-Cmd -game")); + return false; + } + + const bool bLoggingWasEnabled = Logger->IsLoggingEnabled(); + if (!bLoggingWasEnabled) + { + Logger->SetLoggingEnabled(true); + } + Logger->FlushQueuedMessages(); + + // 1.5x capacity, through the real path: EnqueueLogMessage -> FlushToDisk broadcast -> HandleLogLine. + // Unique tag so lines from any other run (or another producer mid-test) cannot be mistaken for ours. + const FString Tag = FString::Printf(TEXT("[LoggerCacheTest %s]"), *FGuid::NewGuid().ToString(EGuidFormats::Short)); + const int32 Pushed = FeedbackCacheCapacity + FeedbackCacheCapacity / 2; + for (int32 Index = 0; Index < Pushed; ++Index) + { + Logger->EnqueueLogMessage(FString::Printf(TEXT("%s marker %05d"), *Tag, Index)); + } + Logger->FlushQueuedMessages(); + + const TArray Cached = Feedback->GetCachedLogLines(); + if (!bLoggingWasEnabled) + { + Logger->SetLoggingEnabled(false); + } + + TestEqual(TEXT("cache holds exactly its capacity"), Cached.Num(), FeedbackCacheCapacity); + if (Cached.Num() == 0) + { + return false; + } + + // Marker indices in cache order. + TArray Markers; + const FString MarkerPrefix = Tag + TEXT(" marker "); + for (const FString& Line : Cached) + { + const int32 At = Line.Find(MarkerPrefix); + if (At != INDEX_NONE) + { + Markers.Add(FCString::Atoi(*Line.Mid(At + MarkerPrefix.Len()))); + } + } + + TestTrue(TEXT("the newest cached line is the newest marker"), + Cached.Last().EndsWith(FString::Printf(TEXT("marker %05d"), Pushed - 1))); + bool bStrictlyIncreasing = Markers.Num() > 0; + for (int32 Index = 1; Index < Markers.Num(); ++Index) + { + bStrictlyIncreasing &= Markers[Index] == Markers[Index - 1] + 1; + } + TestTrue(TEXT("markers come back oldest first and contiguous - no seam where the ring wrapped"), bStrictlyIncreasing); + if (Markers.Num() > 0) + { + TestTrue(TEXT("the oldest markers were evicted"), Markers[0] >= Pushed - FeedbackCacheCapacity); + } + // Nothing else is normally logged between the two flushes, in which case the cache is exactly the + // newest FeedbackCacheCapacity markers. + if (Markers.Num() == FeedbackCacheCapacity) + { + TestEqual(TEXT("first cached line is marker Pushed - capacity"), Markers[0], Pushed - FeedbackCacheCapacity); + } + AddInfo(FString::Printf(TEXT("cache %d lines, %d of them markers, first marker %d, last %d"), + Cached.Num(), Markers.Num(), Markers.Num() ? Markers[0] : -1, Markers.Num() ? Markers.Last() : -1)); + return true; +} + +#endif // !UE_BUILD_SHIPPING From 465510eaf24cde8376eb7bfee5dcaf4e0bb63776 Mon Sep 17 00:00:00 2001 From: Nicholas Harding <40708936+sir306@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:21:43 +1300 Subject: [PATCH 3/6] Document the pedestrian data format for other simulators External evacuation and crowd modellers need to export trajectories Mobius can load, and the HDF5 reader decodes most fields by position with no type conversion, so a wrong dtype loads garbage instead of failing. - docs/PEDESTRIAN-DATA-FORMAT.md: field-by-field Mobius HDF5 and JSON layout, units, axes, time model, heading convention (degrees CCW from +x, checked in the app), the rules the importer relies on, and a checklist. Notes that WKT geometry is mirrored against Mobius-format agents. - docs/HDF5-EXPORT-GUIDE.md: get the files (sparse clone), write with Python (h5py) or with the bundled HDF5 C library built standalone, validate, open in Mobius, troubleshoot. - docs/EXODUS-EXPORT.md: short buildingEXODUS route via a long-format CSV. - docs/examples/pedestrian-hdf5/: C writer + CMake, mobius_h5_writer.py (library, from-csv with linear/hold resampling and floor heights, demo) and validate_mobius_h5.py, which mirrors the importer's checks. - Fix links to the renamed plugin (Hdf5DataPlugin -> MobiusDataImporter), correct the ThirdParty README's claim that the HDF5 install tree is tracked, and mirror the format page into packaged builds. AI-assisted with Claude Code; all changes reviewed, tested, and verified by a human maintainer in line with this repository's AI-tooling policy. --- README.md | 10 +- .../Source/ThirdParty/README.md | 23 +- .../cmake/SyncBuildDocs.cmake.in | 1 + docs/AGENT-SAMPLE-DATA.md | 8 +- docs/ARCHITECTURE.md | 2 +- docs/DATA-PIPELINE.md | 9 +- docs/EXODUS-EXPORT.md | 133 +++++ docs/HDF5-EXPORT-GUIDE.md | 318 ++++++++++++ docs/PEDESTRIAN-DATA-FORMAT.md | 331 ++++++++++++ docs/README.md | 6 + docs/examples/pedestrian-hdf5/CMakeLists.txt | 39 ++ .../pedestrian-hdf5/mobius_h5_writer.py | 481 ++++++++++++++++++ .../examples/pedestrian-hdf5/requirements.txt | 6 + .../pedestrian-hdf5/validate_mobius_h5.py | 354 +++++++++++++ .../pedestrian-hdf5/write_mobius_h5.c | 332 ++++++++++++ 15 files changed, 2034 insertions(+), 19 deletions(-) create mode 100644 docs/EXODUS-EXPORT.md create mode 100644 docs/HDF5-EXPORT-GUIDE.md create mode 100644 docs/PEDESTRIAN-DATA-FORMAT.md create mode 100644 docs/examples/pedestrian-hdf5/CMakeLists.txt create mode 100644 docs/examples/pedestrian-hdf5/mobius_h5_writer.py create mode 100644 docs/examples/pedestrian-hdf5/requirements.txt create mode 100644 docs/examples/pedestrian-hdf5/validate_mobius_h5.py create mode 100644 docs/examples/pedestrian-hdf5/write_mobius_h5.c diff --git a/README.md b/README.md index 87cf44a90..c7a869910 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Project Mobius is an Unreal Engine 5.5 application for loading pedestrian trajec ## Why HDF5 For larger trajectory files, convert JSON to HDF5 with the bundled -[`json_to_hdf5_converter.py`](UnrealFolder/ProjectMobius/Plugins/Hdf5DataPlugin/Scripts/json_to_hdf5_converter.py) +[`json_to_hdf5_converter.py`](UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Scripts/json_to_hdf5_converter.py) script. In one 5,000-agent sample, HDF5 cut load time from `21.29 s` to `3.18 s` and reduced memory use from `317,555 KB` to `55,283 KB`. @@ -88,6 +88,11 @@ and reduced memory use from `317,555 KB` to `55,283 KB`. More on supported formats, the conversion command, and the benchmark: [docs/DATA-PIPELINE.md](docs/DATA-PIPELINE.md) +To export trajectories from your own evacuation or crowd model, see +[Pedestrian Data Format](docs/PEDESTRIAN-DATA-FORMAT.md) for the exact file layout and the +[HDF5 Export Guide](docs/HDF5-EXPORT-GUIDE.md) for writing it from Python or with the bundled +HDF5 C library ([buildingEXODUS users](docs/EXODUS-EXPORT.md)). + ## Quick Start ### Prerequisites @@ -169,6 +174,9 @@ Full build, packaging, and automation notes live in - [Architecture](docs/ARCHITECTURE.md) - [Data Pipeline](docs/DATA-PIPELINE.md) - [Agent Sample Data](docs/AGENT-SAMPLE-DATA.md) +- [Pedestrian Data Format](docs/PEDESTRIAN-DATA-FORMAT.md) +- [HDF5 Export Guide](docs/HDF5-EXPORT-GUIDE.md) +- [Exporting from buildingEXODUS](docs/EXODUS-EXPORT.md) - [Research and Publications](docs/RESEARCH.md) - [UE Automation Guide](UnrealFolder/ProjectMobius/BuildDocs/UE-Automation-Guide.md) - [Contributing](CONTRIBUTING.md) diff --git a/UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/README.md b/UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/README.md index 93a661f5a..2c93cba7f 100644 --- a/UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/README.md +++ b/UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/README.md @@ -15,15 +15,17 @@ ## HDF5 +> **Writing pedestrian data from another simulator?** You can build this HDF5 source on its own, +> without Unreal or the superbuild: see `docs/HDF5-EXPORT-GUIDE.md` at the repository root. + ## Layout - `hdf5-2.0.0/` — full upstream HDF5 2.0.0 source (CMake-only since 2.0; autotools was dropped - upstream). Kept in-tree so the prebuilt libraries below can be rebuilt without hunting down the - exact upstream snapshot. -- `hdf5-2.0.0/install/` — the build output the UE module actually consumes: - - `include/` — public headers, **including the generated `H5pubconf.h`** (deliberately tracked so - a fresh clone compiles without running CMake; it carries harmless machine stamps from whoever - generated it last). + upstream). This source is what git tracks; the libraries below are built from it. +- `hdf5-2.0.0/install/` — the build output the UE module actually consumes. **None of it is in + git** (see `.gitignore`): a fresh clone gets it by running the superbuild + (`python superbuild.py` from `UnrealFolder/ProjectMobius`). + - `include/` — public headers, including the generated `H5pubconf.h`. - `lib/libhdf5.lib`, `lib/libhdf5_hl.lib`, `lib/zlib-static.lib` — static libs, linked by `UeHdf5Library.Build.cs`. - `bin/` — MSVC runtime redistributables (`UeHdf5Library.Build.cs` checks the dir exists; nothing @@ -33,7 +35,7 @@ ## What is intentionally NOT tracked (see repo `.gitignore`) -`install/cmake/`, `install/lib/pkgconfig/`, `install/lib/libhdf5.settings`, `hdf5-2.0.0/build*` — +The whole `install/` tree and `hdf5-2.0.0/build*` — machine-generated at configure/install time and re-stamped with absolute local paths on every `cmake --install`. If they show up as modified/untracked after a rebuild, that is expected. @@ -66,8 +68,7 @@ the Windows one for the Mac libs. Notes: - `ZLIB_USE_LOCALCONTENT=OFF` makes the superbuild download the zlib TGZ at configure time (URLs in `config/CacheURLs.cmake`) — network access required. -- After an install, only `install/include/H5pubconf.h` may show a real tracked diff (path/date - stamps). Revert it unless the rebuild was intentional; commit it together with the new libs when - it was. +- `install/` is regenerated as a whole (headers, including the platform-generated `H5pubconf.h`, + and libs together), so nothing under it needs reverting or committing after a rebuild. - Keep the record layout expectations of the importer in mind: the libs must stay ABI-compatible - with `H5pubconf.h` as committed. + with the `H5pubconf.h` installed beside them. diff --git a/UnrealFolder/ProjectMobius/cmake/SyncBuildDocs.cmake.in b/UnrealFolder/ProjectMobius/cmake/SyncBuildDocs.cmake.in index 2278f6cc9..8be68f67b 100644 --- a/UnrealFolder/ProjectMobius/cmake/SyncBuildDocs.cmake.in +++ b/UnrealFolder/ProjectMobius/cmake/SyncBuildDocs.cmake.in @@ -7,6 +7,7 @@ set(_doc_files "BUILD-AND-RUN.md" "CONTROLS.md" "DATA-PIPELINE.md" + "PEDESTRIAN-DATA-FORMAT.md" "RESEARCH.md" ) diff --git a/docs/AGENT-SAMPLE-DATA.md b/docs/AGENT-SAMPLE-DATA.md index 68638f7ed..98dd66a5a 100644 --- a/docs/AGENT-SAMPLE-DATA.md +++ b/docs/AGENT-SAMPLE-DATA.md @@ -2,9 +2,9 @@ ## Scope -This guide documents the sample agent datasets currently shipped under the packaged Windows build tree and the HDF5 plugin test data that Project Mobius can ingest. +This guide documents the sample agent datasets kept in the repository as HDF5 importer test data that Project Mobius can ingest. -It covers the Juelich trajectory HDF5 datasets in `UnrealFolder/Build/Windows/ProjectMobius/Plugins/Hdf5DataPlugin/Hdf5TestData/JuelichTestCases/trajectories_hdf5/`. +It covers the Juelich trajectory HDF5 datasets in `UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Hdf5TestData/JuelichTestCases/trajectories_hdf5/`. The goal is to make it clear what each file contains, what format it uses, and what Project Mobius does with it at load time. @@ -96,8 +96,8 @@ Important conversion details: The relevant loader code paths are: -- `Plugins/Hdf5DataPlugin/Source/Hdf5DataPlugin/Public/Hdf5SimulationReader.h` -- `Plugins/Hdf5DataPlugin/Source/Hdf5DataPlugin/Private/Hdf5SimulationReader.cpp` +- `Plugins/MobiusDataImporter/Source/MobiusDataImporter/Public/Hdf5SimulationReader.h` +- `Plugins/MobiusDataImporter/Source/MobiusDataImporter/Private/Hdf5SimulationReader.cpp` - `Source/ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp` These files contain the format detection logic, the Juelich-to-Mobius conversion path, and the fallback rotation and speed calculation used when the source data does not provide those fields. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 95be553f4..fa54a6463 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -52,7 +52,7 @@ simulation playback, UI, analytics, and supporting infrastructure. | Plugin | Purpose | |--------|---------| -| `Hdf5DataPlugin` | HDF5 trajectory file reading | +| `MobiusDataImporter` | Pedestrian (JSON/HDF5) and B-RISK data import, including the bundled HDF5 library | | `UE4_Assimp` | Geometry and mesh import support | | `OpenCV` | Epic's built-in Unreal engine plugin that provides the OpenCV runtime used by visualization-related code | diff --git a/docs/DATA-PIPELINE.md b/docs/DATA-PIPELINE.md index 5e0ff46db..af6ae1ce2 100644 --- a/docs/DATA-PIPELINE.md +++ b/docs/DATA-PIPELINE.md @@ -10,6 +10,9 @@ workflow. - `.h5` simulation files loaded through `FHdf5SimulationReader` - HDF5 support includes both Mobius-format and Juelich-format datasets +The exact layout of each field is defined in [Pedestrian Data Format](PEDESTRIAN-DATA-FORMAT.md). +To write these files from another simulator, see the [HDF5 Export Guide](HDF5-EXPORT-GUIDE.md). + ## Geometry Inputs The project also supports loading geometry and environment data from common @@ -25,12 +28,12 @@ formats such as: The bundled converter script lives at: -`UnrealFolder/ProjectMobius/Plugins/Hdf5DataPlugin/Scripts/json_to_hdf5_converter.py` +`UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Scripts/json_to_hdf5_converter.py` Example usage: ```bash -cd UnrealFolder/ProjectMobius/Plugins/Hdf5DataPlugin/Scripts +cd UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Scripts python json_to_hdf5_converter.py path/to/input.json path/to/output.h5 ``` @@ -69,6 +72,8 @@ That benchmark works out to: ## Related References +- [Pedestrian Data Format](PEDESTRIAN-DATA-FORMAT.md) +- [HDF5 Export Guide](HDF5-EXPORT-GUIDE.md) - [Build and Run](BUILD-AND-RUN.md) - [Architecture](ARCHITECTURE.md) - [UE Automation Guide](../UnrealFolder/ProjectMobius/BuildDocs/UE-Automation-Guide.md) diff --git a/docs/EXODUS-EXPORT.md b/docs/EXODUS-EXPORT.md new file mode 100644 index 000000000..51831c17e --- /dev/null +++ b/docs/EXODUS-EXPORT.md @@ -0,0 +1,133 @@ +# Exporting from buildingEXODUS + +How to bring occupant movement from **buildingEXODUS** (Fire Safety Engineering Group, +University of Greenwich) into Project Mobius. This page is about the evacuation model, +not the Sandia *Exodus II* finite-element mesh format. + +> **Status.** The Mobius side of this workflow (the CSV converter and the validator) is +> tested. The buildingEXODUS side is not: we have not yet had an EXODUS export to work +> from, so this page does not name EXODUS menus, file names or column layouts. **Check +> step 1 against your EXODUS version**, and if you can share a small sample export, +> please open an issue so this page can document the exact route. + +## The short version + +1. Get **one row per occupant per output time** out of EXODUS: occupant id, time, + x, y and floor. +2. Convert it with the bundled script, which resamples onto an even time step, turns + floors into heights and computes heading and speed: + + ```bash + python mobius_h5_writer.py from-csv exodus_positions.csv run.h5 --dt 0.5 --interp linear --floor-z "0=0,1=3.5" + ``` + + Use `--interp linear` if the export is a snapshot of every occupant at a fixed + interval, `--interp hold` if it writes a row only when an occupant moves + ([why](#3-exodus-specific-points)). + +3. Check it, then open it in Mobius: + + ```bash + python validate_mobius_h5.py run.h5 + ``` + +Both scripts are in [`docs/examples/pedestrian-hdf5/`](examples/pedestrian-hdf5/); +install with `pip install -r requirements.txt`. No Unreal Engine or C compiler is needed. + +## 1. What to get out of EXODUS + +Mobius needs only occupant positions over time. Per occupant, per output time: + +| Column | Meaning | +| --- | --- | +| `id` | Occupant identifier, any number or text, stable for the whole run | +| `time` | Simulation time in seconds | +| `x`, `y` | Plan position in metres, in the same coordinate system as the building geometry | +| `floor` | Floor/level the occupant is on (needed for multi-storey buildings) | + +Occupants who have left the building should simply have no further rows. Heading and +speed are **not** needed: the converter computes both from the movement. + +EXODUS can export runs to its vrEXODUS replay tool and can save scenario data to +spreadsheet-style output files. Use whichever route your licence and version provide to +get the table above, then save it as CSV. If your export is +**wide** (one column per occupant, or one block per occupant), reshape it to the long +form below first. A spreadsheet or a few lines of Python will do it. + +```text +id,time,x,y,floor +1,0.0,12.25,4.75,0 +1,1.0,12.75,4.75,0 +1,2.0,13.25,5.25,0 +2,0.0,3.75,18.25,1 +2,1.0,3.75,17.75,1 +... +``` + +## 2. Convert + +```bash +python mobius_h5_writer.py from-csv exodus_positions.csv run.h5 \ + --dt 0.5 --interp linear \ + --floor-z "0=0,1=3.5,2=7" \ + --col-id id --col-time time --col-x x --col-y y --col-floor floor +``` + +| Option | Use | +| --- | --- | +| `--dt` | Output interval in seconds. Use your export interval, or smaller to get smoother playback from irregular output. Always pass it explicitly. | +| `--interp` | `linear` for fixed-interval snapshots, `hold` for logs that write a row only when an occupant moves. | +| `--floor-z` | Floor label → floor height in metres. Get the heights from your EXODUS geometry. | +| `--floor-height` | Alternative to `--floor-z` when every storey has the same height and floors are numbered 0, 1, 2… | +| `--col-…` | Map your column names if they differ from `id/time/x/y/floor`. | +| `--scale` | Multiply coordinates to get metres (e.g. `0.001` if exported in mm). | + +The converter renumbers occupants 0 … N−1 (the EXODUS id is kept in the name, so +`Ctrl` + hover in Mobius shows `Agent 17` for occupant 17), resamples every occupant +onto the `--dt` grid between their first and last row, makes the earliest time in the +file time 0, computes heading and speed from the movement, and writes the Mobius HDF5 +file. It stops with a message +instead of writing a file Mobius would misread (duplicate times for an occupant, +non-numeric values, a floor with no height). + +## 3. EXODUS-specific points + +- **Node-based movement.** EXODUS moves occupants across a grid of nodes, so exported + positions usually sit on node centres and change in node-sized steps. Heading is + computed from successive positions, so headings follow the grid directions (straight + and diagonal moves). That is the data, not a fault. With `--interp linear`, a larger + `--dt` averages the direction over several node steps; a smaller one keeps every move. +- **Event logs vs snapshots.** If your export records a position only when an occupant + moves, use `--interp hold`: the occupant stays where it was last recorded until its + next row, so time spent queueing at a door stays visible in density maps and flow + counts. With `linear`, a 10 s wait followed by one step would turn into a slow + 10 s creep. Choose `--dt` at or below the typical time between moves. Expect brief + speed spikes in `hold` mode, since each node step happens within one `--dt`. +- **Stairs.** If your export only labels positions with a floor, an occupant's height + jumps from one floor to the next when the label changes on the stairs. If it gives + stairs their own label, map that label to a sensible height (for example the landing) + with `--floor-z`; if it gives a height along the stair, supply it as a `z` column, + which is added to the floor height. +- **Coordinates.** Export the building geometry for Mobius from the **same CAD/DXF** + that the EXODUS geometry was built from, with the same origin and in metres, so the + occupants line up with walls and doors. Mobius does not shift or scale trajectories. +- **What does not transfer.** Occupant attributes (age, gender, response time, travel + speeds) and EXODUS fire-hazard results (FED, toxicity) are not imported. Mobius takes + fire and tenability data from B-RISK separately. To keep an attribute visible in + Mobius, build the agent names yourself (for example `"17 Adult Male"`) and pass them + as `entity_names` to `write_mobius_h5()` from Python. + +## 4. Check and open + +```bash +python validate_mobius_h5.py run.h5 +ProjectMobius.exe -MobiusPedestrian="C:/runs/run.h5" -MobiusGeometry="C:/runs/building.fbx" +``` + +Check the validator's `info` lines against what you know of the run: occupant count, +time span (about the evacuation time), and the x/y/z extents of the building. + +## See also + +- [Pedestrian Data Format](PEDESTRIAN-DATA-FORMAT.md): what every field means +- [HDF5 Export Guide](HDF5-EXPORT-GUIDE.md): the converter, the Python writer and the C route in full diff --git a/docs/HDF5-EXPORT-GUIDE.md b/docs/HDF5-EXPORT-GUIDE.md new file mode 100644 index 000000000..17b430ed8 --- /dev/null +++ b/docs/HDF5-EXPORT-GUIDE.md @@ -0,0 +1,318 @@ +# HDF5 Export Guide + +How to write your simulation's pedestrian output as a Project Mobius `.h5` file, using +either Python or the HDF5 C library that ships with this repository. + +What the file must contain is defined in [Pedestrian Data Format](PEDESTRIAN-DATA-FORMAT.md). +This page is the practical side: getting the code, building the library, writing the +file, checking it, and opening it in Mobius. + +## Contents + +- [Choose a route](#choose-a-route) +- [Get the files from the repository](#get-the-files-from-the-repository) +- [Route A: Python](#route-a-python) +- [Route B: C or C++ with the bundled HDF5](#route-b-c-or-c-with-the-bundled-hdf5) +- [Validate the file](#validate-the-file) +- [Open it in Mobius](#open-it-in-mobius) +- [Troubleshooting](#troubleshooting) + +## Choose a route + +| Your situation | Use | +| --- | --- | +| Your model already writes a CSV/text file of positions over time | [Route A](#convert-a-csv-export), `from-csv`: no programming | +| You post-process results in Python, or can | [Route A](#write-from-your-own-python-code), `write_mobius_h5()` | +| You want your C/C++ simulator to write the file directly | [Route B](#route-b-c-or-c-with-the-bundled-hdf5), the bundled HDF5 C library | +| Your model is buildingEXODUS | [Exporting from buildingEXODUS](EXODUS-EXPORT.md) (Route A underneath) | + +Both routes produce equivalent files (string field sizes differ, nothing Mobius cares +about). Everything below lives in +[`docs/examples/pedestrian-hdf5/`](examples/pedestrian-hdf5/): + +| File | What it is | +| --- | --- | +| [`mobius_h5_writer.py`](examples/pedestrian-hdf5/mobius_h5_writer.py) | Python writer: importable `write_mobius_h5()` function, plus `from-csv` and `demo` commands | +| [`validate_mobius_h5.py`](examples/pedestrian-hdf5/validate_mobius_h5.py) | Checks any `.h5` against the rules the Mobius importer relies on | +| [`write_mobius_h5.c`](examples/pedestrian-hdf5/write_mobius_h5.c) | Self-contained C example using the HDF5 C API | +| [`CMakeLists.txt`](examples/pedestrian-hdf5/CMakeLists.txt) | Builds the C example against your HDF5 install | +| [`requirements.txt`](examples/pedestrian-hdf5/requirements.txt) | Python dependencies (`numpy`, `h5py`) | + +## Get the files from the repository + +Repository: + +You do **not** need Unreal Engine, Git LFS or the full repository to write data files. + +**Python only (Route A):** download `mobius_h5_writer.py`, `validate_mobius_h5.py` and +`requirements.txt` from [`docs/examples/pedestrian-hdf5/`](examples/pedestrian-hdf5/) +through the GitHub web interface (open each file, then *Download raw file*). Nothing +else is needed. + +**C/C++ (Route B), or to have everything locally:** fetch only the docs and the bundled +HDF5 source with a sparse clone (about 30 MB to download, 200 MB on disk, instead of the +multi-gigabyte full repository): + +```bash +git clone --depth 1 --filter=blob:none --sparse https://github.com/sir306/ProjectMobius.git +cd ProjectMobius +git sparse-checkout set docs UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/hdf5-2.0.0 +``` + +The bundled library is the **complete, unmodified HDF5 2.0.0 source**, at +`UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/hdf5-2.0.0/`. +It is the same version Mobius reads with; files written with older HDF5 releases read +fine too, so the library your tool already links is usually good enough. Its licence is in +`hdf5-2.0.0/LICENSE` and summarised in +[THIRD-PARTY-LICENSES.md](../THIRD-PARTY-LICENSES.md). + +## Route A: Python + +### Install + +Python 3.8 or newer: + +```bash +pip install -r docs/examples/pedestrian-hdf5/requirements.txt +``` + +`h5py` wheels contain their own copy of HDF5 (2.0.0 in current releases), so no +compiler or separate HDF5 install is needed. + +### Write from your own Python code + +Put `mobius_h5_writer.py` next to your script (or on `sys.path`). Collect one entry per +agent per timestep the agent is present, then call `write_mobius_h5()` once at the end: + +```python +from mobius_h5_writer import write_mobius_h5, heading_and_speed_from_positions + +# One entry per (agent, timestep) the agent is in the building. +timestep_idx = [0, 0, 1, 1, 2] # time = timestep_idx * 0.1 s +entity_id = [0, 1, 0, 1, 0] # dense 0..N-1; remap your own IDs first +x = [0.0, 5.0, 0.12, 4.9, 0.24] # metres +y = [1.0, 2.0, 1.0, 2.0, 1.0] +z = [0.0, 0.0, 0.0, 0.0, 0.0] # floor height, metres + +# Use your model's own heading/speed if it has them; otherwise compute them: +rotation, speed = heading_and_speed_from_positions(timestep_idx, entity_id, x, y, z, 0.1) + +write_mobius_h5( + "run.h5", timestep_idx, entity_id, x, y, z, + sampling_interval_s=0.1, + rotation_deg=rotation, # degrees, CCW from +x = degrees(atan2(dy, dx)) + speed_mps=speed, # m/s + entity_names=["Occupant 101", "Occupant 102"], # e.g. your original IDs +) +``` + +The function refuses input that would load badly (sparse IDs, duplicate rows, NaN +positions, negative speeds) and handles every type detail for you: fixed-length strings, +32-bit fields, `duration`, and gzip compression for large files. + +`heading_and_speed_from_positions()` applies the same rule Mobius uses when a file has +no heading or speed (face the next position, hold the heading while standing still). +Leaving `rotation_deg`/`speed_mps` out also works, but computing them here keeps large +files quick to load in Mobius releases from before October 2026, which derive them +slowly. + +### Convert a CSV export + +If your model can export a "long" table, **one row per agent per output time**, convert +it directly: + +```text +id,time,x,y,floor +1,0.0,2.00,5.00,1 +1,0.5,2.50,5.00,1 +2,0.0,0.00,1.00,0 +... +``` + +```bash +python mobius_h5_writer.py from-csv positions.csv run.h5 --dt 0.1 --floor-z "0=0,1=3.5" +``` + +- Required columns: `id`, `time` (seconds), `x`, `y` (metres). Optional: `z`, `floor`. + Different column names: `--col-id OccupantNo --col-time T --col-x X ...`. +- `--dt` is the output interval. Every agent is resampled onto `0, dt, 2·dt, …` + between its first and last appearance, so the CSV times do not need to be evenly + spaced. Without `--dt` the median gap in the CSV is used; pass it explicitly. +- `--interp` chooses how positions between CSV rows are filled: + - `linear` (default) for **periodic snapshots** (a row for every agent every *n* + seconds): straight-line motion between rows. + - `hold` for **event logs** (a row only when an agent moves): the agent stays at its + last recorded position until the next row. Linear interpolation would turn a wait + in a queue into a slow creep and hide the congestion. +- `--floor-z "0=0,1=3.5,2=7"` maps a floor label to a floor height in metres; + `--floor-height 3.5` does the same for numeric floors with a uniform storey height. + Both add to `z` if a `z` column is present too. If the CSV has a floor column, one of + the two is required (or `--col-floor none` to ignore it), so a multi-storey run cannot + silently collapse onto one floor. +- `--scale 0.001` converts millimetres to metres (and so on). +- IDs can be any text or numbers; they are renumbered 0 … N−1 and the original goes into + the agent name (`Agent 17`; change the prefix with `--name-prefix`). +- The earliest time in the CSV becomes time 0 in Mobius. +- Heading and speed are computed from the resampled positions with + `heading_and_speed_from_positions()`. `--positions-only` leaves them out for Mobius to + derive instead (fast in current builds, slow in releases from before October 2026). + +### Quick demo + +```bash +python mobius_h5_writer.py demo demo.h5 +``` + +writes three people crossing a 10 m corridor, which you can open in Mobius to confirm +the toolchain end to end. + +## Route B: C or C++ with the bundled HDF5 + +### 1. Build the bundled HDF5 + +Needs CMake 3.21+ and a C compiler (Visual Studio on Windows, GCC/Clang on Linux, +Xcode Command Line Tools on macOS). From the repository root, pick an install folder +for the library (`HDF5_PREFIX` below): + +**Windows (PowerShell):** + +```powershell +$HDF5_SRC = "UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/hdf5-2.0.0" +$HDF5_PREFIX = "C:/libs/hdf5-2.0.0" +cmake -S $HDF5_SRC -B build-hdf5 -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON -DHDF5_BUILD_TOOLS=ON -DHDF5_BUILD_HL_LIB=OFF -DHDF5_BUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF "-DCMAKE_INSTALL_PREFIX=$HDF5_PREFIX" +cmake --build build-hdf5 --config Release --parallel +cmake --install build-hdf5 --config Release +``` + +**Linux / macOS:** + +```bash +HDF5_SRC=UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/ThirdParty/hdf5-2.0.0 +HDF5_PREFIX=$HOME/libs/hdf5-2.0.0 +cmake -S "$HDF5_SRC" -B build-hdf5 -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ + -DHDF5_BUILD_TOOLS=ON -DHDF5_BUILD_HL_LIB=OFF \ + -DHDF5_BUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF \ + -DCMAKE_INSTALL_PREFIX="$HDF5_PREFIX" +cmake --build build-hdf5 --config Release --parallel +cmake --install build-hdf5 --config Release +``` + +This installs the static C library, headers, a CMake package and the command-line tools +(`h5dump`, `h5ls`, …) into `HDF5_PREFIX`. It needs no network access. + +Options worth knowing: + +- **gzip compression** (optional, only worth it for big files): add + `-DHDF5_ENABLE_ZLIB_SUPPORT=ON -DHDF5_ALLOW_EXTERNAL_SUPPORT=TGZ + -DZLIB_TGZ_ORIGPATH=https://github.com/madler/zlib/releases/download/v1.3.1 + -DZLIB_TGZ_NAME=zlib-1.3.1.tar.gz -DHDF5_USE_ZLIB_STATIC=ON`, the flags the Mobius + superbuild itself uses. This downloads zlib during configure. The example enables gzip + automatically when the library supports it. +- **Fortran** or the **C++ wrapper API**: add `-DHDF5_BUILD_FORTRAN=ON` or + `-DHDF5_BUILD_CPP_LIB=ON`. The C API works from C++ unchanged, so the wrapper is + optional. + +### 2. Build and run the example + +```bash +cmake -S docs/examples/pedestrian-hdf5 -B build-example -DCMAKE_PREFIX_PATH="" +cmake --build build-example --config Release +``` + +Run it (the executable is in `build-example/Release/` on Windows, `build-example/` elsewhere): + +```bash +write_mobius_h5 example.h5 # positions, heading, speed and mode +write_mobius_h5 example.h5 --positions-only # positions only; Mobius derives heading/speed +/bin/h5dump -H example.h5 # inspect the layout +``` + +If CMake cannot find HDF5, pass the folder containing `hdf5-config.cmake` instead: +`-DHDF5_DIR=/cmake` on Windows (search the prefix for that file elsewhere). + +### 3. Put it in your simulator + +[`write_mobius_h5.c`](examples/pedestrian-hdf5/write_mobius_h5.c) is written to be +copied. Keep these parts as they are: + +- the `entity_t` and `sample_t` structs: member order and widths are the contract; +- `write_metadata`, `write_entities`, `write_samples` and `write_table`. + +Replace the demo loop in `main()` with your own: + +1. Number your agents 0 … N−1 as they are created (keep your own ID in `name`). +2. At each output interval `dt`, append one `sample_t` per agent currently in the + building, with `timestep_idx` = output count so far. Skip agents that have left. + Fill `rotation`/`speed` from your model; if it has no heading, use + `atan2(dy, dx)` towards the agent's next position (hold the last value while it stands + still) and distance ÷ time for speed. Positions-only output + (`with_rotation_speed_mode = 0`) also loads; Mobius then derives heading and speed + itself, which releases from before October 2026 do slowly for thousands of agents. +3. At the end, fill `sim_time_s` per agent and write the file: + `duration = (num_timesteps − 1) × dt`, `max_num_entities = N`. + +Linking in your own build: + +- **CMake:** `find_package(HDF5 CONFIG REQUIRED)` and link the target `hdf5-static` + (see the example's [`CMakeLists.txt`](examples/pedestrian-hdf5/CMakeLists.txt)). +- **Other build systems:** add `/include` to the include path and link + `/lib/libhdf5.lib` plus `shlwapi.lib` on Windows, or + `/lib/libhdf5.a` plus the system libraries it needs (typically + `-lm -ldl -lpthread` on Linux). `/lib/libhdf5.settings` lists exactly how + the library was built. + +For very long runs you can write samples in batches instead of holding them all in +memory: create `/simulation/samples` chunked with an unlimited maximum size and grow it +with `H5Dset_extent` after each batch. The resulting file is identical to the +single-write version. + +## Validate the file + +Always run the validator before sending a file on or opening it in Mobius: + +```bash +python docs/examples/pedestrian-hdf5/validate_mobius_h5.py run.h5 +``` + +```text +[PASS] run.h5 + info rotation matches the direction of travel for 100% of moving samples + info 2 entities, 9 timesteps (4 s at 0.5 s), 16 samples + info x [0.00, 6.00] m, y [1.00, 5.00] m, z [0.00, 3.50] m +``` + +It exits with code 1 on any error, so it can gate an automated export. Check the +`info` lines too: an `x`/`y` range in thousands usually means millimetres, and a time +span far from your run length usually means `--dt` or `sampling_rate` is wrong. + +## Open it in Mobius + +- In the app: load the `.h5` from the pedestrian data **Browse** button. +- From the command line or another program: + + ```bash + ProjectMobius.exe -MobiusPedestrian="C:/runs/run.h5" -MobiusGeometry="C:/runs/building.fbx" + ``` + + See [Launch with Preloaded Files](BUILD-AND-RUN.md#launch-with-preloaded-files). + +`Ctrl` + hover over an agent shows its entity details (`name`, `sim_time_s`, +`max_speed`, `m_plane`, `map`), a quick way to confirm IDs mapped as intended. + +## Troubleshooting + +| Symptom in Mobius | Usual cause | +| --- | --- | +| *"HDF5 file has unrecognized format"* | No `/metadata` group or no `/simulation/samples` dataset. | +| Agents everywhere / at absurd coordinates / all at the origin | A table member has the wrong type (64-bit float, big-endian) or the members are in the wrong order. Run the validator. | +| Garbled agent names, or a crash on load | A variable-length string inside `/entities` or `/simulation/samples`. Use fixed-length strings. | +| *"Unexpected entity IDs"* | `entity_id` not 0 … N−1 (often 1-based), or `max_num_entities` set to the peak instead of the total. | +| *"Incomplete entity data"* | Some entity numbers never appear in the samples (gaps in your numbering). | +| *"Data may be truncated"* | `duration` larger than `(num_timesteps − 1) × sampling_rate`. | +| Playback far too fast or slow | `sampling_rate` given as a frequency (10) instead of an interval (0.1), or output not evenly spaced. | +| Agents 100× too small/large, or mirrored against the building | Positions not in metres, or geometry exported with a different origin/axes. With a WKT floor outline, mirroring is expected: negate the WKT's y values ([why](PEDESTRIAN-DATA-FORMAT.md#optional-root-attribute-wkt_geometry)). | +| All agents face one way | Zeros written in `rotation`. Leave the member out instead. | +| An error dialog per sample about NaN or speed | NaN placeholders for absent agents, or negative speeds. Write no row instead. | +| *"Unable to import Mobius HDF5 data"* with a compression error in the log | `lzf`/`szip`/Blosc compression. Use gzip or none. | +| File loads as Jülich data | It contains a root `/trajectory` dataset. Remove it. | diff --git a/docs/PEDESTRIAN-DATA-FORMAT.md b/docs/PEDESTRIAN-DATA-FORMAT.md new file mode 100644 index 000000000..23a0df78b --- /dev/null +++ b/docs/PEDESTRIAN-DATA-FORMAT.md @@ -0,0 +1,331 @@ +# Pedestrian Data Format + +This page defines the pedestrian (agent trajectory) data Project Mobius loads, field +by field, so that any evacuation or crowd model can export results Mobius will +replay correctly. + +- **Writing a file from your own simulator?** Read this page for *what* to write, then + [HDF5 Export Guide](HDF5-EXPORT-GUIDE.md) for *how* (Python or the HDF5 C library + bundled in this repository). +- **Using buildingEXODUS?** See [Exporting from buildingEXODUS](EXODUS-EXPORT.md). +- **Checking a file?** Run + [`validate_mobius_h5.py`](examples/pedestrian-hdf5/validate_mobius_h5.py) on it. It + applies every rule on this page. + +> **Why this page is strict.** The Mobius HDF5 reader decodes most fields by +> **position and fixed width**, without converting types. A position written as +> 64-bit float, or a name written as a variable-length string, does not produce an +> error. It loads garbage. Follow the types in the tables below exactly. + +## Contents + +- [Which format to use](#which-format-to-use) +- [The data model in one minute](#the-data-model-in-one-minute) +- [Coordinates, units and time](#coordinates-units-and-time) +- [Mobius HDF5 layout](#mobius-hdf5-layout) +- [Rules the importer relies on](#rules-the-importer-relies-on) +- [Heading (rotation) convention](#heading-rotation-convention) +- [Floors, stairs and geometry alignment](#floors-stairs-and-geometry-alignment) +- [What Mobius does on load](#what-mobius-does-on-load) +- [Mobius JSON layout](#mobius-json-layout) +- [Checklist](#checklist) +- [Source references](#source-references) + +## Which format to use + +| Format | Use it when | Notes | +| --- | --- | --- | +| **Mobius HDF5** (`.h5`) | **Always, for new exports.** | Fast to load and small. Heading and speed are optional (Mobius can derive them). | +| Mobius JSON (`.json`) | Small hand-made test files. | Human-readable, but slow and memory-heavy for big runs. Heading and speed **must** be supplied. Convert to HDF5 with [`json_to_hdf5_converter.py`](../UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Scripts/json_to_hdf5_converter.py). | +| Jülich trajectory HDF5 (`.h5` with `/trajectory`) | Only for datasets from the Jülich pedestrian-experiment archive. | Read-only compatibility path, described in [Agent Sample Data](AGENT-SAMPLE-DATA.md). **Do not use it for new exports**: it handles axes differently and ignores any heading or speed you add. | + +The file extension must be **`.h5`** (any case). Mobius does not open `.hdf5` or `.hdf` +as pedestrian data. + +## The data model in one minute + +- An **entity** is one person (agent) in the run. Entities are numbered + **0, 1, 2, … N-1** with no gaps. +- A **timestep** is one output instant. Timesteps are numbered 0, 1, 2, … and are + **evenly spaced**: timestep *k* happens at *k* × `sampling_rate` seconds. +- A **sample** is one entity at one timestep: where it is, which way it faces, how fast it + moves. +- An entity that is not in the building at a timestep (not yet arrived, already + evacuated) simply has **no sample** at that timestep. Never write a placeholder row. + +## Coordinates, units and time + +| Quantity | Unit | Convention | +| --- | --- | --- | +| Position `x`, `y` | metres | Plan view, **right-handed**: +x to the right (east), +y up the page (north). The same frame as a CAD floor plan. | +| Position `z` | metres | Up. Height of the floor the person is standing on (0 on the ground floor), **not** head height. | +| Heading `rotation` | degrees | Direction of travel, counter-clockwise from +x. See [Heading convention](#heading-rotation-convention). | +| Speed `speed` | m/s | Magnitude, never negative. | +| Time | seconds | `time = timestep_idx × sampling_rate`. The run starts at time 0. | + +Mobius converts metres to Unreal centimetres and mirrors the y axis into Unreal's +left-handed frame itself. Do **not** pre-convert either. Seen from above in Mobius, agents +keep the layout of your plan; they are not mirrored. This was checked in the app on +2026-10-22. + +## Mobius HDF5 layout + +```text +run.h5 +├── /metadata group, 5 scalar attributes +├── /entities table, one row per entity +└── /simulation + ├── samples table, one row per entity per timestep present + ├── timesteps float32[num_timesteps] (recommended) + └── samples_per_timestep int32[num_timesteps] (recommended) +``` + +A file is recognised as Mobius format when it has a `/metadata` group **and** a +`/simulation/samples` dataset, and **no** root `/trajectory` dataset (a `/trajectory` +makes Mobius treat the whole file as Jülich format). + +### `/metadata` attributes + +Scalar attributes on the `/metadata` group. Numeric attributes are converted on read, so +any integer or float type works. + +| Attribute | Type | Meaning | If missing | +| --- | --- | --- | --- | +| `sampling_rate` | float | **Seconds between timesteps**, e.g. `0.1` for 10 Hz output. Despite the name it is an interval, not a frequency. | 0.1 | +| `duration` | float | Time of the **last** timestep in seconds: `(num_timesteps − 1) × sampling_rate`. Used as the playback length. | taken from the samples | +| `max_num_entities` | integer | **Total** number of entities in the run (N), which is also the row count of `/entities`. Not the peak number present at one time. `0` means "use the `/entities` row count". | `/entities` row count | +| `is_si` | integer 0/1 | `1` = positions in metres. Always write `1`. | 1 | +| `is_deg` | integer 0/1 | `1` = `rotation` in degrees, `0` = radians. | 1 | + +Write `is_si` and `is_deg` as 8-bit unsigned integers (`uint8`). Python `bool` also +works with h5py, but plain integers are unambiguous for every writer. + +### `/entities` table + +A 1-D compound dataset, **row *i* describes entity *i***. Mobius reads the six members +**by position with these exact types**: keep the order and the widths. + +| # | Member | HDF5 type | Meaning | +| --- | --- | --- | --- | +| 0 | `id` | 32-bit little-endian signed int (`H5T_STD_I32LE`, numpy ``), at least one +byte longer than the longest value so a terminating NUL always fits. Use ASCII or UTF-8. +A variable-length string (what h5py writes for a Python `str`) is decoded as pointer +bytes and shows garbage or crashes. + +### Compression + +Optional. Mobius can read **gzip (deflate)**, **shuffle** and **fletcher32** filters on +chunked datasets. It cannot read `lzf`, `szip`, Blosc or any other plugin filter: those +files fail to load. + +### Optional root attribute: `wkt_geometry` + +A string attribute on the root group holding a Well-Known-Text `POLYGON` or +`GEOMETRYCOLLECTION` of the floor outline in the same metres. When present, Mobius +offers to load it as geometry. + +> **WKT geometry is mirrored against Mobius-format agents.** Mobius mirrors the agents' +> y axis into Unreal's frame (see [Coordinates](#coordinates-units-and-time)) but loads +> WKT, both this attribute and a separate `.wkt` file, **without** that mirror. A WKT +> outline therefore appears flipped across the x axis relative to the agents. This was +> verified in the app on 2026-10-22. Until it is fixed, **negate every y value** in WKT +> you pair with Mobius-format or JSON data (`POLYGON((0 0, 10 0, 10 4, …))` becomes +> `POLYGON((0 0, 10 0, 10 -4, …))`); that was verified to line the two up. Jülich-format +> files are unaffected, because Mobius mirrors neither their agents nor their WKT. + +## Rules the importer relies on + +Breaking these does not always stop the load, which is why they are listed here. + +1. **Dense entity numbers.** `entity_id` runs 0 … N−1 with no gaps, N = + `max_num_entities` = rows in `/entities`. Remap your simulator's own IDs (which may + start at 1 or have gaps) and keep the original in `name`. Samples with an id ≥ N are + dropped with an *"Unexpected entity IDs"* warning; ids that never appear raise + *"Incomplete entity data"*. +2. **Even time spacing.** Sample *k* is placed at *k* × `sampling_rate`; the values in + `/simulation/timesteps` are not used for placement. If your model writes output at + irregular times (event-driven models often do), resample to a fixed interval first. + [`mobius_h5_writer.py from-csv`](HDF5-EXPORT-GUIDE.md#convert-a-csv-export) does + this. +3. **One sample per entity per timestep.** +4. **No NaN or infinite values, no negative speeds.** Each bad value raises its own error + dialog and is replaced by zero. To show that an agent is absent, write no row. +5. **`duration` = time of the last timestep** = `(num_timesteps − 1) × sampling_rate`. + A larger value triggers a false *"Data may be truncated"* warning. +6. **Little-endian 32-bit numbers in the tables.** Every x86-64 and ARM (Apple Silicon) + machine writes little-endian by default; just do not request 64-bit or big-endian + types for table members. +7. **Whole file in memory.** Mobius reads the entire samples table at load and keeps the + processed trajectories in RAM, so memory grows with agents × timesteps. 0.1 s output + is typical; for very long or very large runs a coarser `sampling_rate` reduces memory + in proportion. + +## Heading (rotation) convention + +`rotation` is the direction the person faces, measured **counter-clockwise from +x in +your plan frame**, in degrees (or radians with `is_deg = 0`). For a person walking with +plan velocity (dx, dy): + +```text +rotation = degrees(atan2(dy, dx)) +``` + +| Walking towards | rotation | +| --- | --- | +| +x (east, right on the plan) | 0 | +| +y (north, up the plan) | 90 | +| −x (west) | 180 (or −180) | +| −y (south) | 270 (or −90) | + +Any range works (0–360 or −180–180). Two checks confirm this convention: + +- **Bundled Simulex sample.** In `iso-test-1.h5` the stored rotation matches + `atan2(dy, dx)` of the movement for 98.6 % of moving samples. +- **In the app, 2026-10-22.** A demo file written with this rule renders the same as the + same file with `rotation` left out for Mobius to derive: agent silhouettes overlap at + IoU ≥ 0.98, with no measurable offset. Agents walking towards the camera show their + faces; agents walking away show their backs. + +## Floors, stairs and geometry alignment + +- `position_z` is the height of the floor surface in metres. On stairs, use the height of + the stair surface under the person (interpolating between landings is fine). +- Multi-storey output from models that report a floor number instead of a height: convert + floor number → floor height before writing (`--floor-z` in the CSV converter). +- Export the building geometry from the **same CAD model, origin and units** as the + simulation, so (0, 0, 0) is the same point in both. Mobius does not offset or rescale + trajectories to fit the geometry. +- WKT floor outlines need their y values negated to line up with Mobius-format agents + (see [`wkt_geometry`](#optional-root-attribute-wkt_geometry)). + +## What Mobius does on load + +1. Detects the format (`/trajectory` → Jülich; `/metadata` + `/simulation/samples` → + Mobius). +2. Reads metadata, the entity table and every sample. +3. Converts positions to centimetres (`× 100`) and mirrors y into Unreal's frame. +4. Converts `rotation` to an Unreal yaw. +5. If the samples table has **no** `rotation` member, derives heading from the direction + to the agent's next position (holding the last heading while stationary). If it has no + `speed` member, derives speed from the distance to the next position divided by + `sampling_rate`. This takes one pass over the samples (well under a second for + millions of samples); builds from before October 2026 used a much slower loop. +6. Checks entity coverage and timestep count, and reports problems in an error dialog. + +## Mobius JSON layout + +Accepted for small files. The same rules apply (dense ids, even spacing, metres, +heading convention), with these differences: + +- `rotation` and `speed` are **required** in every sample. JSON cannot ask Mobius to + derive them; missing values load as 0. +- Timesteps are numbered by their **position** in the `simulation` array. The `time` + value is not used, so the array must be evenly spaced and contain an entry (possibly + with an empty `samples` array) for every timestep. +- Metadata accepts `isSI`/`is_si` and `isDeg`/`is_deg`. + +```json +{ + "metadata": { + "duration": 0.1, + "sampling_rate": 0.1, + "max_num_entities": 2, + "isSI": true, + "isDeg": true + }, + "entities": [ + { "id": 0, "name": "Agent 0", "simTimeS": "0.1", "max_speed": 1.2, "m_plane": "Floor 0", "map": 0 }, + { "id": 1, "name": "Agent 1", "simTimeS": "0.0", "max_speed": 1.0, "m_plane": "Floor 0", "map": 0 } + ], + "simulation": [ + { "time": 0.0, "samples": [ + { "entity": 0, "position": { "x": 0.00, "y": 1.0, "z": 0.0 }, "rotation": 0.0, "speed": 1.2, "mode": "walk" }, + { "entity": 1, "position": { "x": 5.00, "y": 2.0, "z": 0.0 }, "rotation": 180.0, "speed": 1.0, "mode": "walk" } + ]}, + { "time": 0.1, "samples": [ + { "entity": 0, "position": { "x": 0.12, "y": 1.0, "z": 0.0 }, "rotation": 0.0, "speed": 1.2, "mode": "walk" } + ]} + ] +} +``` + +Convert a JSON file to HDF5 with +[`json_to_hdf5_converter.py`](../UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Scripts/json_to_hdf5_converter.py): + +```bash +python json_to_hdf5_converter.py run.json run.h5 +``` + +## Checklist + +- [ ] File ends in `.h5`, has `/metadata`, `/entities`, `/simulation/samples`, and no `/trajectory`. +- [ ] `sampling_rate` is the interval in seconds (0.1 for 10 Hz), and output is evenly spaced. +- [ ] `duration` = `(num_timesteps − 1) × sampling_rate`. +- [ ] `max_num_entities` = total agents = rows in `/entities`; `entity_id` runs 0 … N−1. +- [ ] `/entities` members in order `id, name, sim_time_s, max_speed, m_plane, map` as `int32, string, float32, float32, string, int32`. +- [ ] `/simulation/samples` starts `timestep_idx, entity_id, position_x, position_y, position_z` as `int32, int32, float32 ×3`. +- [ ] Every string member is fixed-length with room for a NUL. +- [ ] Positions in metres, right-handed plan frame, z = floor height. +- [ ] `rotation` / `speed` are real values (from the model or computed from positions), never zeros. Leaving them out is fine for current Mobius builds. +- [ ] No NaN, no negative speed, no placeholder rows for absent agents. +- [ ] Compression, if any, is gzip. +- [ ] [`validate_mobius_h5.py`](examples/pedestrian-hdf5/validate_mobius_h5.py) reports PASS. + +## Source references + +The authority for every rule above is the importer code: + +- `UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/MobiusDataImporter/Private/Hdf5SimulationReader.cpp`: format detection and field decoding +- `UnrealFolder/ProjectMobius/Plugins/MobiusDataImporter/Source/MobiusDataImporter/Private/MobiusAgentDataImporter.cpp`: file-type dispatch +- `UnrealFolder/ProjectMobius/Source/ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp`: units, axes, heading, derivation and the load-time checks +- `UnrealFolder/ProjectMobius/Source/ProjectMobiusTests/Private/Tests/Hdf5ImportMatrixTest.cpp`: automated import test that writes Mobius and Jülich files with the HDF5 C API diff --git a/docs/README.md b/docs/README.md index c72cda122..ec8a77300 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,12 @@ Packaged builds mirror a selected subset of these pages into UnrealFolder/Projec - [Data Pipeline](DATA-PIPELINE.md) - [Agent Sample Data](AGENT-SAMPLE-DATA.md) +## Exporting Pedestrian Data From Other Models + +- [Pedestrian Data Format](PEDESTRIAN-DATA-FORMAT.md): field-by-field definition of the files Mobius loads +- [HDF5 Export Guide](HDF5-EXPORT-GUIDE.md): write those files from Python or with the bundled HDF5 C library +- [Exporting from buildingEXODUS](EXODUS-EXPORT.md) + ## Research - [Research and Publications](RESEARCH.md) diff --git a/docs/examples/pedestrian-hdf5/CMakeLists.txt b/docs/examples/pedestrian-hdf5/CMakeLists.txt new file mode 100644 index 000000000..a081bd821 --- /dev/null +++ b/docs/examples/pedestrian-hdf5/CMakeLists.txt @@ -0,0 +1,39 @@ +# Builds write_mobius_h5 against an installed HDF5, normally the copy you built +# from the HDF5 source bundled in this repository: +# +# cmake -S . -B build -DCMAKE_PREFIX_PATH= +# cmake --build build --config Release +# +# If CMake cannot find HDF5, point it at the folder holding hdf5-config.cmake +# instead: -DHDF5_DIR=/cmake on Windows (search the prefix for that file +# on Linux/macOS). +# +# See docs/HDF5-EXPORT-GUIDE.md for building HDF5 itself. + +cmake_minimum_required(VERSION 3.21) +project(mobius_h5_writer_example C) + +find_package(HDF5 CONFIG REQUIRED) + +add_executable(write_mobius_h5 write_mobius_h5.c) +set_target_properties(write_mobius_h5 PROPERTIES C_STANDARD 99 C_STANDARD_REQUIRED ON) + +# The bundled HDF5 exports un-namespaced targets (hdf5-static / hdf5-shared); +# some package managers add an hdf5:: namespace. Prefer static so the program +# runs without an HDF5 DLL beside it. +set(_mobius_hdf5_target "") +foreach(_candidate hdf5-static hdf5::hdf5-static hdf5-shared hdf5::hdf5-shared) + if(TARGET ${_candidate}) + set(_mobius_hdf5_target ${_candidate}) + break() + endif() +endforeach() +if(NOT _mobius_hdf5_target) + message(FATAL_ERROR "The HDF5 package found at ${HDF5_DIR} exports none of hdf5-static, hdf5-shared (with or without the hdf5:: namespace).") +endif() +message(STATUS "Linking ${_mobius_hdf5_target}") +target_link_libraries(write_mobius_h5 PRIVATE ${_mobius_hdf5_target}) + +if(NOT MSVC) + target_link_libraries(write_mobius_h5 PRIVATE m) +endif() diff --git a/docs/examples/pedestrian-hdf5/mobius_h5_writer.py b/docs/examples/pedestrian-hdf5/mobius_h5_writer.py new file mode 100644 index 000000000..a76a2ee79 --- /dev/null +++ b/docs/examples/pedestrian-hdf5/mobius_h5_writer.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +""" +Write pedestrian trajectories in the Project Mobius HDF5 layout. + +Use it three ways: + +1. As a library from your own simulator or post-processing script: + + from mobius_h5_writer import write_mobius_h5, heading_and_speed_from_positions + rot, spd = heading_and_speed_from_positions(timestep_idx, entity_id, x, y, z, 0.1) + write_mobius_h5("run.h5", timestep_idx, entity_id, x, y, z, + sampling_interval_s=0.1, rotation_deg=rot, speed_mps=spd) + +2. To convert a "long" CSV (one row per agent per output time) exported by any + evacuation model, resampling it onto a uniform time grid: + + python mobius_h5_writer.py from-csv positions.csv run.h5 --dt 0.1 + + Required columns: id, time, x, y. Optional: z, floor (floor needs --floor-z + or --floor-height). Rename yours with --col-id/--col-time/--col-x/--col-y/ + --col-z/--col-floor. Heading and speed are computed from the positions. + +3. To produce a small demo file that you can open in Mobius straight away: + + python mobius_h5_writer.py demo demo.h5 + +Layout reference: docs/PEDESTRIAN-DATA-FORMAT.md +Check any file you write with validate_mobius_h5.py before sending it on. + +Requires: numpy, h5py (pip install -r requirements.txt) + +MIT License - Copyright (c) 2026 ProjectMobius contributors +""" + +import argparse +import csv +import math +import sys +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +import h5py +import numpy as np + + +def _fixed_strings(values: Sequence[str]) -> np.ndarray: + """Encode strings as fixed-length, NUL-terminated byte strings. + + Mobius reads compound string members as C strings, so the field must be at + least one byte longer than the longest value. Variable-length strings (what + h5py writes for a plain Python str) are NOT readable inside a compound. + """ + encoded = [str(v).encode("utf-8") for v in values] + width = max((len(v) for v in encoded), default=0) + 1 + return np.array(encoded, dtype=f"S{width}") + + +def heading_and_speed_from_positions(timestep_idx, entity_id, x, y, z, sampling_interval_s: float): + """Per-sample heading (degrees, CCW from +x) and speed (m/s) from positions. + + Uses the same rule Mobius applies when a file has no rotation/speed members: + heading points at the agent's next position and is held while the agent stands + still (every component of the step <= 1 mm); speed is the 3-D distance to the + next position over the time between them; an agent's last sample repeats its + previous values. Computing it here keeps files quick to load in Mobius + releases from before October 2026, whose own derivation was slow for + thousands of agents. + + Returns (rotation_deg, speed_mps), aligned with the input order. + """ + t_idx = np.asarray(timestep_idx, dtype=np.int64) + ids = np.asarray(entity_id, dtype=np.int64) + pos = np.column_stack([np.asarray(v, dtype=np.float64) for v in (x, y, z)]) + rotation = np.zeros(t_idx.size) + speed = np.zeros(t_idx.size) + + order = np.lexsort((t_idx, ids)) # group by agent, then time + bounds = np.flatnonzero(np.diff(ids[order])) + 1 + for group in np.split(order, bounds): + if group.size < 2: + continue # a single sample has no movement to measure + step = np.diff(pos[group], axis=0) + elapsed = np.diff(t_idx[group]) * sampling_interval_s + if (elapsed <= 0).any(): + raise ValueError(f"entity {ids[group[0]]} has more than one sample in the same timestep") + moving = (np.abs(step) > 0.001).any(axis=1) + + heading = np.full(group.size, np.nan) + heading[:-1][moving] = np.degrees(np.arctan2(step[moving, 1], step[moving, 0])) + valid = np.flatnonzero(~np.isnan(heading)) + if valid.size: + # Hold the last heading while standing still (and on the last sample); + # before the first move, face the way the agent will first walk. + filled = np.maximum.accumulate(np.where(~np.isnan(heading), np.arange(group.size), -1)) + heading = np.where(filled >= 0, heading[np.maximum(filled, 0)], heading[valid[0]]) + else: + heading[:] = 0.0 + + spd = np.empty(group.size) + spd[:-1] = np.linalg.norm(step, axis=1) / elapsed + spd[-1] = spd[-2] + rotation[group] = heading + speed[group] = spd + return rotation, speed + + +def write_mobius_h5( + path, + timestep_idx, + entity_id, + x, + y, + z, + sampling_interval_s: float, + rotation_deg=None, + speed_mps=None, + mode=None, + entity_names: Optional[Sequence[str]] = None, + entity_planes: Optional[Sequence[str]] = None, + entity_max_speed=None, + entity_map=None, + compress: bool = True, +) -> Dict[str, int]: + """Write one Mobius-format HDF5 file. + + All per-sample arguments are 1-D sequences of equal length, one entry per + agent per timestep the agent is present. + + timestep_idx int, >= 0. The sample's time is timestep_idx * sampling_interval_s. + entity_id int, dense 0..N-1 (remap your own IDs first). + x, y, z metres, right-handed, z up (z = height of the floor stood on). + rotation_deg optional heading in degrees, counter-clockwise from +x, + i.e. degrees(atan2(dy, dx)). + speed_mps optional speed in m/s (>= 0). + If your model has neither, compute both with + heading_and_speed_from_positions(). Leaving them out also + works (Mobius derives them; slowly for big runs in releases + from before October 2026). + mode optional per-sample text (e.g. "walk"); informational only. + entity_* optional per-entity values, indexed by entity_id. + """ + if not sampling_interval_s > 0: + raise ValueError("sampling_interval_s must be > 0 (seconds between samples)") + + t_idx = np.asarray(timestep_idx, dtype=np.int64) + ids = np.asarray(entity_id, dtype=np.int64) + pos = [np.asarray(v, dtype=np.float64) for v in (x, y, z)] + n = t_idx.size + if any(a.size != n for a in [ids, *pos]): + raise ValueError("timestep_idx, entity_id, x, y and z must all have the same length") + if n == 0: + raise ValueError("no samples to write") + if t_idx.min() < 0: + raise ValueError("timestep_idx must be >= 0") + if ids.min() < 0: + raise ValueError("entity_id must be >= 0") + if t_idx.max() > np.iinfo(np.int32).max or ids.max() > np.iinfo(np.int32).max: + raise ValueError("timestep_idx and entity_id must fit in int32") + if not all(np.isfinite(a).all() for a in pos): + raise ValueError("positions contain NaN or Inf; write no row for absent agents instead") + + num_entities = int(ids.max()) + 1 + present = np.zeros(num_entities, dtype=bool) + present[ids] = True + if not present.all(): + missing = np.flatnonzero(~present) + raise ValueError( + f"entity_id must be dense 0..N-1; {missing.size} id(s) never appear, first: {missing[:5].tolist()}" + ) + + key = t_idx * num_entities + ids + if np.unique(key).size != n: + raise ValueError("an entity has more than one sample in the same timestep") + + order = np.lexsort((ids, t_idx)) # sort by timestep, then entity + + fields = [ + ("timestep_idx", "= 0 and the same length as the samples") + fields.append(("speed", " 4096: + # gzip (+ shuffle) is the only compression Mobius can read. Never use lzf/szip/blosc. + sim.create_dataset("samples", data=samples, chunks=(4096,), compression="gzip", + compression_opts=4, shuffle=True) + else: + sim.create_dataset("samples", data=samples) + sim.create_dataset("timesteps", + data=(np.arange(num_timesteps) * sampling_interval_s).astype(" Dict[str, float]: + """'0=0,1=3.5,2=7' -> {'0': 0.0, '1': 3.5, '2': 7.0}""" + mapping: Dict[str, float] = {} + if spec: + for item in spec.split(","): + label, _, value = item.partition("=") + if not _: + raise ValueError(f"--floor-z entry '{item}' is not of the form label=metres") + mapping[label.strip()] = float(value) + return mapping + + +def convert_csv(args) -> int: + with open(args.input, newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f, delimiter=args.delimiter) + header = reader.fieldnames or [] + for required in (args.col_id, args.col_time, args.col_x, args.col_y): + if required not in header: + print(f"Error: column '{required}' not found. Columns are: {header}", file=sys.stderr) + return 1 + has_z = args.col_z in header + has_floor = args.col_floor in header + floor_z = _parse_floor_z(args.floor_z) + if has_floor and not floor_z and args.floor_height is None: + print(f"Error: the CSV has a '{args.col_floor}' column but no floor heights were given. " + f"Pass --floor-z (e.g. \"0=0,1=3.5\") or --floor-height, or --col-floor none " + f"to ignore the column.", file=sys.stderr) + return 1 + + per_agent: Dict[str, List[tuple]] = {} + start_floor: Dict[str, tuple] = {} # id -> (earliest time, floor label) + for line_no, row in enumerate(reader, start=2): + try: + t = float(row[args.col_time]) + px = float(row[args.col_x]) * args.scale + py = float(row[args.col_y]) * args.scale + pz = float(row[args.col_z]) * args.scale if has_z else 0.0 + except (TypeError, ValueError): + print(f"Error: line {line_no}: non-numeric time/x/y/z: {row}", file=sys.stderr) + return 1 + if has_floor: + label = row[args.col_floor].strip() + if label in floor_z: + pz += floor_z[label] + elif args.floor_height is not None: + try: + pz += float(label) * args.floor_height + except ValueError: + print(f"Error: line {line_no}: floor '{label}' is not a number, so --floor-height " + f"cannot place it; list it in --floor-z", file=sys.stderr) + return 1 + else: + print(f"Error: line {line_no}: floor '{label}' has no entry in --floor-z", file=sys.stderr) + return 1 + agent = row[args.col_id].strip() + start_floor[agent] = min(start_floor.get(agent, (t, label)), (t, label)) + per_agent.setdefault(row[args.col_id].strip(), []).append((t, px, py, pz)) + + if not per_agent: + print("Error: CSV has no data rows", file=sys.stderr) + return 1 + + all_times = np.array(sorted({r[0] for rows in per_agent.values() for r in rows})) + t0 = float(all_times[0]) + dt = args.dt + if dt is None: + steps = np.diff(all_times) + steps = steps[steps > 1e-9] + if steps.size == 0: + print("Error: only one distinct time in the CSV; pass --dt", file=sys.stderr) + return 1 + dt = float(np.median(steps)) + print(f"No --dt given; using the median output interval {dt:g} s") + + # Dense ids in a stable order: numeric ids numerically, otherwise as text. + def sort_key(raw: str): + try: + return (0, float(raw), raw) + except ValueError: + return (1, 0.0, raw) + original_ids = sorted(per_agent, key=sort_key) + + out_t, out_id, out_x, out_y, out_z = [], [], [], [], [] + for dense_id, raw_id in enumerate(original_ids): + rows = np.array(sorted(per_agent[raw_id])) + times, xs, ys, zs = rows[:, 0], rows[:, 1], rows[:, 2], rows[:, 3] + if np.unique(times).size != times.size: + print(f"Error: agent '{raw_id}' has two rows with the same time", file=sys.stderr) + return 1 + k_first = math.ceil((times[0] - t0) / dt - 1e-6) + k_last = math.floor((times[-1] - t0) / dt + 1e-6) + if k_last < k_first: + continue # agent present for less than one interval + grid = t0 + np.arange(k_first, k_last + 1) * dt + out_t.append(np.arange(k_first, k_last + 1)) + out_id.append(np.full(grid.size, dense_id)) + if args.interp == "hold": + # Event logs: a row marks where the agent IS from that time on, so hold it + # there until the next row. Keeps queueing and waiting visible. + src = np.clip(np.searchsorted(times, grid + 1e-9, side="right") - 1, 0, times.size - 1) + out_x.append(xs[src]) + out_y.append(ys[src]) + out_z.append(zs[src]) + else: + # Periodic snapshots: linear interpolation, exact when already on the grid. + out_x.append(np.interp(grid, times, xs)) + out_y.append(np.interp(grid, times, ys)) + out_z.append(np.interp(grid, times, zs)) + + kept = [original_ids[int(ids[0])] for ids in out_id] + if len(kept) != len(original_ids): + # Re-number so ids stay dense after dropping agents shorter than one interval. + remap = {old: new for new, old in enumerate(int(ids[0]) for ids in out_id)} + out_id = [np.full(ids.size, remap[int(ids[0])]) for ids in out_id] + print(f"Dropped {len(original_ids) - len(kept)} agent(s) present for less than one interval") + + t_all, id_all = np.concatenate(out_t), np.concatenate(out_id) + x_all, y_all, z_all = np.concatenate(out_x), np.concatenate(out_y), np.concatenate(out_z) + rotation = speed = None + if not args.positions_only: + rotation, speed = heading_and_speed_from_positions(t_all, id_all, x_all, y_all, z_all, dt) + + stats = write_mobius_h5( + args.output, t_all, id_all, x_all, y_all, z_all, + sampling_interval_s=dt, + rotation_deg=rotation, + speed_mps=speed, + entity_names=[f"{args.name_prefix}{raw}" for raw in kept], + # Informational: the floor each agent starts on, shown in the agent inspector. + entity_planes=[f"Floor {start_floor[raw][1]}" for raw in kept] if start_floor else None, + ) + print(f"Wrote {args.output}: {stats['entities']} agents, {stats['timesteps']} timesteps " + f"({(stats['timesteps'] - 1) * dt:g} s at {dt:g} s), {stats['samples']} samples. " + f"Time 0 in Mobius = {t0:g} s in the CSV.") + return 0 + + +def write_demo(args) -> int: + """Three people crossing a 10 m corridor, sampled every 0.1 s for 10 s.""" + dt = 0.1 + agents = [(0.0, 0.0, 1.0, 1.2), (0.0, 0.0, 2.0, 1.0), (2.0, 10.0, 3.0, -1.3)] # start t, x0, lane y, vx + t_idx, ids, xs, ys, rot, spd = [], [], [], [], [], [] + for step in range(101): + t = step * dt + for e, (t_start, x0, lane, vx) in enumerate(agents): + x = x0 + vx * (t - t_start) + if t < t_start or not 0.0 <= x <= 10.0: + continue # absent: no row + t_idx.append(step); ids.append(e); xs.append(x); ys.append(lane) + rot.append(math.degrees(math.atan2(0.0, vx))) # 0 = +x, 180 = -x + spd.append(abs(vx)) + stats = write_mobius_h5(args.output, t_idx, ids, xs, ys, np.zeros(len(xs)), dt, + rotation_deg=None if args.positions_only else rot, + speed_mps=None if args.positions_only else spd, + mode=None if args.positions_only else ["walk"] * len(xs)) + print(f"Wrote {args.output}: {stats}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + demo = sub.add_parser("demo", help="write a small demo file") + demo.add_argument("output", type=Path) + demo.add_argument("--positions-only", action="store_true", + help="omit rotation/speed/mode so Mobius derives them") + demo.set_defaults(func=write_demo) + + conv = sub.add_parser("from-csv", help="convert a long-format trajectory CSV") + conv.add_argument("input", type=Path) + conv.add_argument("output", type=Path) + conv.add_argument("--dt", type=float, default=None, + help="output sampling interval in seconds (default: median CSV interval)") + conv.add_argument("--interp", choices=["linear", "hold"], default="linear", + help="'linear' for periodic snapshots (default); 'hold' for event logs that " + "write a row only when an agent moves, so waiting stays visible") + conv.add_argument("--positions-only", action="store_true", + help="do not compute heading/speed; Mobius derives them on load " + "(slow for big runs in releases from before October 2026)") + conv.add_argument("--delimiter", default=",") + conv.add_argument("--scale", type=float, default=1.0, + help="multiply x/y/z by this to get metres (e.g. 0.001 for mm)") + conv.add_argument("--floor-z", default=None, + help="floor label to floor height in metres, e.g. '0=0,1=3.5,2=7'") + conv.add_argument("--floor-height", type=float, default=None, + help="uniform storey height in metres for numeric floor labels") + conv.add_argument("--name-prefix", default="Agent ") + for col, default in [("id", "id"), ("time", "time"), ("x", "x"), ("y", "y"), ("z", "z"), ("floor", "floor")]: + conv.add_argument(f"--col-{col}", default=default, help=f"CSV column holding {col} (default '{default}')") + conv.set_defaults(func=convert_csv) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/examples/pedestrian-hdf5/requirements.txt b/docs/examples/pedestrian-hdf5/requirements.txt new file mode 100644 index 000000000..944ac0352 --- /dev/null +++ b/docs/examples/pedestrian-hdf5/requirements.txt @@ -0,0 +1,6 @@ +# Python dependencies for mobius_h5_writer.py and validate_mobius_h5.py +# Install with: pip install -r requirements.txt +# (h5py wheels include their own HDF5 library; no compiler or separate HDF5 install is needed.) + +h5py>=3.0.0 +numpy>=1.20.0 diff --git a/docs/examples/pedestrian-hdf5/validate_mobius_h5.py b/docs/examples/pedestrian-hdf5/validate_mobius_h5.py new file mode 100644 index 000000000..fbaa99289 --- /dev/null +++ b/docs/examples/pedestrian-hdf5/validate_mobius_h5.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +""" +Check that an HDF5 file will load correctly as Project Mobius pedestrian data. + + python validate_mobius_h5.py run.h5 [more.h5 ...] + +Exit code 0 when every file passes (warnings allowed), 1 when any file has an +error. The checks mirror what the Mobius importer actually does +(Plugins/MobiusDataImporter/.../Hdf5SimulationReader.cpp and +Source/ProjectMobius/.../AgentDataSubsystem.cpp), including the things it does +silently: it decodes most fields by POSITION with fixed widths, so a float64 +position or a variable-length string does not fail to load, it loads garbage. + +Layout reference: docs/PEDESTRIAN-DATA-FORMAT.md + +Requires: numpy, h5py (pip install -r requirements.txt) + +MIT License - Copyright (c) 2026 ProjectMobius contributors +""" + +import math +import sys +from pathlib import Path + +import h5py +import numpy as np + +# (position, name, kind) - kind is "i4", "f4" or "str". Order is the contract. +ENTITY_MEMBERS = [ + (0, "id", "i4"), + (1, "name", "str"), + (2, "sim_time_s", "f4"), + (3, "max_speed", "f4"), + (4, "m_plane", "str"), + (5, "map", "i4"), +] +SAMPLE_MEMBERS = [ + (0, "timestep_idx", "i4"), + (1, "entity_id", "i4"), + (2, "position_x", "f4"), + (3, "position_y", "f4"), + (4, "position_z", "f4"), +] +OPTIONAL_SAMPLE_MEMBERS = {"rotation": "f4", "speed": "f4", "mode": "str"} +# HDF5 filter ids the bundled library can decode: 1 deflate (gzip), 2 shuffle, 3 fletcher32, +# 5 n-bit, 6 scale-offset. Not 4 (szip, built without it) and no plugin filters (lzf, blosc, ...). +READABLE_FILTERS = {1, 2, 3, 5, 6} + + +class Report: + def __init__(self, path): + self.path = path + self.errors = [] + self.warnings = [] + self.info = [] + + def error(self, msg): + self.errors.append(msg) + + def warn(self, msg): + self.warnings.append(msg) + + def note(self, msg): + self.info.append(msg) + + def print(self): + status = "FAIL" if self.errors else "PASS" + print(f"\n[{status}] {self.path}") + for msg in self.errors: + print(f" ERROR {msg}") + for msg in self.warnings: + print(f" WARNING {msg}") + for msg in self.info: + print(f" info {msg}") + + +def describe_member(member_type): + """Return 'i4', 'f4', 'str', 'vlen-str' or a description of anything else.""" + cls = member_type.get_class() + size = member_type.get_size() + if cls == h5py.h5t.STRING: + return "vlen-str" if member_type.is_variable_str() else "str" + order = "LE" if member_type.get_order() == h5py.h5t.ORDER_LE else "BE" + if cls == h5py.h5t.INTEGER: + signed = member_type.get_sign() == h5py.h5t.SGN_2 + if size == 4 and signed and order == "LE": + return "i4" + return f"{'' if signed else 'unsigned '}{8 * size}-bit {order} integer" + if cls == h5py.h5t.FLOAT: + if size == 4 and order == "LE": + return "f4" + return f"{8 * size}-bit {order} float" + return f"HDF5 class {cls} ({size} bytes)" + + +KIND_TEXT = {"i4": "32-bit little-endian signed int", "f4": "32-bit little-endian float", + "str": "fixed-length string"} + + +def check_compound(rep, dset, label, required, optional=None): + """Check member order/types. Returns {name: index} of the members found.""" + ftype = dset.id.get_type() + if ftype.get_class() != h5py.h5t.COMPOUND: + rep.error(f"{label} must be a compound (table) dataset, found {dset.dtype}") + return {} + nmembers = ftype.get_nmembers() + names = [ftype.get_member_name(i).decode() for i in range(nmembers)] + found = {n: i for i, n in enumerate(names)} + + for pos, name, kind in required: + if pos >= nmembers: + rep.error(f"{label}: needs at least {len(required)} members, has {nmembers} ({names})") + return found + actual = describe_member(ftype.get_member_type(pos)) + if names[pos] != name: + rep.warn(f"{label}: member {pos} is named '{names[pos]}', expected '{name}' " + f"(Mobius reads this member by POSITION, so it will be used as {name})") + if actual != kind: + if actual == "vlen-str": + rep.error(f"{label}.{names[pos]} is a variable-length string; Mobius needs a fixed-length " + f"string here (in h5py use numpy dtype 'S', not str)") + else: + rep.error(f"{label}.{names[pos]} (member {pos}) is {actual}; Mobius reads it as " + f"{KIND_TEXT[kind]} WITHOUT conversion, so the values would be garbage") + + for name, kind in (optional or {}).items(): + if name in found: + actual = describe_member(ftype.get_member_type(found[name])) + if actual != kind: + what = "a variable-length string" if actual == "vlen-str" else actual + rep.error(f"{label}.{name} is {what}; Mobius reads it as {KIND_TEXT[kind]} without conversion") + return found + + +def check_strings(rep, dset, label, member): + """Fixed strings must leave room for a NUL: Mobius reads them as C strings.""" + if dset.shape[0] == 0 or member not in (dset.dtype.names or ()): + return + size = dset.dtype[member].itemsize + values = dset[member] + lengths = np.char.str_len(values) if values.size else np.array([0]) + if lengths.max() >= size: + rep.error(f"{label}.{member}: some values fill all {size} bytes with no terminating NUL; " + f"make the field at least one byte longer than the longest value") + + +def check_heading(rep, t_idx, e_id, xyz, rotation_deg): + """Compare stored headings with the direction each agent actually moves next.""" + order = np.lexsort((t_idx, e_id)) + t, e = t_idx[order], e_id[order] + dx = np.diff(xyz[0][order]) + dy = np.diff(xyz[1][order]) + step = (np.diff(e) == 0) & (np.diff(t) == 1) & (np.hypot(dx, dy) > 0.05) + if step.sum() < 10: + return + travel = np.degrees(np.arctan2(dy[step], dx[step])) + diff = (rotation_deg[order][:-1][step] - travel + 180.0) % 360.0 - 180.0 + agree = np.mean(np.abs(diff) < 30.0) + if agree < 0.5: + rep.warn(f"rotation matches the direction of travel for only {100 * agree:.0f}% of moving samples. " + f"It must be degrees counter-clockwise from +x, i.e. degrees(atan2(dy, dx)), not a compass " + f"bearing. If your model has no heading, compute it from positions or leave it out") + else: + rep.note(f"rotation matches the direction of travel for {100 * agree:.0f}% of moving samples") + + +def read_attr(rep, group, name, default): + if name not in group.attrs: + rep.warn(f"/metadata has no '{name}' attribute; Mobius will use {default}") + return None + value = group.attrs[name] + arr = np.asarray(value) + if arr.shape != (): + rep.error(f"/metadata '{name}' must be a scalar, found shape {arr.shape}") + return None + if arr.dtype.kind not in "biuf": + rep.error(f"/metadata '{name}' must be numeric, found {arr.dtype}") + return None + return arr.item() + + +def validate(path): + rep = Report(path) + p = Path(path) + if p.suffix.lower() != ".h5": + rep.error(f"extension is '{p.suffix}'; Mobius only opens '.h5' files as pedestrian data") + if not h5py.is_hdf5(path): + rep.error("not an HDF5 file") + return rep + + with h5py.File(path, "r") as f: + if "trajectory" in f: + rep.error("file has a root /trajectory dataset, so Mobius reads it as a Juelich trajectory " + "file and ignores /metadata, /entities and /simulation. Remove /trajectory.") + return rep + if "metadata" not in f or not isinstance(f["metadata"], h5py.Group): + rep.error("missing /metadata group") + if "simulation/samples" not in f: + rep.error("missing /simulation/samples dataset") + if "entities" not in f: + rep.error("missing /entities dataset (the import fails without it)") + if rep.errors: + return rep + + # ---------------- metadata ---------------- + meta = f["metadata"] + duration = read_attr(rep, meta, "duration", "0 (playback length from the samples)") + dt = read_attr(rep, meta, "sampling_rate", "0.1 s") + max_entities = read_attr(rep, meta, "max_num_entities", "the /entities row count") + is_si = read_attr(rep, meta, "is_si", "true (metres)") + is_deg = read_attr(rep, meta, "is_deg", "true (degrees)") + dt = 0.1 if dt is None else float(dt) + if dt <= 0: + rep.error(f"sampling_rate is {dt}; it must be the seconds between samples (> 0)") + dt = 0.1 + elif dt > 5: + rep.warn(f"sampling_rate is {dt}. It is the INTERVAL in seconds, not a frequency: " + f"10 Hz output is 0.1") + if is_si is not None and not bool(is_si): + rep.warn("is_si is false: Mobius then multiplies positions by 10 instead of 100. " + "Write positions in metres and set is_si = 1") + + # ---------------- entities ---------------- + ents = f["entities"] + check_compound(rep, ents, "/entities", ENTITY_MEMBERS) + n_rows = ents.shape[0] + if not rep.errors: + check_strings(rep, ents, "/entities", "name") + check_strings(rep, ents, "/entities", "m_plane") + ids = ents["id"] + if n_rows and not np.array_equal(ids, np.arange(n_rows)): + rep.warn("/entities.id is not 0,1,2,... in row order. Mobius shows row i as the " + "details of entity_id i, whatever its id field says") + + # ---------------- samples ---------------- + samples = f["simulation/samples"] + found = check_compound(rep, samples, "/simulation/samples", SAMPLE_MEMBERS, OPTIONAL_SAMPLE_MEMBERS) + for name in ("rotation", "speed"): + if name not in found: + rep.note(f"no '{name}' member: Mobius derives {name} from the movement on load (slow for " + f"thousands of agents in releases from before October 2026; " + f"mobius_h5_writer.heading_and_speed_from_positions computes it up front)") + filters = samples.id.get_create_plist() + for i in range(filters.get_nfilters()): + code = filters.get_filter(i)[0] + if code not in READABLE_FILTERS: + rep.error(f"/simulation/samples uses HDF5 filter id {code}; the bundled HDF5 can only " + f"decode gzip/shuffle. Rewrite without lzf/szip/blosc/etc.") + if rep.errors: + return rep + + data = samples[...] + n = data.shape[0] + if n == 0: + rep.error("/simulation/samples is empty") + return rep + t_idx = data[data.dtype.names[0]].astype(np.int64) + e_id = data[data.dtype.names[1]].astype(np.int64) + xyz = [data[data.dtype.names[k]].astype(np.float64) for k in (2, 3, 4)] + + declared = int(max_entities) if max_entities else n_rows + if t_idx.min() < 0: + rep.error(f"{int((t_idx < 0).sum())} samples have a negative timestep_idx") + if e_id.min() < 0: + rep.error(f"{int((e_id < 0).sum())} samples have a negative entity_id (they are skipped)") + over = e_id >= declared + if over.any(): + rep.error(f"{int(over.sum())} samples have entity_id >= {declared} (max_num_entities or the " + f"entity count). IDs must be dense 0..N-1; these samples are dropped") + seen = np.unique(e_id[(e_id >= 0) & ~over]) + if seen.size < declared: + rep.warn(f"only {seen.size} of {declared} declared entities appear in the samples " + f"(Mobius reports 'Incomplete entity data')") + if max_entities and n_rows and int(max_entities) != n_rows: + rep.warn(f"max_num_entities = {int(max_entities)} but /entities has {n_rows} rows. It must be " + f"the TOTAL number of agents in the run, not the peak at one time") + if n_rows < declared: + rep.warn(f"/entities has {n_rows} rows for {declared} entities; agents beyond row " + f"{n_rows - 1} have no details in the inspector") + + valid = (t_idx >= 0) & (e_id >= 0) + key = t_idx[valid] * (int(e_id.max()) + 1) + e_id[valid] + dupes = key.size - np.unique(key).size + if dupes: + rep.error(f"{dupes} samples repeat an (entity_id, timestep_idx) pair; write at most one " + f"row per agent per timestep") + + bad_pos = ~(np.isfinite(xyz[0]) & np.isfinite(xyz[1]) & np.isfinite(xyz[2])) + if bad_pos.any(): + rep.error(f"{int(bad_pos.sum())} samples have NaN/Inf positions (each raises an error dialog). " + f"Write no row for an absent agent instead") + if "rotation" in found: + rot = data["rotation"].astype(np.float64) + if not np.isfinite(rot).all(): + rep.error(f"{int((~np.isfinite(rot)).sum())} samples have NaN/Inf rotation") + else: + check_heading(rep, t_idx, e_id, xyz, rot if is_deg is None or bool(is_deg) else np.degrees(rot)) + if "speed" in found: + spd = data["speed"].astype(np.float64) + bad = ~np.isfinite(spd) | (spd < 0) + if bad.any(): + rep.error(f"{int(bad.sum())} samples have a negative or NaN speed (each raises an error " + f"dialog and is replaced by 0)") + elif np.all(spd == 0) and n > 1: + rep.warn("every speed is 0. If your model has no speed, leave the speed member out so " + "Mobius derives it from movement") + if "mode" in found: + check_strings(rep, samples, "/simulation/samples", "mode") + + n_steps = int(t_idx.max()) + 1 + if duration: + expected = math.ceil(np.float32(duration) / np.float32(dt)) + if n_steps < expected: + rep.warn(f"duration {duration} s / sampling_rate {dt} s implies {expected} timesteps but " + f"the samples only reach {n_steps} (Mobius warns 'Data may be truncated'). " + f"Set duration = (number of timesteps - 1) * sampling_rate = {(n_steps - 1) * dt:g}") + empty_steps = n_steps - np.unique(t_idx).size + if empty_steps: + rep.note(f"{empty_steps} timesteps have no agents at all (allowed)") + + if "timesteps" in f["simulation"]: + stored = f["simulation/timesteps"][...] + if stored.size and not np.allclose(stored, np.arange(stored.size) * dt, atol=max(1e-3, dt * 1e-3)): + rep.warn("/simulation/timesteps is not 0, dt, 2dt, ... Mobius ignores it and places " + "sample k at k * sampling_rate, so resample non-uniform output first") + + span = (n_steps - 1) * dt + rep.note(f"{declared} entities, {n_steps} timesteps ({span:g} s at {dt:g} s), {n} samples") + rep.note("x [{:.2f}, {:.2f}] m, y [{:.2f}, {:.2f}] m, z [{:.2f}, {:.2f}] m".format( + np.nanmin(xyz[0]), np.nanmax(xyz[0]), np.nanmin(xyz[1]), np.nanmax(xyz[1]), + np.nanmin(xyz[2]), np.nanmax(xyz[2]))) + if is_deg is not None and not bool(is_deg): + rep.note("is_deg is false: rotation is read as radians") + if "trajectory" not in f and "wkt_geometry" in f.attrs: + rep.warn("root wkt_geometry attribute found: Mobius will offer to load it as geometry, but it " + "loads WKT without the y mirror it applies to these agents, so the outline shows flipped " + "across the x axis. Negate every y value in the WKT to line it up") + return rep + + +def main(argv): + if len(argv) < 2: + print(__doc__) + return 2 + failed = False + for path in argv[1:]: + rep = validate(path) + rep.print() + failed |= bool(rep.errors) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/docs/examples/pedestrian-hdf5/write_mobius_h5.c b/docs/examples/pedestrian-hdf5/write_mobius_h5.c new file mode 100644 index 000000000..63989f7c1 --- /dev/null +++ b/docs/examples/pedestrian-hdf5/write_mobius_h5.c @@ -0,0 +1,332 @@ +/* + * write_mobius_h5.c + * + * Minimal, self-contained example of writing pedestrian trajectories in the + * Project Mobius HDF5 layout with the HDF5 C library that ships (as source) in + * this repository. Copy the two structs and the write_* functions into your own + * simulator and feed them your agents instead of the demo corridor below. + * + * Layout reference: docs/PEDESTRIAN-DATA-FORMAT.md + * Build guide: docs/HDF5-EXPORT-GUIDE.md (or CMakeLists.txt next to this file) + * + * Usage: + * write_mobius_h5 [output.h5] [--positions-only] + * + * --positions-only leave out the rotation/speed/mode members so Mobius derives + * heading and speed from the movement itself. Releases from + * before October 2026 do that slowly for thousands of agents; + * to support them, compute heading (atan2 towards the next + * position) and speed yourself. + * + * MIT License - Copyright (c) 2026 ProjectMobius contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "hdf5.h" + +/* Fixed-length string sizes, INCLUDING the terminating NUL. Mobius reads these + * strings as C strings, so every value must be shorter than its field. */ +#define NAME_LEN 32 +#define PLANE_LEN 16 +#define MODE_LEN 8 + +/* One row of /entities. Mobius reads these six members BY POSITION with these + * exact widths, so do not reorder them or change their types. */ +typedef struct +{ + int32_t id; /* must equal the row index: 0, 1, 2, ... */ + char name[NAME_LEN]; + float sim_time_s; + float max_speed; + char m_plane[PLANE_LEN]; + int32_t map; +} entity_t; + +/* One row of /simulation/samples: one agent at one timestep. The first five + * members are read BY POSITION; rotation, speed and mode are optional and are + * looked up by name. */ +typedef struct +{ + int32_t timestep_idx; /* time = timestep_idx * sampling_rate */ + int32_t entity_id; /* 0 .. num_entities-1 */ + float position_x; /* metres */ + float position_y; /* metres */ + float position_z; /* metres, height of the floor the agent stands on */ + float rotation; /* degrees, counter-clockwise from +x */ + float speed; /* metres per second, >= 0 */ + char mode[MODE_LEN]; /* free text, e.g. "walk" (informational only) */ +} sample_t; + +#define CHECK(expr) \ + do \ + { \ + if ((expr) < 0) \ + { \ + fprintf(stderr, "HDF5 call failed at %s:%d: %s\n", __FILE__, \ + __LINE__, #expr); \ + exit(1); \ + } \ + } while (0) + +static hid_t make_fixed_string(size_t size) +{ + hid_t type = H5Tcopy(H5T_C_S1); + CHECK(type); + CHECK(H5Tset_size(type, size)); + CHECK(H5Tset_strpad(type, H5T_STR_NULLTERM)); + return type; +} + +static void write_scalar_attr(hid_t loc, const char *name, hid_t type, const void *value) +{ + hid_t space = H5Screate(H5S_SCALAR); + hid_t attr = H5Acreate2(loc, name, type, space, H5P_DEFAULT, H5P_DEFAULT); + CHECK(attr); + CHECK(H5Awrite(attr, type, value)); + H5Aclose(attr); + H5Sclose(space); +} + +/* Write a 1-D dataset. mem_type describes the C struct; the file type is a + * packed copy of it, so members you leave out of mem_type take no space. */ +static void write_table(hid_t file, const char *path, hid_t mem_type, + const void *rows, hsize_t count) +{ + hid_t file_type = H5Tcopy(mem_type); + CHECK(file_type); + if (H5Tget_class(file_type) == H5T_COMPOUND) + { + CHECK(H5Tpack(file_type)); + } + + hid_t space = H5Screate_simple(1, &count, NULL); + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + + /* gzip is optional. Use it only when this HDF5 build has it; Mobius reads + * gzip (deflate) and shuffle, and NO other compression filter. */ + if (count >= 4096 && H5Zfilter_avail(H5Z_FILTER_DEFLATE) > 0) + { + hsize_t chunk = 4096; + CHECK(H5Pset_chunk(dcpl, 1, &chunk)); + CHECK(H5Pset_shuffle(dcpl)); + CHECK(H5Pset_deflate(dcpl, 4)); + } + + hid_t dset = H5Dcreate2(file, path, file_type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset); + CHECK(H5Dwrite(dset, mem_type, H5S_ALL, H5S_ALL, H5P_DEFAULT, rows)); + + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(space); + H5Tclose(file_type); +} + +static void write_metadata(hid_t file, double duration_s, double sampling_interval_s, + int64_t num_entities) +{ + const uint8_t yes = 1; + hid_t meta = H5Gcreate2(file, "/metadata", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(meta); + + write_scalar_attr(meta, "duration", H5T_NATIVE_DOUBLE, &duration_s); + /* NOTE: despite its name this is the time BETWEEN samples, in seconds (0.1 = 10 Hz). */ + write_scalar_attr(meta, "sampling_rate", H5T_NATIVE_DOUBLE, &sampling_interval_s); + write_scalar_attr(meta, "max_num_entities", H5T_NATIVE_INT64, &num_entities); + write_scalar_attr(meta, "is_si", H5T_NATIVE_UINT8, &yes); /* positions in metres */ + write_scalar_attr(meta, "is_deg", H5T_NATIVE_UINT8, &yes); /* rotation in degrees */ + + H5Gclose(meta); +} + +static void write_entities(hid_t file, const entity_t *entities, hsize_t count) +{ + hid_t name_type = make_fixed_string(NAME_LEN); + hid_t plane_type = make_fixed_string(PLANE_LEN); + + hid_t type = H5Tcreate(H5T_COMPOUND, sizeof(entity_t)); + CHECK(type); + CHECK(H5Tinsert(type, "id", offsetof(entity_t, id), H5T_NATIVE_INT32)); + CHECK(H5Tinsert(type, "name", offsetof(entity_t, name), name_type)); + CHECK(H5Tinsert(type, "sim_time_s", offsetof(entity_t, sim_time_s), H5T_NATIVE_FLOAT)); + CHECK(H5Tinsert(type, "max_speed", offsetof(entity_t, max_speed), H5T_NATIVE_FLOAT)); + CHECK(H5Tinsert(type, "m_plane", offsetof(entity_t, m_plane), plane_type)); + CHECK(H5Tinsert(type, "map", offsetof(entity_t, map), H5T_NATIVE_INT32)); + + write_table(file, "/entities", type, entities, count); + + H5Tclose(type); + H5Tclose(plane_type); + H5Tclose(name_type); +} + +static void write_samples(hid_t file, const sample_t *samples, hsize_t count, + int with_rotation_speed_mode) +{ + hid_t mode_type = make_fixed_string(MODE_LEN); + + hid_t type = H5Tcreate(H5T_COMPOUND, sizeof(sample_t)); + CHECK(type); + CHECK(H5Tinsert(type, "timestep_idx", offsetof(sample_t, timestep_idx), H5T_NATIVE_INT32)); + CHECK(H5Tinsert(type, "entity_id", offsetof(sample_t, entity_id), H5T_NATIVE_INT32)); + CHECK(H5Tinsert(type, "position_x", offsetof(sample_t, position_x), H5T_NATIVE_FLOAT)); + CHECK(H5Tinsert(type, "position_y", offsetof(sample_t, position_y), H5T_NATIVE_FLOAT)); + CHECK(H5Tinsert(type, "position_z", offsetof(sample_t, position_z), H5T_NATIVE_FLOAT)); + if (with_rotation_speed_mode) + { + CHECK(H5Tinsert(type, "rotation", offsetof(sample_t, rotation), H5T_NATIVE_FLOAT)); + CHECK(H5Tinsert(type, "speed", offsetof(sample_t, speed), H5T_NATIVE_FLOAT)); + CHECK(H5Tinsert(type, "mode", offsetof(sample_t, mode), mode_type)); + } + + write_table(file, "/simulation/samples", type, samples, count); + + H5Tclose(type); + H5Tclose(mode_type); +} + +/* ------------------------------------------------------------------------- */ +/* Demo scenario: three people in a 10 m corridor, sampled every 0.1 s. */ +/* Replace this part with your own simulator's output. */ +/* ------------------------------------------------------------------------- */ + +#define NUM_ENTITIES 3 + +typedef struct +{ + float start_time; /* s */ + float start_x; /* m */ + float lane_y; /* m */ + float velocity_x; /* m/s, sign gives the direction */ +} demo_agent_t; + +static const demo_agent_t demo_agents[NUM_ENTITIES] = { + { 0.0f, 0.0f, 1.0f, 1.2f }, /* leaves the corridor at x = 10 m, then has no rows */ + { 0.0f, 0.0f, 2.0f, 1.0f }, + { 2.0f, 10.0f, 3.0f, -1.3f }, /* enters at t = 2 s walking towards -x */ +}; + +int main(int argc, char **argv) +{ + const char *out_path = "mobius_example.h5"; + int with_rotation_speed_mode = 1; + + for (int i = 1; i < argc; ++i) + { + if (strcmp(argv[i], "--positions-only") == 0) + { + with_rotation_speed_mode = 0; + } + else + { + out_path = argv[i]; + } + } + + const double dt = 0.1; /* sampling interval, s */ + const int32_t num_timesteps = 101; /* t = 0.0 .. 10.0 s */ + + entity_t entities[NUM_ENTITIES]; + int32_t first_step[NUM_ENTITIES]; + int32_t last_step[NUM_ENTITIES]; + memset(entities, 0, sizeof(entities)); + + for (int32_t e = 0; e < NUM_ENTITIES; ++e) + { + entities[e].id = e; + snprintf(entities[e].name, NAME_LEN, "Agent %d", e); + snprintf(entities[e].m_plane, PLANE_LEN, "Floor 0"); + entities[e].max_speed = fabsf(demo_agents[e].velocity_x); + entities[e].map = 0; + first_step[e] = -1; + last_step[e] = -1; + } + + size_t capacity = (size_t)num_timesteps * NUM_ENTITIES; + sample_t *samples = (sample_t *)calloc(capacity, sizeof(sample_t)); + int32_t *samples_per_timestep = (int32_t *)calloc((size_t)num_timesteps, sizeof(int32_t)); + float *timesteps = (float *)calloc((size_t)num_timesteps, sizeof(float)); + if (!samples || !samples_per_timestep || !timesteps) + { + fprintf(stderr, "out of memory\n"); + return 1; + } + + /* Rows are written timestep by timestep. An agent that is not in the + * building at a timestep simply has no row there. */ + size_t count = 0; + for (int32_t step = 0; step < num_timesteps; ++step) + { + const float t = (float)(step * dt); + timesteps[step] = t; + + for (int32_t e = 0; e < NUM_ENTITIES; ++e) + { + const demo_agent_t *a = &demo_agents[e]; + if (t < a->start_time) + { + continue; + } + const float x = a->start_x + a->velocity_x * (t - a->start_time); + if (x < 0.0f || x > 10.0f) + { + continue; + } + + sample_t *s = &samples[count++]; + s->timestep_idx = step; + s->entity_id = e; + s->position_x = x; + s->position_y = a->lane_y; + s->position_z = 0.0f; + /* Heading: atan2(dy, dx) in degrees, so +x is 0 and -x is 180. */ + s->rotation = a->velocity_x >= 0.0f ? 0.0f : 180.0f; + s->speed = fabsf(a->velocity_x); + snprintf(s->mode, MODE_LEN, "walk"); + + ++samples_per_timestep[step]; + if (first_step[e] < 0) + { + first_step[e] = step; + } + last_step[e] = step; + } + } + + for (int32_t e = 0; e < NUM_ENTITIES; ++e) + { + entities[e].sim_time_s = first_step[e] < 0 ? 0.0f : (float)((last_step[e] - first_step[e]) * dt); + } + + hid_t file = H5Fcreate(out_path, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + CHECK(file); + + /* duration = time of the LAST timestep = (num_timesteps - 1) * dt */ + write_metadata(file, (num_timesteps - 1) * dt, dt, NUM_ENTITIES); + write_entities(file, entities, NUM_ENTITIES); + + hid_t sim = H5Gcreate2(file, "/simulation", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(sim); + H5Gclose(sim); + + write_samples(file, samples, count, with_rotation_speed_mode); + write_table(file, "/simulation/timesteps", H5T_NATIVE_FLOAT, timesteps, (hsize_t)num_timesteps); + write_table(file, "/simulation/samples_per_timestep", H5T_NATIVE_INT32, + samples_per_timestep, (hsize_t)num_timesteps); + + CHECK(H5Fclose(file)); + + printf("Wrote %s: %d entities, %d timesteps, %zu samples%s\n", out_path, NUM_ENTITIES, + num_timesteps, count, with_rotation_speed_mode ? "" : " (positions only)"); + + free(timesteps); + free(samples_per_timestep); + free(samples); + return 0; +} From efb41060c04fccf0d323ec3be19ceed8b5f7fe72 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 21 Oct 2026 23:22:16 +0000 Subject: [PATCH 4/6] Update tech debt report [skip ci] --- docs/TECH-DEBT.md | 378 +++++++++++++++++++++++----------------------- 1 file changed, 189 insertions(+), 189 deletions(-) diff --git a/docs/TECH-DEBT.md b/docs/TECH-DEBT.md index e945ac44b..401d80d44 100644 --- a/docs/TECH-DEBT.md +++ b/docs/TECH-DEBT.md @@ -2,7 +2,7 @@ > Auto-generated by [todo-tracker.yml](../.github/workflows/todo-tracker.yml) — updated weekly and on source changes. -**Generated:** 2026-10-07 21:49 UTC +**Generated:** 2026-10-21 23:22 UTC ## Summary @@ -43,39 +43,61 @@ - Line 54: `// TODO (owner ruling 2026-08-03, deferred for deadline): this drives the SUBSYSTEM directly -- it sets` **ProjectMobiusTests/Private/Tests/TrajectoryHeatmapInGameTest.cpp** - Line 51: `// TODO (owner ruling 2026-08-03, carried over from the pre-rewrite file, still true): every scenario here` -**MobiusWidgets/Private/Slate/Components/SFieldAndTitleText.cpp** -- Line 260: `// TODO: add in padding if needed` -**MobiusWidgets/Private/Slate/Components/SAgentFollowIndicator.cpp** -- Line 68: `//TODO: need to do this once, likely store a variable and check for nullptr` -- Line 138: `// TODO: we can improve this to be the only SMeshWidget, we just need to figure out how to handle the different textures for each instance` -**MobiusWidgets/Private/Slate/Components/SWindowTitleBarWidget.cpp** -- Line 123: `// TODO: Investigate further why calling the parent implementation here seems to cause issues.` -**MobiusWidgets/Private/Slate/Components/SPedestrianAgentHoverMeshWidget.cpp** -- Line 158: `// TODO: we should add small white placeholder text like ---` -**MobiusWidgets/Private/UI/MobiusSettingPanel.cpp** -- Line 40: `// TODO:` -**MobiusWidgets/Private/UI/Components/FloorStatsWidget.cpp** -- Line 629: `// TODO: currently we have a limitation for updates that are sent to the overlay,` +**Visualization/Private/DynamicPixelRenderingTexture.cpp** +- Line 72: `// Level of Service Colors for each band - in linear color format TODO: this needs to be a parameter` +- Line 106: `//TODO: add extra parameters for mip settings, compression settings, and srgb settings to give more control` +- Line 169: `if (!bIsBlurRequired && *(PixelPtrToUpdate + 2) > COLOR_TO_BYTE(BlurTriggerThreshold))// TODO: this value needs to either be a parameter or passed in - this is the lower band` +- Line 683: `* TODO: This needs to be fixed in the future, but for now we will use the Mat class. Potential fixes include OpenCV stream handling` +- Line 726: `std::vector points;// TODO: need to test this with a vector of points` +- Line 891: `// TODO: future work may want to add a bool to save texture as grayscale` +- Line 1100: `//TODO: ADD VARIABLES TO STORE ALREADY CALCULATED CVD COLOURS -- MAYBE BUT BLUR Could mess this up` +- Line 1134: `//TODO: ADD VARIABLES TO STORE ALREADY CALCULATED CVD COLOURS -- MAYBE BUT BLUR Could mess this up` +**Visualization/Private/QuadTreeSubsystem.cpp** +- Line 102: `// check tree not null -TODO: Add bool if we using a quad or oct tree(eventually all be oct)` +**Visualization/Private/HeatmapVisualizer.cpp** +- Line 232: `//TODO FIX THIS SIZE ISSUE` +- Line 332: `//TODO: covert to parallel for loop` +**Visualization/Public/QuadTreeSubsystem.h** +- Line 71: `void RemoveEntityFromTree(FVector EntityLocation);//TODO: this needs to be a hash` +- Line 86: `//TODO: work out the columns and rows for the quadtree for spline graph to use` +**Visualization/Public/HeatmapVisualizer.h** +- Line 63: `// TODO: This method is likely going to need to be async so that we can update the render target in real time and not block the main thread` +**Visualization/Public/QuadTreeDataMap.h** +- Line 90: `//TODO: If a hash can be made to work out the tree location this would be a lot faster` +- Line 105: `* TODO: Add ID hashing to remove entities - this way we don't have to recursively search for the entity and can remove it in O(TreeDepth) time` **MobiusWidgets/Private/UI/Components/Scalability/AdjustAvatarModel.cpp** - Line 43: `//TODO: we have method duplicates that we need to rename and call where appropriate` -**MobiusWidgets/Private/UI/Components/BaseLoadingWidget.cpp** -- Line 156: `// TODO: Future will customize this image to do more effects like change colour` -**MobiusWidgets/Private/UI/Components/FlowSectionCounter.cpp** -- Line 63: `// TODO: backgrounds adjust padding/colors here` -- Line 107: `// TODO: determine if center/align inside our root grid cell here if needed` +**MobiusWidgets/Private/UI/Components/FloorStatsWidget.cpp** +- Line 629: `// TODO: currently we have a limitation for updates that are sent to the overlay,` +**MobiusWidgets/Private/UI/Components/PedestrianDataDisplay.cpp** +- Line 142: `// TODO: for now widget field 6 will be collapsed and not used as the gait speed is not yet implemented` +- Line 227: `WidgetHeadGridPanel->SetVisibility(ESlateVisibility::Visible);// should change to visible but self not hit testable -> TODO: update BP logic to handle this` +- Line 358: `// Check if the the required width and height are not the same as the current coefficients -> TODO: add a tolerance check here so we don't resize if the values are very close` **MobiusWidgets/Private/UI/Components/BaseChangePedestrianMaterial.cpp** - Line 133: `// TODO: this check need to be better and handled differently` - Line 270: `// Does the MaterialInst match the starting material of the pedestrian? // TODO` - Line 436: `// TODO: we may want to display a value to a user` - Line 487: `//PropertySliderText->SetText(FText::FromString("Current " + CurrentMaterialProperty + ": " + FString::SanitizeFloat(CurrentVal))); //TODO this needs to be done in slider changed too` - Line 517: `// TODO: until i figure the best way to handle resizing and scaling of the widget, we will hide the slider and text this way the widget will not be resized` -**MobiusWidgets/Private/UI/Components/PedestrianDataDisplay.cpp** -- Line 142: `// TODO: for now widget field 6 will be collapsed and not used as the gait speed is not yet implemented` -- Line 227: `WidgetHeadGridPanel->SetVisibility(ESlateVisibility::Visible);// should change to visible but self not hit testable -> TODO: update BP logic to handle this` -- Line 358: `// Check if the the required width and height are not the same as the current coefficients -> TODO: add a tolerance check here so we don't resize if the values are very close` +**MobiusWidgets/Private/UI/Components/BaseLoadingWidget.cpp** +- Line 156: `// TODO: Future will customize this image to do more effects like change colour` +**MobiusWidgets/Private/UI/Components/FlowSectionCounter.cpp** +- Line 63: `// TODO: backgrounds adjust padding/colors here` +- Line 107: `// TODO: determine if center/align inside our root grid cell here if needed` +**MobiusWidgets/Private/UI/MobiusSettingPanel.cpp** +- Line 40: `// TODO:` **MobiusWidgets/Private/UI/InWorld/FlowCounterWidget.cpp** - Line 154: `// TODO: may need to assess if the ptr array count matches the uniform grid children count` - Line 195: `// TODO: implement scroll logic` +**MobiusWidgets/Private/Slate/Components/SAgentFollowIndicator.cpp** +- Line 68: `//TODO: need to do this once, likely store a variable and check for nullptr` +- Line 138: `// TODO: we can improve this to be the only SMeshWidget, we just need to figure out how to handle the different textures for each instance` +**MobiusWidgets/Private/Slate/Components/SPedestrianAgentHoverMeshWidget.cpp** +- Line 158: `// TODO: we should add small white placeholder text like ---` +**MobiusWidgets/Private/Slate/Components/SFieldAndTitleText.cpp** +- Line 260: `// TODO: add in padding if needed` +**MobiusWidgets/Private/Slate/Components/SWindowTitleBarWidget.cpp** +- Line 123: `// TODO: Investigate further why calling the parent implementation here seems to cause issues.` **MobiusWidgets/Private/Util/WidgetUtilHelpers.cpp** - Line 196: `//TODO: FIX THIS METHOD` - Line 208: `// TODO: this calculation is wrong` @@ -90,158 +112,14 @@ - Line 67: `// TODO: add methods for updating specific section counter values` **MobiusWidgets/Public/UI/InWorld/AgentInfoDisplay.h** - Line 52: `// TODO: should remove storing data from the widget and instead use a subsystem to manage this data` -**Visualization/Private/HeatmapVisualizer.cpp** -- Line 232: `//TODO FIX THIS SIZE ISSUE` -- Line 332: `//TODO: covert to parallel for loop` -**Visualization/Private/DynamicPixelRenderingTexture.cpp** -- Line 72: `// Level of Service Colors for each band - in linear color format TODO: this needs to be a parameter` -- Line 106: `//TODO: add extra parameters for mip settings, compression settings, and srgb settings to give more control` -- Line 169: `if (!bIsBlurRequired && *(PixelPtrToUpdate + 2) > COLOR_TO_BYTE(BlurTriggerThreshold))// TODO: this value needs to either be a parameter or passed in - this is the lower band` -- Line 683: `* TODO: This needs to be fixed in the future, but for now we will use the Mat class. Potential fixes include OpenCV stream handling` -- Line 726: `std::vector points;// TODO: need to test this with a vector of points` -- Line 891: `// TODO: future work may want to add a bool to save texture as grayscale` -- Line 1100: `//TODO: ADD VARIABLES TO STORE ALREADY CALCULATED CVD COLOURS -- MAYBE BUT BLUR Could mess this up` -- Line 1134: `//TODO: ADD VARIABLES TO STORE ALREADY CALCULATED CVD COLOURS -- MAYBE BUT BLUR Could mess this up` -**Visualization/Private/QuadTreeSubsystem.cpp** -- Line 102: `// check tree not null -TODO: Add bool if we using a quad or oct tree(eventually all be oct)` -**Visualization/Public/QuadTreeDataMap.h** -- Line 90: `//TODO: If a hash can be made to work out the tree location this would be a lot faster` -- Line 105: `* TODO: Add ID hashing to remove entities - this way we don't have to recursively search for the entity and can remove it in O(TreeDepth) time` -**Visualization/Public/QuadTreeSubsystem.h** -- Line 71: `void RemoveEntityFromTree(FVector EntityLocation);//TODO: this needs to be a hash` -- Line 86: `//TODO: work out the columns and rows for the quadtree for spline graph to use` -**Visualization/Public/HeatmapVisualizer.h** -- Line 63: `// TODO: This method is likely going to need to be async so that we can update the render target in real time and not block the main thread` -**ProjectMobius/Private/BRisk/BRiskDataSubsystem.cpp** -- Line 596: `// TODO(b-risk geometry toggle / mobius builder): the OFF path below reloads the imported building` -**ProjectMobius/Private/Widgets/Simulation/TimeDilationWidget.cpp** -- Line 59: `//TODO: change to FTEXT so we can format the text better and have it localized` -**ProjectMobius/Private/Widgets/Simulation/SimulationPlayBar.cpp** -- Line 352: `//TODO: THIS NEEDS TO BE FIXED - currently only works when sim is paused first then dragged and unpaused not live adjust(somewhere is updating to previous value after new value)` -- Line 396: `// Round current step to the nearest step size - TODO: this ` -- Line 519: `// TODO: look into UWidgetUtilHelpers for this functionality and see if its better or useable` -**ProjectMobius/Private/Controller/MobiusController.cpp** -- Line 515: `// TODO: Current implementation is a placeholder and will load the file each time and then pick a random transform` -**ProjectMobius/Private/MassAI/MassProcessor/Collisions/PedestrianCollisionProcessor.cpp** -- Line 47: `//TODO: add tag for when we do toggle logic if we want to toggle the collision processor on and off` -- Line 74: `//TODO: We have two methods that rely on the agent subsystem we should assign it to a variable and use it` -- Line 101: `//TODO: below the query needs to be optimized so its scalable - current implementation is not scalable and is a hack together prototype` -- Line 120: `//TODO: We have two methods that rely on the agent subsystem we should assign it to a variable and use it` -- Line 252: `if (EntityCollision.Capsule.Get() == HoveredCapsuleComponent)//TODO: DOUBLE CLICK ` -**ProjectMobius/Private/MassAI/MassProcessor/PedestrianMovementProcessor.cpp** -- Line 171: `// TODO: This processor needs breaking up into smaller processors as it is doing too much at once` -- Line 196: `//TODO: Move data to subsystem and get it from there so not constantly getting large data sets` -- Line 413: `//TODO: ****** when destroying respawning back this should only be called when needed not every execute ******` -- Line 494: `// TODO: This one should be at the start to only check once per call not per loop iteration per call` -**ProjectMobius/Private/MassAI/MassProcessor/Representation/AgentHeatmapProcessor.cpp** -- Line 151: `// TODO: Getting mallocs and realloc collisions with parallel -> maybe need to look at null check or something` -- Line 448: `// TODO: this loop is not parallelized, implement ParallelFor for performance and a threadsafe container for HeatmapLocations for simplicity` -**ProjectMobius/Private/MassAI/MassProcessor/Representation/NiagaraAgentRepProcessor.cpp** -- Line 122: `//TODO: Make flag for this so no constant gets etc` -- Line 127: `// Get the agent representation actor //TODO: this works but it could be better` -- Line 150: `//TODO: need to look at mass ai signals and how to use them -> this should be the equivalent to delegates and events -> and the reloading the shared fragments should only occur then` -- Line 158: `//TODO: this check does not work at all, it never returns true despite data changes!!` -- Line 347: `// Half of the former TODO here is now done: a failed agent's mesh is hidden (bHiddenByTenabilityFailure` -- Line 548: `//TODO: FIX THIS - when going low spec to high the animations remain paused and only resume when we pause and unpause` -**ProjectMobius/Private/MassAI/MassProcessor/Representation/DisplayAgentUIStatsProcessor.cpp** -- Line 207: `//TODO: Moving away from the agent mesh viewer, we will need to update this to use the new system ->once we have collisions` -**ProjectMobius/Private/MassAI/MassProcessor/Analytics/FlowCounterProcessor.cpp** -- Line 63: `//TODO: check if paused -> don't want to keep executing if not needed` -**ProjectMobius/Private/MassAI/SubSystems/MassEntitySpawnSubsystem.cpp** -- Line 120: `//TODO: add a bool to check if the archetype has been created and if so, don't create it again` -- Line 179: `//TODO: We dont want to simulate time till this is done, also we need a better way to build shared fragment and update the archetype on data changes` -**ProjectMobius/Private/MassAI/SubSystems/MassRepresentation/MRS_RepresentationSubsystem.cpp** -- Line 260: `// TODO: no separate gender logic yet` -- Line 578: `// TODO: refactor this method` -- Line 599: `//TODO: at the moment this method will be fine` -**ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp** -- Line 115: `//TODO: optimize this so that the file path is not hardcoded and we only perform this operation once` -- Line 173: `//TODO: this is a temp fix but we need to find a better way to update loading text from this subsystems` -- Line 296: `else // no valid age demographic found -> TODO: for now just set it to adult but need to think on how we want to handle this` -- Line 868: `* @note TODO: Some agents appear to be moving but have 0 speed calculated. Investigate where` -- Line 976: `Subsys->ProgressQueue.Enqueue(0.0f);// TODO: NEED to broadcast new load text here` -- Line 1168: `// TODO: this has no way to know progress of loading file at the moment so we need to think on how to do this better` -**ProjectMobius/Private/MassAI/MassObserverProcessor/PedestrianInitializeMOP.cpp** -- Line 202: `//TODO: This is a poor assumption and should really be searching and checking` -- Line 347: `//TODO: MOVE INTO OWN OBSERVOR` -**ProjectMobius/Private/MassAI/MassObserverProcessor/Representation/AgentRepresentation_MOP.cpp** -- Line 155: `//TODO: need to do index size and offset check here` -- Line 168: `// TODO: this check will always fail on data swap as the frags are reset so sizes will be 0 like the offset` -- Line 252: `// TODO(§4): confirm this handle name once the chair emitter is actually added to both Niagara` -**ProjectMobius/Private/MassAI/SignalProcessors/EnableCollisionSignalProcessor.cpp** -- Line 34: `//TODO: will want to break this into two different signal processors, one for enabling collisions and one for disabling collisions -> this way the disable logic can be more complex and handle the disa` -**ProjectMobius/Public/Widgets/Simulation/SimulationPlayBar.h** -- Line 108: `* Decrement the current playback by 0.1 seconds - TODO: see whether its better to do by a timestep or by a fixed value` -**ProjectMobius/Public/Widgets/Simulation/TimeDilationWidget.h** -- Line 76: `//TODO: Change UTextblocks to RichTextBlocks for better formatting and styling` -**ProjectMobius/Public/MassAI/MassProcessor/PedestrianMovementProcessor.h** -- Line 109: `* TODO: This will be eventually replaced to be a lookup method but for now simple check` -**ProjectMobius/Public/MassAI/MassProcessor/Representation/AgentHeatmapProcessor.h** -- Line 118: `/** Subsystem for heatmaps -- TODO: move this to its own processor */` -- Line 215: `UE::TConsumeAllMpmcQueue LocationQueue; //TODO: Need to handle garbage collection of this queue` -**ProjectMobius/Public/MassAI/Fragments/SharedFragments/SimulationFragment.h** -- Line 121: `// TODO: Add buffer method and not store all data in this struct` -- Line 122: `// TODO: This Map logic needs improving as it is not efficient with large data sets and looping over all data is poor` -**ProjectMobius/Public/MassAI/Fragments/SharedFragments/RepresentationFragments/AgentRepresentationFragment.h** -- Line 90: `int32 NumberOfChildren = 0;// TODO: Maybe do different genders` -- Line 99: `//TODO: currently use bool to switch between low spec static effect and med VAT effect -> when we use enum we will use it here too` -- Line 142: `int32 NumberOfChildren = 0;// TODO: Maybe do different genders` -- Line 154: `//TODO: currently use bool to switch between low spec static effect and med VAT effect -> when we use enum we will use it here too` -**ProjectMobius/Public/MassAI/Fragments/SharedFragments/RepresentationFragments/AgentNiagaraDataFrag.h** -- Line 53: `TArray MaleAdultAnimationStates; // TODO: likely when we bring dif age brackets we want all of them to be in one array` -- Line 77: `TArray FemaleAdultAnimationStates; // TODO: likely when we bring dif age brackets we want all of them to be in one array` -**ProjectMobius/Public/MassAI/Fragments/EntityInfoFragment.h** -- Line 218: `uint8 showPedestrianStats = 0;// 0 = don't show, 1 = show, TODO: 2 = show and highlight` -- Line 288: `struct PROJECTMOBIUS_API FEntityCollisionFragment: public FMassFragment //TODO: this may want to be a FObjectWrapperFragment ?? TBD` -**ProjectMobius/Public/MassAI/SubSystems/MassRepresentation/MRS_RepresentationSubsystem.h** -- Line 63: `static const FVatMovementFrames AvatarGaitSpeedBands[] = { // TODO: need more, narrower bands: fastShuffle, VerySlowWalk etc.` -- Line 169: `* @param[float] AgentMaxSpeed The agents max speed TODO: this is a quick fix to create variations in agent movement speeds` -- Line 213: `//TODO: while still using bool on avatar choice in frag use this method to convert current setting to a bool` -**ProjectMobius/Public/MassAI/SubSystems/MassEntitySpawnSubsystem.h** -- Line 134: `void BuildPedestrianRepresentationFragmentData(); // TODO: add inputs to allow for customisation of the pedestrian representation currently hardcoded values` -**ProjectMobius/Public/MassAI/SubSystems/AgentDataSubsystem.h** -- Line 347: `* @warning TODO: Some agents appear to be moving but have 0 speed calculated.` -**HIT_ThesisWork/Private/ExperimentScorer.cpp** -- Line 209: `Result = FFileHelper::SaveStringToFile(FString("Test Data"), *CSVFilePath);//TODO: Replace "Test Data" with the actual data and look at my other code for how to format the data and add more complexity` -- Line 345: `bColourDeficiencyOverrided = false; // TODO: change to true once images for thesis done` -**HIT_ThesisWork/Public/ExperimentScorer.h** -- Line 285: `const int32 SimulationLimit = 5;// TODO: change this if recruitment increases` **MobiusCore/Private/GameInstances/ProjectMobiusGameInstance.cpp** - Line 38: `// TODO: Set default values for these variables -- talk to peter if particular values are needed that` -**MobiusCore/Private/Actors/FlowCounter.cpp** -- Line 50: `// TODO: Move these helpers to a utility class - likely will be useful elsewhere` -- Line 277: `// TODO: Need to clean up the line offset so we are consistent about ground vs line height as this is confusing` -- Line 357: `// TODO: setup alternative for VR` -- Line 533: `//TODO: likely need to communicate pillars location changing wait for changes to mass ai so we don't get data corruption/mismatches` -- Line 583: `// TODO: depending on mass ai logic we may want to reset the agents passed through counter as well` -- Line 680: `//TODO: Our z limits shouldn't be based on the trigger box extents as if the pillars are at different z locations, the trigger box will be rotated` -- Line 785: `// TODO: this is a temporary fix to account for the Z search limits being in world space and the line intersection check being in local space` -- Line 851: `else if (FlowCounter->PreviousTrackedAgentLocations.Contains(Data.AgentID)) // TODO: check if we actually want to check if intersected or disregard ` -- Line 870: `// TODO: this is a temporary fix to account for the Z search limits being in world space and the line intersection check being in local space -> also minz may not be correctly set` -- Line 1029: `// TODO: until we handle new sim time for large amounts of actors then we should only ` -- Line 1154: `// TODO: Review this logic with Pete to ensure this is how we should be checking` -- Line 1221: `//TODO: If we store in multiple buckets in the case when start and end locations are the same we may want to remove from all buckets` -**MobiusCore/Private/Actors/HeatmapPixelTextureVisualizer.cpp** -- Line 262: `GenerateMeshVerticesUVsAndTriangles(MeshSize, FIntPoint(TextureWidth, TextureHeight), bIs3DHeatmap); // TODO work out a resolution for the mesh to be generated` -- Line 1035: `//TODO: for now hardcode the radius as 11 - this is to be changed to a variable and be a float over integer for more precise locations` -- Line 1080: `//TODO:OPTIMIZE` -- Line 1105: `DynamicTexture->ClearTexture(); // TODO: create a buffer that represents clear so no need to keep doing loops` -- Line 1135: `// TODO: this convert not needed as was part of user study test -> it should be converted into a material logic not cpu logic` -- Line 1262: `//TODO:OPTIMIZE` -- Line 1281: `DynamicTexture->ClearTexture(); // TODO: create a buffer that represents clear so no need to keep doing loops` -- Line 1308: `// TODO: this convert not needed as was part of user study test -> it should be converted into a material logic not cpu logic` -- Line 1894: `NumTriangles.X = FMath::CeilToInt32(MeshSize.X / 25); // TODO: working in meters so 100 is 1 meter so divide by 25 to get 4 triangles per meter` -- Line 1896: `//TODO: this is a fix triangle size based on being to a 10 tenth of the mesh size - this is as resolution goes up so does the number of triangles required` -- Line 2502: `float StepSize = 650.0f; // TODO: This is a hardcoded value for now for demo 500 ensures the building is covered - its too large for the mesh and has overflow around the edges ` **MobiusCore/Private/BuildingGenerator/RuntimeMeshBuilder.cpp** - Line 784: `FDatasmithRuntimeImportOptions ImportOptions;// TODO: set the import options` - Line 935: `// TODO:check valid data` - Line 940: `/** TODO: if we get here and the runnable is not valid or the async loader is not valid we need to handle this` -**MobiusCore/Private/Subsystems/StatisticSubsystem.cpp** -- Line 128: `// TODO: switch to delta add/remove instead of full-array rebroadcast.` -- Line 133: `//TODO: we are going to need to expand this functionality when we increase the number of flow counters and their Z bounds` -- Line 185: `//TODO: this should be private method that we call after filtered in to correspond flowcounter groups - but prototype only using one so direct call to this is fine` -- Line 276: `// TODO: Instantaneous flow rate is the important one need to add` +**MobiusCore/Private/Interfaces/AssimpInterface.cpp** +- Line 37: `// May need to use rotate instead of unrotate -- TODO: See the difference between the two` **MobiusCore/Private/Subsystems/PerformanceUtilSubsystem.cpp** - Line 149: `//TODO: investigate the best way to clear the delegate` - Line 223: `//TODO: This method is simplistic and only calculates FPS based on DeltaTime.` @@ -258,49 +136,171 @@ - Line 53: `UpdateMaxTimeSteps(FMath::CeilToInt(TotalTime / TimeBetweenSteps)); //TODO: TimeBetweenSteps should be read from file or a setting for the user` - Line 203: `UpdateMaxTimeSteps(FMath::CeilToInt32(TotalTime / TimeBetweenSteps)); //TODO: TimeBetweenSteps should be read from file or a setting for the user` - Line 318: `// TODO make this better` +**MobiusCore/Private/Subsystems/StatisticSubsystem.cpp** +- Line 128: `// TODO: switch to delta add/remove instead of full-array rebroadcast.` +- Line 133: `//TODO: we are going to need to expand this functionality when we increase the number of flow counters and their Z bounds` +- Line 185: `//TODO: this should be private method that we call after filtered in to correspond flowcounter groups - but prototype only using one so direct call to this is fine` +- Line 276: `// TODO: Instantaneous flow rate is the important one need to add` **MobiusCore/Private/Subsystems/HeatmapSubsystem.cpp** - Line 125: `//TODO: we just want to update the heatmap if the location is the same and values are different` - Line 140: `FString HeatmapName = FString::Printf(TEXT("Heatmap_%i"), HeatmapIndex);// TODO: create a floor classification method based on array size` - Line 510: `// TODO: this sort of works for now but likely will cause problems later on` - Line 779: `// TODO: THis method causing a small performance hit, need to investigate, likely due to the way task is executed and requiring game thread for some operations` -**MobiusCore/Private/Components/FlowCounterSpawnerComponent.cpp** -- Line 261: `// TODO: Manipulate the rotation, size etc to fit door` -**MobiusCore/Private/Interfaces/AssimpInterface.cpp** -- Line 37: `// May need to use rotate instead of unrotate -- TODO: See the difference between the two` +**MobiusCore/Private/Actors/HeatmapPixelTextureVisualizer.cpp** +- Line 262: `GenerateMeshVerticesUVsAndTriangles(MeshSize, FIntPoint(TextureWidth, TextureHeight), bIs3DHeatmap); // TODO work out a resolution for the mesh to be generated` +- Line 1035: `//TODO: for now hardcode the radius as 11 - this is to be changed to a variable and be a float over integer for more precise locations` +- Line 1080: `//TODO:OPTIMIZE` +- Line 1105: `DynamicTexture->ClearTexture(); // TODO: create a buffer that represents clear so no need to keep doing loops` +- Line 1135: `// TODO: this convert not needed as was part of user study test -> it should be converted into a material logic not cpu logic` +- Line 1262: `//TODO:OPTIMIZE` +- Line 1281: `DynamicTexture->ClearTexture(); // TODO: create a buffer that represents clear so no need to keep doing loops` +- Line 1308: `// TODO: this convert not needed as was part of user study test -> it should be converted into a material logic not cpu logic` +- Line 1894: `NumTriangles.X = FMath::CeilToInt32(MeshSize.X / 25); // TODO: working in meters so 100 is 1 meter so divide by 25 to get 4 triangles per meter` +- Line 1896: `//TODO: this is a fix triangle size based on being to a 10 tenth of the mesh size - this is as resolution goes up so does the number of triangles required` +- Line 2502: `float StepSize = 650.0f; // TODO: This is a hardcoded value for now for demo 500 ensures the building is covered - its too large for the mesh and has overflow around the edges ` +**MobiusCore/Private/Actors/FlowCounter.cpp** +- Line 50: `// TODO: Move these helpers to a utility class - likely will be useful elsewhere` +- Line 277: `// TODO: Need to clean up the line offset so we are consistent about ground vs line height as this is confusing` +- Line 357: `// TODO: setup alternative for VR` +- Line 533: `//TODO: likely need to communicate pillars location changing wait for changes to mass ai so we don't get data corruption/mismatches` +- Line 583: `// TODO: depending on mass ai logic we may want to reset the agents passed through counter as well` +- Line 680: `//TODO: Our z limits shouldn't be based on the trigger box extents as if the pillars are at different z locations, the trigger box will be rotated` +- Line 785: `// TODO: this is a temporary fix to account for the Z search limits being in world space and the line intersection check being in local space` +- Line 851: `else if (FlowCounter->PreviousTrackedAgentLocations.Contains(Data.AgentID)) // TODO: check if we actually want to check if intersected or disregard ` +- Line 870: `// TODO: this is a temporary fix to account for the Z search limits being in world space and the line intersection check being in local space -> also minz may not be correctly set` +- Line 1029: `// TODO: until we handle new sim time for large amounts of actors then we should only ` +- Line 1154: `// TODO: Review this logic with Pete to ensure this is how we should be checking` +- Line 1221: `//TODO: If we store in multiple buckets in the case when start and end locations are the same we may want to remove from all buckets` **MobiusCore/Private/AsyncAssimpMeshLoader.cpp** - Line 334: `//TODO: Need to workout a way to handle normals for WKT data -> filters just don't work for this` - Line 905: `//TODO: this method needs to be refactored to use matrix transformations as it doesn't work when it comes to normals and this is where the issue is for translucent materials` +**MobiusCore/Private/Components/FlowCounterSpawnerComponent.cpp** +- Line 261: `// TODO: Manipulate the rotation, size etc to fit door` **MobiusCore/Public/GameInstances/ProjectMobiusGameInstance.h** - Line 113: `//TODO: convert methods below so we can broadcast events when the data is changed` - Line 288: `//TODO: we will bring datatables back as it makes it to manage and requires less memory allocation` - Line 371: `//TODO: we will bring datatables back as it makes it to manage and requires less memory allocation` -**MobiusCore/Public/Actors/FlowCounter.h** -- Line 19: `// Structs used internally for bucketing agents -> TODO: Move it to the FlowCounterStructs.h file as we may want to use it elsewhere` -- Line 255: `// TODO: we will need to also bucket this off as a per segment rolling average as we want that data` -- Line 307: `/* TODO: this property is redundant, we should just update methods to use the line that would be at the base of the pillars` -- Line 314: `/* TODO: we will want a reference to a user widget this way we can create a cpp version in the widget module and` -**MobiusCore/Public/Actors/HeatmapPixelTextureVisualizer.h** -- Line 214: `// TODO: This method is likely going to need to be async so that we can update the render target in real time and not block the main thread` -- Line 305: `/**TODO: look over this as it is late when you wrote it` -- Line 569: `* TODO: Currently using integer values, this should be changed to a float value for more precise locations and scale to texture and mesh size` -- Line 572: `int32 CircleRadius = 110; // 110 = 1.1m for our scaled data - TODO: SORT THIS OUT FOR BETTER SCALING` -- Line 698: `// TODO(perf, owner-flagged 2026-08-10): reduce the trajectory field's memory footprint by condensing` -- Line 1032: `int32 ScaledCircleSize; // TODO: This should be a float value for more precise locations and scale to texture and mesh size` -**MobiusCore/Public/EnumsAndStructs/AgentMeshViewer.h** -- Line 28: `//TODO: // Add more properties as needed, such as direction, health, etc.` **MobiusCore/Public/BuildingGenerator/RuntimeMeshBuilder.h** - Line 308: `/** Internal method to set the material on the mesh - TODO: this will need to be converted to be called via a delegate */` - Line 486: `/** TODO: We eventually want to get the mesh material and apply our materials to it as a mask or material function to it` **MobiusCore/Public/Subsystems/PerformanceUtilSubsystem.h** - Line 21: `DECLARE_MULTICAST_DELEGATE(FOnManualScalabilityChanged);//TODO: investigate whether separate param delegates will be better for performance or not` +**MobiusCore/Public/Subsystems/StatisticSubsystem.h** +- Line 57: `/** TODO: this will be used to pull logic from the heatmap subsystem and the floor stats widget so only one subsystem is used` **MobiusCore/Public/Subsystems/HeatmapSubsystem.h** - Line 118: `* TODO: we will in future want to pass more parameters to this function like type of heatmap, size, etc` - Line 249: `//TODO: this method needs to be placed in a more appropriate place as it is not really a heatmap method but we have bound the visualiser chart logic to this system` -**MobiusCore/Public/Subsystems/StatisticSubsystem.h** -- Line 57: `/** TODO: this will be used to pull logic from the heatmap subsystem and the floor stats widget so only one subsystem is used` **MobiusCore/Public/Subsystems/TimeDilationSubSystem.h** - Line 44: `/** Delegate to broadcast the new simulation time when it changes */ //TODO: do we need this delegate? left comment for now` - Line 190: `//TODO: @Peter -- This an assumption to be clarified but I am assuming all timesteps are calculated` **MobiusCore/Public/Subsystems/MobiusControllerSubsystem.h** - Line 87: `/**TODO: TBD if we can or want to use interfaces with mass ai entities, if so then we can use this method to perform a line trace` +**MobiusCore/Public/Actors/HeatmapPixelTextureVisualizer.h** +- Line 214: `// TODO: This method is likely going to need to be async so that we can update the render target in real time and not block the main thread` +- Line 305: `/**TODO: look over this as it is late when you wrote it` +- Line 569: `* TODO: Currently using integer values, this should be changed to a float value for more precise locations and scale to texture and mesh size` +- Line 572: `int32 CircleRadius = 110; // 110 = 1.1m for our scaled data - TODO: SORT THIS OUT FOR BETTER SCALING` +- Line 698: `// TODO(perf, owner-flagged 2026-08-10): reduce the trajectory field's memory footprint by condensing` +- Line 1032: `int32 ScaledCircleSize; // TODO: This should be a float value for more precise locations and scale to texture and mesh size` +**MobiusCore/Public/Actors/FlowCounter.h** +- Line 19: `// Structs used internally for bucketing agents -> TODO: Move it to the FlowCounterStructs.h file as we may want to use it elsewhere` +- Line 255: `// TODO: we will need to also bucket this off as a per segment rolling average as we want that data` +- Line 307: `/* TODO: this property is redundant, we should just update methods to use the line that would be at the base of the pillars` +- Line 314: `/* TODO: we will want a reference to a user widget this way we can create a cpp version in the widget module and` +**MobiusCore/Public/EnumsAndStructs/AgentMeshViewer.h** +- Line 28: `//TODO: // Add more properties as needed, such as direction, health, etc.` +**HIT_ThesisWork/Private/ExperimentScorer.cpp** +- Line 209: `Result = FFileHelper::SaveStringToFile(FString("Test Data"), *CSVFilePath);//TODO: Replace "Test Data" with the actual data and look at my other code for how to format the data and add more complexity` +- Line 345: `bColourDeficiencyOverrided = false; // TODO: change to true once images for thesis done` +**HIT_ThesisWork/Public/ExperimentScorer.h** +- Line 285: `const int32 SimulationLimit = 5;// TODO: change this if recruitment increases` +**ProjectMobius/Private/MassAI/MassObserverProcessor/Representation/AgentRepresentation_MOP.cpp** +- Line 155: `//TODO: need to do index size and offset check here` +- Line 168: `// TODO: this check will always fail on data swap as the frags are reset so sizes will be 0 like the offset` +- Line 252: `// TODO(§4): confirm this handle name once the chair emitter is actually added to both Niagara` +**ProjectMobius/Private/MassAI/MassObserverProcessor/PedestrianInitializeMOP.cpp** +- Line 202: `//TODO: This is a poor assumption and should really be searching and checking` +- Line 347: `//TODO: MOVE INTO OWN OBSERVOR` +**ProjectMobius/Private/MassAI/SignalProcessors/EnableCollisionSignalProcessor.cpp** +- Line 34: `//TODO: will want to break this into two different signal processors, one for enabling collisions and one for disabling collisions -> this way the disable logic can be more complex and handle the disa` +**ProjectMobius/Private/MassAI/SubSystems/AgentDataSubsystem.cpp** +- Line 115: `//TODO: optimize this so that the file path is not hardcoded and we only perform this operation once` +- Line 173: `//TODO: this is a temp fix but we need to find a better way to update loading text from this subsystems` +- Line 296: `else // no valid age demographic found -> TODO: for now just set it to adult but need to think on how we want to handle this` +- Line 900: `* @note TODO: Some agents appear to be moving but have 0 speed calculated. Investigate where` +- Line 957: `Subsys->ProgressQueue.Enqueue(0.0f);// TODO: NEED to broadcast new load text here` +- Line 1149: `// TODO: this has no way to know progress of loading file at the moment so we need to think on how to do this better` +**ProjectMobius/Private/MassAI/SubSystems/MassEntitySpawnSubsystem.cpp** +- Line 120: `//TODO: add a bool to check if the archetype has been created and if so, don't create it again` +- Line 179: `//TODO: We dont want to simulate time till this is done, also we need a better way to build shared fragment and update the archetype on data changes` +**ProjectMobius/Private/MassAI/SubSystems/MassRepresentation/MRS_RepresentationSubsystem.cpp** +- Line 260: `// TODO: no separate gender logic yet` +- Line 578: `// TODO: refactor this method` +- Line 599: `//TODO: at the moment this method will be fine` +**ProjectMobius/Private/MassAI/MassProcessor/PedestrianMovementProcessor.cpp** +- Line 171: `// TODO: This processor needs breaking up into smaller processors as it is doing too much at once` +- Line 196: `//TODO: Move data to subsystem and get it from there so not constantly getting large data sets` +- Line 413: `//TODO: ****** when destroying respawning back this should only be called when needed not every execute ******` +- Line 494: `// TODO: This one should be at the start to only check once per call not per loop iteration per call` +**ProjectMobius/Private/MassAI/MassProcessor/Representation/NiagaraAgentRepProcessor.cpp** +- Line 122: `//TODO: Make flag for this so no constant gets etc` +- Line 127: `// Get the agent representation actor //TODO: this works but it could be better` +- Line 150: `//TODO: need to look at mass ai signals and how to use them -> this should be the equivalent to delegates and events -> and the reloading the shared fragments should only occur then` +- Line 158: `//TODO: this check does not work at all, it never returns true despite data changes!!` +- Line 347: `// Half of the former TODO here is now done: a failed agent's mesh is hidden (bHiddenByTenabilityFailure` +- Line 548: `//TODO: FIX THIS - when going low spec to high the animations remain paused and only resume when we pause and unpause` +**ProjectMobius/Private/MassAI/MassProcessor/Representation/DisplayAgentUIStatsProcessor.cpp** +- Line 207: `//TODO: Moving away from the agent mesh viewer, we will need to update this to use the new system ->once we have collisions` +**ProjectMobius/Private/MassAI/MassProcessor/Representation/AgentHeatmapProcessor.cpp** +- Line 151: `// TODO: Getting mallocs and realloc collisions with parallel -> maybe need to look at null check or something` +- Line 448: `// TODO: this loop is not parallelized, implement ParallelFor for performance and a threadsafe container for HeatmapLocations for simplicity` +**ProjectMobius/Private/MassAI/MassProcessor/Analytics/FlowCounterProcessor.cpp** +- Line 63: `//TODO: check if paused -> don't want to keep executing if not needed` +**ProjectMobius/Private/MassAI/MassProcessor/Collisions/PedestrianCollisionProcessor.cpp** +- Line 47: `//TODO: add tag for when we do toggle logic if we want to toggle the collision processor on and off` +- Line 74: `//TODO: We have two methods that rely on the agent subsystem we should assign it to a variable and use it` +- Line 101: `//TODO: below the query needs to be optimized so its scalable - current implementation is not scalable and is a hack together prototype` +- Line 120: `//TODO: We have two methods that rely on the agent subsystem we should assign it to a variable and use it` +- Line 252: `if (EntityCollision.Capsule.Get() == HoveredCapsuleComponent)//TODO: DOUBLE CLICK ` +**ProjectMobius/Private/Controller/MobiusController.cpp** +- Line 515: `// TODO: Current implementation is a placeholder and will load the file each time and then pick a random transform` +**ProjectMobius/Private/Widgets/Simulation/SimulationPlayBar.cpp** +- Line 352: `//TODO: THIS NEEDS TO BE FIXED - currently only works when sim is paused first then dragged and unpaused not live adjust(somewhere is updating to previous value after new value)` +- Line 396: `// Round current step to the nearest step size - TODO: this ` +- Line 519: `// TODO: look into UWidgetUtilHelpers for this functionality and see if its better or useable` +**ProjectMobius/Private/Widgets/Simulation/TimeDilationWidget.cpp** +- Line 59: `//TODO: change to FTEXT so we can format the text better and have it localized` +**ProjectMobius/Private/BRisk/BRiskDataSubsystem.cpp** +- Line 596: `// TODO(b-risk geometry toggle / mobius builder): the OFF path below reloads the imported building` +**ProjectMobius/Public/MassAI/SubSystems/AgentDataSubsystem.h** +- Line 349: `* @warning TODO: Some agents appear to be moving but have 0 speed calculated.` +**ProjectMobius/Public/MassAI/SubSystems/MassRepresentation/MRS_RepresentationSubsystem.h** +- Line 63: `static const FVatMovementFrames AvatarGaitSpeedBands[] = { // TODO: need more, narrower bands: fastShuffle, VerySlowWalk etc.` +- Line 169: `* @param[float] AgentMaxSpeed The agents max speed TODO: this is a quick fix to create variations in agent movement speeds` +- Line 213: `//TODO: while still using bool on avatar choice in frag use this method to convert current setting to a bool` +**ProjectMobius/Public/MassAI/SubSystems/MassEntitySpawnSubsystem.h** +- Line 134: `void BuildPedestrianRepresentationFragmentData(); // TODO: add inputs to allow for customisation of the pedestrian representation currently hardcoded values` +**ProjectMobius/Public/MassAI/MassProcessor/Representation/AgentHeatmapProcessor.h** +- Line 118: `/** Subsystem for heatmaps -- TODO: move this to its own processor */` +- Line 215: `UE::TConsumeAllMpmcQueue LocationQueue; //TODO: Need to handle garbage collection of this queue` +**ProjectMobius/Public/MassAI/MassProcessor/PedestrianMovementProcessor.h** +- Line 109: `* TODO: This will be eventually replaced to be a lookup method but for now simple check` +**ProjectMobius/Public/MassAI/Fragments/EntityInfoFragment.h** +- Line 218: `uint8 showPedestrianStats = 0;// 0 = don't show, 1 = show, TODO: 2 = show and highlight` +- Line 288: `struct PROJECTMOBIUS_API FEntityCollisionFragment: public FMassFragment //TODO: this may want to be a FObjectWrapperFragment ?? TBD` +**ProjectMobius/Public/MassAI/Fragments/SharedFragments/SimulationFragment.h** +- Line 121: `// TODO: Add buffer method and not store all data in this struct` +- Line 122: `// TODO: This Map logic needs improving as it is not efficient with large data sets and looping over all data is poor` +**ProjectMobius/Public/MassAI/Fragments/SharedFragments/RepresentationFragments/AgentRepresentationFragment.h** +- Line 90: `int32 NumberOfChildren = 0;// TODO: Maybe do different genders` +- Line 99: `//TODO: currently use bool to switch between low spec static effect and med VAT effect -> when we use enum we will use it here too` +- Line 142: `int32 NumberOfChildren = 0;// TODO: Maybe do different genders` +- Line 154: `//TODO: currently use bool to switch between low spec static effect and med VAT effect -> when we use enum we will use it here too` +**ProjectMobius/Public/MassAI/Fragments/SharedFragments/RepresentationFragments/AgentNiagaraDataFrag.h** +- Line 53: `TArray MaleAdultAnimationStates; // TODO: likely when we bring dif age brackets we want all of them to be in one array` +- Line 77: `TArray FemaleAdultAnimationStates; // TODO: likely when we bring dif age brackets we want all of them to be in one array` +**ProjectMobius/Public/Widgets/Simulation/SimulationPlayBar.h** +- Line 108: `* Decrement the current playback by 0.1 seconds - TODO: see whether its better to do by a timestep or by a fixed value` +**ProjectMobius/Public/Widgets/Simulation/TimeDilationWidget.h** +- Line 76: `//TODO: Change UTextblocks to RichTextBlocks for better formatting and styling` From 70b558d0b8e261a1eb574135335f704bd411a1d1 Mon Sep 17 00:00:00 2001 From: Nicholas Harding <40708936+sir306@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:30:53 +1300 Subject: [PATCH 5/6] Give each session its own custom log file MobiusCustomLog.txt took every session's lines forever (115 MB on one dev machine) and interleaved processes running side by side, such as the editor and a headless test run. Each session now writes MobiusCustomLog_.txt, with a _2 suffix if two start in the same second. Its first line is "=== Mobius session started