From 20431ff21100c99449c24215b6768d1fef754fc9 Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:57:42 +0100 Subject: [PATCH 1/6] Added audio data cache --- .gitignore | 3 + AssetEditor.sln | 15 + AssetEditor/App.xaml.cs | 17 + Editors/Audio/AudioCacheTests.cs | 496 +++++++++++++++ .../AudioExplorer/AudioExplorerViewModel.cs | 8 +- .../ExplorerListSelectionFilter.cs | 8 +- .../AudioProjectConverterViewModel.cs | 9 +- Editors/Audio/DependencyInjectionContainer.cs | 2 + Editors/Audio/Directory.Build.props | 14 + Editors/Audio/Editors.Audio.csproj | 5 + .../Compiler/AudioProjectCompilerService.cs | 15 +- .../Audio/Shared/Storage/AudioRepository.cs | 599 ++++++++++++++---- Editors/Audio/Shared/Storage/BnkLoader.cs | 296 ++++++--- .../Storage/CacheDatabase/AudioCache.cs | 468 ++++++++++++++ .../CacheDatabase/AudioCacheDbContext.cs | 94 +++ .../Storage/CacheDatabase/AudioCacheHelper.cs | 166 +++++ Editors/Audio/Shared/Storage/DatLoader.cs | 42 +- .../Generators/Hirc/HircChunkGenerator.cs | 2 +- .../Generators/SoundBankGeneratorService.cs | 7 +- .../HircExploration/HircTreeChildrenParser.cs | 8 +- Editors/Audio/Test.Audio.csproj | 28 + .../Audio/DialogueEventAndEventNamePrinter.cs | 3 +- .../Reports/Audio/DialogueEventInfoPrinter.cs | 3 +- Shared/GameFiles/Wwise/Bkhd/AkBankHeader.cs | 11 +- Shared/GameFiles/Wwise/BnkFile.cs | 114 ++++ Shared/GameFiles/Wwise/BnkParser.cs | 73 --- Shared/GameFiles/Wwise/Data/DataChunk.cs | 15 +- Shared/GameFiles/Wwise/Didx/DidxChunk.cs | 27 +- Shared/GameFiles/Wwise/Hirc/HircChunk.cs | 83 ++- Shared/GameFiles/Wwise/Hirc/HircHeader.cs | 39 ++ Shared/GameFiles/Wwise/Hirc/HircIndexEntry.cs | 10 + Shared/GameFiles/Wwise/Hirc/HircItem.cs | 67 +- Shared/GameFiles/Wwise/ParsedBnkFile.cs | 15 - Shared/GameFiles/Wwise/Stid/StidChunk.cs | 11 +- .../Shared.Core/Misc/DirectoryHelper.cs | 2 +- .../Shared.Core/Misc/MemoryOptimiser.cs | 42 -- 36 files changed, 2359 insertions(+), 458 deletions(-) create mode 100644 Editors/Audio/AudioCacheTests.cs create mode 100644 Editors/Audio/Directory.Build.props create mode 100644 Editors/Audio/Shared/Storage/CacheDatabase/AudioCache.cs create mode 100644 Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheDbContext.cs create mode 100644 Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheHelper.cs create mode 100644 Editors/Audio/Test.Audio.csproj create mode 100644 Shared/GameFiles/Wwise/BnkFile.cs delete mode 100644 Shared/GameFiles/Wwise/BnkParser.cs create mode 100644 Shared/GameFiles/Wwise/Hirc/HircHeader.cs create mode 100644 Shared/GameFiles/Wwise/Hirc/HircIndexEntry.cs delete mode 100644 Shared/GameFiles/Wwise/ParsedBnkFile.cs delete mode 100644 Shared/SharedCore/Shared.Core/Misc/MemoryOptimiser.cs diff --git a/.gitignore b/.gitignore index 8f8dabbfa..c0af8f363 100644 --- a/.gitignore +++ b/.gitignore @@ -342,3 +342,6 @@ healthchecksdb # MonoGame build outputs /GameWorld/ContentProject/Content/bin/ + +# A folder for research stuff +/Research diff --git a/AssetEditor.sln b/AssetEditor.sln index f32c46102..85dbf7245 100644 --- a/AssetEditor.sln +++ b/AssetEditor.sln @@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Editors.TextureEditor", "Ed EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Editors.Audio", "Editors\Audio\Editors.Audio.csproj", "{410AC3F4-021C-4993-BCCA-1810D153075D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Test.Audio", "Editors\Audio\Test.Audio.csproj", "{5597E9B9-8BEF-407B-A72F-F9D05A540EB5}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Editors.AnimationVisualEditors", "Editors\AnimationEditor\Editors.AnimationVisualEditors.csproj", "{F513A090-CC3F-4DB4-8332-50EB3FDA5A37}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Test.E2EVerification", "Testing\E2EVerification\Test.E2EVerification.csproj", "{0B14A236-AAE0-4E9B-A41E-4AF9DD976BA7}" @@ -288,6 +290,18 @@ Global {410AC3F4-021C-4993-BCCA-1810D153075D}.Release|x64.Build.0 = Release|Any CPU {410AC3F4-021C-4993-BCCA-1810D153075D}.Release|x86.ActiveCfg = Release|Any CPU {410AC3F4-021C-4993-BCCA-1810D153075D}.Release|x86.Build.0 = Release|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Debug|x64.ActiveCfg = Debug|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Debug|x64.Build.0 = Debug|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Debug|x86.ActiveCfg = Debug|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Debug|x86.Build.0 = Debug|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Release|Any CPU.Build.0 = Release|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Release|x64.ActiveCfg = Release|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Release|x64.Build.0 = Release|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Release|x86.ActiveCfg = Release|Any CPU + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5}.Release|x86.Build.0 = Release|Any CPU {F513A090-CC3F-4DB4-8332-50EB3FDA5A37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F513A090-CC3F-4DB4-8332-50EB3FDA5A37}.Debug|Any CPU.Build.0 = Debug|Any CPU {F513A090-CC3F-4DB4-8332-50EB3FDA5A37}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -666,6 +680,7 @@ Global {EAFBC3D8-A146-4BA3-8024-DE4E20D3C884} = {07AC615B-A8FC-4E1A-BDD5-BC11452429A0} {8E55B123-C8E5-4D65-AAAA-C4DC4713CB8E} = {07AC615B-A8FC-4E1A-BDD5-BC11452429A0} {410AC3F4-021C-4993-BCCA-1810D153075D} = {4D35FFE2-8490-4694-8981-654016F2BD3D} + {5597E9B9-8BEF-407B-A72F-F9D05A540EB5} = {4D35FFE2-8490-4694-8981-654016F2BD3D} {F513A090-CC3F-4DB4-8332-50EB3FDA5A37} = {07AC615B-A8FC-4E1A-BDD5-BC11452429A0} {0B14A236-AAE0-4E9B-A41E-4AF9DD976BA7} = {18424B6A-CB8A-4CE1-935C-72459F31521B} {E604D1EB-02AC-4E31-9161-531F8830BFDB} = {A1B2C3D4-E5F6-4A5B-9C8D-1E2F3A4B5C6D} diff --git a/AssetEditor/App.xaml.cs b/AssetEditor/App.xaml.cs index e3970c869..ef5521ce3 100644 --- a/AssetEditor/App.xaml.cs +++ b/AssetEditor/App.xaml.cs @@ -8,6 +8,7 @@ using AssetEditor.ViewModels; using AssetEditor.Views; using CommunityToolkit.Diagnostics; +using Editors.Audio.Shared.Storage; using Editors.Ipc; using Microsoft.Extensions.DependencyInjection; using Shared.Core.DevConfig; @@ -128,7 +129,23 @@ private void LoadCAPackFiles(ApplicationSettingsService settingsService) if (loadRes == null) MessageBox.Show($"Unable to load all CA packfiles in {gamePath}"); else + { packfileService.AddContainer(loadRes); + BuildGameFilesAudioCache(); + } + } + } + + private void BuildGameFilesAudioCache() + { + try + { + var audioRepository = _serviceProvider!.GetRequiredService(); + audioRepository.EnsureGameFilesCache(); + } + catch (Exception exception) + { + Logging.Create().Here().Warning($"Failed to build the game-files audio cache: {exception.Message}"); } } diff --git a/Editors/Audio/AudioCacheTests.cs b/Editors/Audio/AudioCacheTests.cs new file mode 100644 index 000000000..27120a3e8 --- /dev/null +++ b/Editors/Audio/AudioCacheTests.cs @@ -0,0 +1,496 @@ +using System.Text; +using Editors.Audio.Shared.Storage; +using Editors.Audio.Shared.Storage.CacheDatabase; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Moq; +using Shared.Core.Events; +using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models.FileSources; +using Shared.Core.PackFiles.Utility; +using Shared.Core.Settings; +using Shared.GameFormats.Wwise.Enums; + +namespace Test.Audio +{ + [TestFixture] + internal class AudioCacheTests + { + private string _tempDir = string.Empty; + private string _dbFilePath = string.Empty; + private List _inMemoryKeepAliveConnections = []; + + [SetUp] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "AudioCacheTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + _dbFilePath = Path.Combine(_tempDir, "test_cache.db"); + _inMemoryKeepAliveConnections = []; + } + + [TearDown] + public void TearDown() + { + foreach (var connection in _inMemoryKeepAliveConnections) + connection.Dispose(); + + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, true); + } + + private DbContextOptions CreateTestDbOptions() + { + var dbName = "AudioCacheTests_" + Guid.NewGuid().ToString("N"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbName, + Mode = SqliteOpenMode.Memory, + Cache = SqliteCacheMode.Shared + }.ToString(); + + var keepAliveConnection = new SqliteConnection(connectionString); + keepAliveConnection.Open(); + _inMemoryKeepAliveConnections.Add(keepAliveConnection); + + return new DbContextOptionsBuilder() + .UseSqlite(connectionString) + .Options; + } + + private DbContextOptions CreateFileDbOptions() + { + return new DbContextOptionsBuilder() + .UseSqlite($"Data Source={_dbFilePath};Pooling=False") + .Options; + } + + private static void SaveCache(string fingerprint, List containers, DbContextOptions dbOptions) + { + var bnkLoader = new BnkLoader(new Mock().Object); + using var cache = new AudioCache(dbOptions); + cache.Save(fingerprint, true, containers, bnkLoader, new DatLoader.Result()); + } + + private static AudioCacheHelper CreateAudioCacheHelper() + { + var settings = new ApplicationSettingsService(GameTypeEnum.Warhammer3); + var packFileService = new Mock().Object; + return new AudioCacheHelper(settings, new DatLoader(packFileService, settings), new BnkLoader(packFileService)); + } + + [Test] + public void RoundTrip_PreservesBnkMetadata() + { + var bnk1 = PackFile.CreateFromBytes("bnk1.bnk", CreateBnk(1, 111, 0)); + var bnk2 = PackFile.CreateFromBytes("bnk2.bnk", CreateBnk(2, 222, 0)); + var container = CreateContainer(true, [(@"audio\wwise\bnk1.bnk", bnk1), (@"audio\wwise\bnk2.bnk", bnk2)]); + + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint123", [container.Object], dbOptions); + using var loaded = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint123"); + + Assert.That(loaded, Is.Not.Null); + var bnks = loaded.GetBnks(); + Assert.That(bnks, Has.Count.EqualTo(2)); + Assert.That(bnks.Single(x => x.Path == @"audio\wwise\bnk1.bnk").LanguageId, Is.EqualTo(111)); + Assert.That(bnks.Single(x => x.Path == @"audio\wwise\bnk2.bnk").LanguageId, Is.EqualTo(222)); + Assert.That(bnks.All(x => x.IsCA), Is.True); + } + + [Test] + public void LoadCache_ReturnsCorrectHircData() + { + const string BnkPath = @"audio\wwise\test.bnk"; + const uint EventId = 123456; + var bnk = PackFile.CreateFromBytes("test.bnk", CreateBnk(EventId, 678, 1024)); + var container = CreateContainer(true, [(BnkPath, bnk)]); + + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint", [container.Object], dbOptions); + using var loaded = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint"); + + var resolvedPaths = new HashSet([BnkPath], StringComparer.OrdinalIgnoreCase); + var hircs = loaded.FindHircs(EventId, resolvedPaths); + + Assert.That(hircs, Has.Count.EqualTo(1)); + var hirc = hircs.Single(); + Assert.That(hirc.Id, Is.EqualTo(EventId)); + Assert.That(hirc.HircType, Is.EqualTo(AkBkHircType.Event)); + Assert.That(hirc.BnkPath, Is.EqualTo(BnkPath)); + Assert.That(hirc.IsCA, Is.True); + Assert.That(hirc.LanguageId, Is.EqualTo(678)); + + var didx = loaded.FindDidx(resolvedPaths); + Assert.That(didx.Single().Length, Is.EqualTo(1024)); + } + + [Test] + public void LoadCache_PreservesDistinctBnkPathsAcrossHircs() + { + var bnk1 = PackFile.CreateFromBytes("bnk1.bnk", CreateBnk(1, 1, 0)); + var bnk2 = PackFile.CreateFromBytes("bnk2.bnk", CreateBnk(2, 1, 0)); + var container = CreateContainer(true, [(@"audio\wwise\bnk1.bnk", bnk1), (@"audio\wwise\bnk2.bnk", bnk2)]); + + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint", [container.Object], dbOptions); + using var loaded = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint"); + + var resolvedPaths = new HashSet([@"audio\wwise\bnk1.bnk", @"audio\wwise\bnk2.bnk"], StringComparer.OrdinalIgnoreCase); + var allHircs = loaded.FindAllHircs(resolvedPaths); + + Assert.That(allHircs.Single(x => x.Id == 1).BnkPath, Is.EqualTo(@"audio\wwise\bnk1.bnk")); + Assert.That(allHircs.Single(x => x.Id == 2).BnkPath, Is.EqualTo(@"audio\wwise\bnk2.bnk")); + } + + [Test] + public void LoadCache_ReturnsNullForMissingFile() + { + var result = AudioCache.CreateFromFingerPrint(Path.Combine(_tempDir, "nonexistent.db"), "fingerprint"); + Assert.That(result, Is.Null); + } + + [Test] + public void LoadCache_ReturnsNullForWrongFingerprint() + { + var container = CreateContainer(true, []); + var dbOptions = CreateTestDbOptions(); + SaveCache("correctFingerprint", [container.Object], dbOptions); + + var result = AudioCache.CreateFromFingerPrint(dbOptions, "wrongFingerprint"); + Assert.That(result, Is.Null); + } + + [Test] + public void ComputeFingerprint_DeterministicForSameInputs() + { + var bnk = PackFile.CreateFromBytes("test.bnk", CreateBnk(1, 1, 0)); + var container = CreateContainer(true, [(@"audio\wwise\test.bnk", bnk)]); + var helper = CreateAudioCacheHelper(); + + var firstFingerprint = helper.ComputeFingerprint([container.Object], "game files"); + var secondFingerprint = helper.ComputeFingerprint([container.Object], "game files"); + + Assert.That(firstFingerprint, Is.EqualTo(secondFingerprint)); + } + + [Test] + public void ComputeFingerprint_ChangesWhenFileChanges() + { + var helper = CreateAudioCacheHelper(); + var bnk = PackFile.CreateFromBytes("test.bnk", CreateBnk(1, 1, 0)); + var container = CreateContainer(true, [(@"audio\wwise\test.bnk", bnk)]); + var fingerprintBeforeChange = helper.ComputeFingerprint([container.Object], "game files"); + + var modifiedBnk = PackFile.CreateFromBytes("test.bnk", CreateBnk(2, 1, 0)); + var modifiedContainer = CreateContainer(true, [(@"audio\wwise\test.bnk", modifiedBnk)]); + var fingerprintAfterChange = helper.ComputeFingerprint([modifiedContainer.Object], "game files"); + + Assert.That(fingerprintBeforeChange, Is.Not.EqualTo(fingerprintAfterChange)); + } + + [Test] + public void ComputeFingerprint_FileEnumerationOrderIndependent() + { + var bnk1 = PackFile.CreateFromBytes("bnk1.bnk", CreateBnk(1, 1, 0)); + var bnk2 = PackFile.CreateFromBytes("bnk2.bnk", CreateBnk(2, 1, 0)); + var containerForward = CreateContainer(true, [(@"audio\wwise\bnk1.bnk", bnk1), (@"audio\wwise\bnk2.bnk", bnk2)]); + var containerReversed = CreateContainer(true, [(@"audio\wwise\bnk2.bnk", bnk2), (@"audio\wwise\bnk1.bnk", bnk1)]); + var helper = CreateAudioCacheHelper(); + + var forwardFingerprint = helper.ComputeFingerprint([containerForward.Object], "game files"); + var reversedFingerprint = helper.ComputeFingerprint([containerReversed.Object], "game files"); + + Assert.That(forwardFingerprint, Is.EqualTo(reversedFingerprint)); + } + + [Test] + public void ComputeFingerprint_MissingBackingFile() + { + var packParent = new PackedFileSourceParent { FilePath = Path.Combine(_tempDir, "missing.pack") }; + var bnk = new PackFile("missing.bnk", new PackedFileSource(packParent, 0, 100, false, false, CompressionFormat.None, 0)); + var container = CreateContainer(true, [(@"audio\wwise\missing.bnk", bnk)]); + + var fingerprint = CreateAudioCacheHelper().ComputeFingerprint([container.Object], "game files"); + + Assert.That(fingerprint, Is.Not.Null.And.Not.Empty); + } + + [Test] + public void GetCacheFilePath_SanitizesInvalidChars() + { + var path = CreateAudioCacheHelper().GetCacheFilePath("Game:Name/WithChars", "abc123"); + var fileName = Path.GetFileName(path); + + Assert.That(fileName.IndexOfAny(Path.GetInvalidFileNameChars()), Is.EqualTo(-1)); + Assert.That(path.EndsWith(".db"), Is.True); + } + + [Test] + public void RoundTrip_FullCycle() + { + const string BnkPath = @"audio\wwise\test.bnk"; + const uint EventId = 999; + var bnk = PackFile.CreateFromBytes("test.bnk", CreateBnk(EventId, 1, 2048)); + var container = CreateContainer(true, [(BnkPath, bnk)]); + + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint", [container.Object], dbOptions); + using var restored = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint"); + + Assert.That(restored, Is.Not.Null); + var resolvedPaths = new HashSet([BnkPath], StringComparer.OrdinalIgnoreCase); + + Assert.Multiple(() => + { + Assert.That(restored.GetBnks(), Has.Count.EqualTo(1)); + Assert.That(restored.FindHircs(EventId, resolvedPaths), Has.Count.EqualTo(1)); + Assert.That(restored.FindAllHircs(resolvedPaths), Has.Count.EqualTo(1)); + Assert.That(restored.FindDidx(resolvedPaths).Single().Length, Is.EqualTo(2048)); + Assert.That(restored.FindHircIds(1, true, resolvedPaths), Does.Contain(EventId)); + }); + } + + [Test] + public void SaveAndLoadCache_ReturnsQueryableCache() + { + const string bnkPath = @"audio\wwise\test.bnk"; + const uint eventId = 555; + var bnk = PackFile.CreateFromBytes("test.bnk", CreateBnk(eventId, 1, 0)); + var container = CreateContainer(true, [(bnkPath, bnk)]); + var packFileService = new Mock(); + packFileService.Setup(x => x.GetAllPackfileContainers()).Returns([container.Object]); + var settings = new ApplicationSettingsService(GameTypeEnum.Warhammer3); + var helper = new AudioCacheHelper(settings, new DatLoader(packFileService.Object, settings), new BnkLoader(packFileService.Object)); + var source = new AudioCacheSource(_dbFilePath, "fingerprint", true, [container.Object], [container.Object]); + + using var cache = helper.SaveAndLoadCache(source); + + Assert.That(File.Exists(_dbFilePath), Is.True); + Assert.That(cache.FindHircs(eventId, new HashSet([bnkPath], StringComparer.OrdinalIgnoreCase)), Has.Count.EqualTo(1)); + } + + [Test] + public void TryLoadFromCache_ReturnsCacheWhenValid() + { + var container = CreateContainer(true, []); + var dbOptions = CreateFileDbOptions(); + SaveCache("fingerprint", [container.Object], dbOptions); + + var result = CreateAudioCacheHelper().TryLoadFromCache(_dbFilePath, "fingerprint"); + + Assert.That(result, Is.Not.Null); + } + + [Test] + public void TryLoadFromCache_ReturnsNullForMissingFile() + { + var result = CreateAudioCacheHelper().TryLoadFromCache(Path.Combine(_tempDir, "does_not_exist.db"), "fingerprint"); + Assert.That(result, Is.Null); + } + + [Test] + public void TryLoadFromCache_ReturnsNullForCorruptFile() + { + File.WriteAllBytes(_dbFilePath, [0xFF, 0xFE, 0x00, 0x01]); + var result = CreateAudioCacheHelper().TryLoadFromCache(_dbFilePath, "fingerprint"); + Assert.That(result, Is.Null); + } + + [Test] + public void SaveCache_PreservesIsCAFlag() + { + var vanillaBnk = PackFile.CreateFromBytes("vanilla.bnk", CreateBnk(1, 1, 0)); + var moddedBnk = PackFile.CreateFromBytes("modded.bnk", CreateBnk(2, 1, 0)); + var vanillaContainer = CreateContainer(true, [(@"audio\wwise\vanilla.bnk", vanillaBnk)]); + var moddedContainer = CreateContainer(false, [(@"audio\wwise\modded.bnk", moddedBnk)]); + + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint", [vanillaContainer.Object, moddedContainer.Object], dbOptions); + using var loaded = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint"); + + var bnks = loaded.GetBnks(); + Assert.That(bnks.Single(x => x.Path.Contains("vanilla")).IsCA, Is.True); + Assert.That(bnks.Single(x => x.Path.Contains("modded")).IsCA, Is.False); + } + + [Test] + public void SaveCache_NoBnks_RoundTrips() + { + var container = CreateContainer(true, []); + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint", [container.Object], dbOptions); + using var loaded = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint"); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded.GetBnks(), Is.Empty); + } + + [Test] + public void SaveCache_OverwritesExistingCache() + { + var dbOptions = CreateFileDbOptions(); + + var bnk1 = PackFile.CreateFromBytes("bnk1.bnk", CreateBnk(1, 1, 0)); + var container1 = CreateContainer(true, [(@"audio\wwise\bnk1.bnk", bnk1)]); + SaveCache("firstFingerprint", [container1.Object], dbOptions); + + var bnk2 = PackFile.CreateFromBytes("bnk2.bnk", CreateBnk(2, 1, 0)); + var container2 = CreateContainer(true, [(@"audio\wwise\bnk2.bnk", bnk2)]); + SaveCache("secondFingerprint", [container2.Object], dbOptions); + + var oldResult = AudioCache.CreateFromFingerPrint(dbOptions, "firstFingerprint"); + Assert.That(oldResult, Is.Null); + + using var newResult = AudioCache.CreateFromFingerPrint(dbOptions, "secondFingerprint"); + Assert.That(newResult, Is.Not.Null); + Assert.That(newResult.GetBnks().Single().Path, Is.EqualTo(@"audio\wwise\bnk2.bnk")); + } + + [Test] + public void SaveCache_IncludesInMemoryBackedBnks() + { + // Freshly generated bnks are MemorySource-backed until saved to disk, so unlike the packfile cache these must still be indexable. + var bnk = new PackFile("generated.bnk", new MemorySource(CreateBnk(1, 1, 0))); + var container = CreateContainer(false, [(@"audio\wwise\generated.bnk", bnk)]); + + var dbOptions = CreateTestDbOptions(); + SaveCache("fingerprint", [container.Object], dbOptions); + using var loaded = AudioCache.CreateFromFingerPrint(dbOptions, "fingerprint"); + + Assert.That(loaded.GetBnks(), Has.Count.EqualTo(1)); + } + + [Test] + public void MergeDatData_AppendsLayerDataAndLetsProjectNamesOverrideGameNames() + { + var gameData = new CachedAudioDatData + { + NameById = new Dictionary { [1] = "game", [2] = "game-only" }, + StateGroupsByDialogueEvent = new Dictionary> { ["event"] = ["game-group"] }, + StatesByStateGroup = new Dictionary> { ["group"] = ["game-state"] } + }; + var projectData = new CachedAudioDatData + { + NameById = new Dictionary { [1] = "project", [3] = "project-only" }, + StateGroupsByDialogueEvent = new Dictionary> { ["event"] = ["project-group"] }, + StatesByStateGroup = new Dictionary> { ["group"] = ["project-state"] } + }; + + var result = AudioRepository.MergeDatData([gameData, projectData]); + + Assert.Multiple(() => + { + Assert.That(result.NameById[1], Is.EqualTo("project")); + Assert.That(result.NameById[2], Is.EqualTo("game-only")); + Assert.That(result.NameById[3], Is.EqualTo("project-only")); + Assert.That(result.StateGroupsByDialogueEvent["event"], Is.EqualTo(new[] { "game-group", "project-group" })); + Assert.That(result.StatesByStateGroup["group"], Is.EqualTo(new[] { "game-state", "project-state" })); + }); + } + + [Test] + public void CreateCacheSources_ProjectCacheUsesOnlyProjectContainers() + { + var gameContainer = CreateContainer(true, []); + gameContainer.SetupGet(x => x.Name).Returns("game"); + var projectBnk = PackFile.CreateFromBytes("project.bnk", CreateBnk(1, 1, 0)); + var projectContainer = CreateContainer(false, [(@"audio\wwise\project.bnk", projectBnk)]); + projectContainer.SetupGet(x => x.Name).Returns("project"); + + var packFileService = new Mock(); + packFileService.Setup(x => x.GetAllPackfileContainers()).Returns([gameContainer.Object, projectContainer.Object]); + packFileService.Setup(x => x.GetEditablePack()).Returns(projectContainer.Object); + + var settings = new ApplicationSettingsService(GameTypeEnum.Warhammer3); + var bnkLoader = new BnkLoader(packFileService.Object); + var cacheHelper = new AudioCacheHelper(settings, new DatLoader(packFileService.Object, settings), bnkLoader); + var eventHub = new Mock(); + var repository = new AudioRepository(settings, packFileService.Object, cacheHelper, bnkLoader, eventHub.Object); + + var sources = repository.CreateCacheSources(); + + Assert.Multiple(() => + { + Assert.That(sources, Has.Count.EqualTo(2)); + Assert.That(sources[1].BnkContainers, Is.EqualTo(new[] { projectContainer.Object })); + Assert.That(sources[1].DatContainers, Is.EqualTo(new[] { projectContainer.Object })); + Assert.That(sources[1].Fingerprint, Is.EqualTo(cacheHelper.ComputeFingerprint([projectContainer.Object], "project files"))); + }); + } + + private static Mock CreateContainer(bool isCa, List<(string Path, PackFile File)> files) + { + var container = new Mock(); + container.SetupGet(x => x.IsCaPackFile).Returns(isCa); + container.SetupGet(x => x.ContainerType).Returns(PackFileContainerType.Normal); + container + .Setup( + x => x.SearchFiles( + It.IsAny(), + It.IsAny?>())) + .Returns( + (string? _, IReadOnlyList? extensions) => + files + .Where( + x => extensions == null + || extensions.Contains( + Path.GetExtension(x.Path), + StringComparer.OrdinalIgnoreCase)) + .ToList()); + return container; + } + + private static byte[] CreateBnk(uint eventId, uint languageId, int embeddedMediaSize, bool includeHirc = true) + { + using var bnk = new MemoryStream(); + + using (var header = new MemoryStream()) + using (var writer = new BinaryWriter(header, Encoding.UTF8, true)) + { + writer.Write((uint)2147483784); + writer.Write((uint)55); + writer.Write(languageId); + writer.Write((uint)0); + writer.Write((uint)999); + WriteChunk(bnk, "BKHD", header.ToArray()); + } + + if (includeHirc) + { + using var hirc = new MemoryStream(); + using var writer = new BinaryWriter(hirc, Encoding.UTF8, true); + writer.Write((uint)1); + writer.Write((byte)AkBkHircType.Event); + writer.Write((uint)5); + writer.Write(eventId); + writer.Write((byte)0); + WriteChunk(bnk, "HIRC", hirc.ToArray()); + } + + if (embeddedMediaSize > 0) + { + using var didx = new MemoryStream(); + using (var writer = new BinaryWriter(didx, Encoding.UTF8, true)) + { + writer.Write((uint)987); + writer.Write((uint)0); + writer.Write((uint)embeddedMediaSize); + } + WriteChunk(bnk, "DIDX", didx.ToArray()); + WriteChunk(bnk, "DATA", new byte[embeddedMediaSize]); + } + + return bnk.ToArray(); + } + + private static void WriteChunk(Stream output, string tag, byte[] payload) + { + output.Write(Encoding.ASCII.GetBytes(tag)); + using var writer = new BinaryWriter(output, Encoding.UTF8, true); + writer.Write((uint)payload.Length); + output.Write(payload); + } + } +} diff --git a/Editors/Audio/AudioExplorer/AudioExplorerViewModel.cs b/Editors/Audio/AudioExplorer/AudioExplorerViewModel.cs index 6a48e4912..002a953e1 100644 --- a/Editors/Audio/AudioExplorer/AudioExplorerViewModel.cs +++ b/Editors/Audio/AudioExplorer/AudioExplorerViewModel.cs @@ -1,8 +1,6 @@ -using System; -using System.Collections.ObjectModel; +using System.Collections.ObjectModel; using System.Collections.Specialized; using System.ComponentModel; -using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using CommunityToolkit.Mvvm.ComponentModel; @@ -165,7 +163,7 @@ private void OnNodeSelected(HircTreeNode selectedNode) _ = LoadWaveformForNodeAsync(selectedNode); } - private async System.Threading.Tasks.Task LoadWaveformForNodeAsync(HircTreeNode node) + private async Task LoadWaveformForNodeAsync(HircTreeNode node) { if (node?.Hirc is ICAkSound sound) { @@ -287,7 +285,7 @@ private void OnEventSelected(ExplorerListItem newValue) SelectedNode = null; TreeList.Clear(); - var dialogueEvents = _audioRepository.GetHircsByHircType(AkBkHircType.Dialogue_Event); + var dialogueEvents = _audioRepository.GetHircs(AkBkHircType.Dialogue_Event); foreach (var dialogueEvent in dialogueEvents) { var dialogueEventRootNode = hircTreeChildrenParser.BuildHierarchy(dialogueEvent); diff --git a/Editors/Audio/AudioExplorer/ExplorerListSelectionFilter.cs b/Editors/Audio/AudioExplorer/ExplorerListSelectionFilter.cs index fe7f2ab4f..12c17c030 100644 --- a/Editors/Audio/AudioExplorer/ExplorerListSelectionFilter.cs +++ b/Editors/Audio/AudioExplorer/ExplorerListSelectionFilter.cs @@ -1,6 +1,4 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; +using System.IO; using Editors.Audio.Shared.Storage; using Shared.GameFormats.Wwise.Enums; using Shared.GameFormats.Wwise.Hirc; @@ -50,9 +48,9 @@ public void Refresh(bool searchByActionEvent, bool searchByDialogueEvent, bool s { var hircs = new List(); if (searchByActionEvent) - hircs = _audioRepository.GetHircsByHircType(AkBkHircType.Event); + hircs = _audioRepository.GetHircs(AkBkHircType.Event); else if (searchByDialogueEvent) - hircs = _audioRepository.GetHircsByHircType(AkBkHircType.Dialogue_Event); + hircs = _audioRepository.GetHircs(AkBkHircType.Dialogue_Event); selectedList = hircs .Select(hirc => new ExplorerListItem diff --git a/Editors/Audio/AudioProjectConverter/AudioProjectConverterViewModel.cs b/Editors/Audio/AudioProjectConverter/AudioProjectConverterViewModel.cs index 07a064f11..6cb754d35 100644 --- a/Editors/Audio/AudioProjectConverter/AudioProjectConverterViewModel.cs +++ b/Editors/Audio/AudioProjectConverter/AudioProjectConverterViewModel.cs @@ -9,7 +9,6 @@ using Editors.Audio.Shared.GameInformation.Warhammer3; using Editors.Audio.Shared.Storage; using Editors.Audio.Shared.Wwise.HircExploration; -using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -143,18 +142,18 @@ private static Dictionary> BuildHircLookupById(List GetHircItems(List soundBankPaths) { - var parsedSoundBanks = new List(); + var parsedSoundBanks = new List(); foreach (var soundBankPath in soundBankPaths) { var soundBankDataBytes = File.ReadAllBytes(soundBankPath); - var soundBankPackFile = PackFile.CreateFromBytes(soundBankPath, soundBankDataBytes); - var parsedSoundBank = BnkParser.Parse(soundBankPackFile, soundBankPath, false); + var parsedSoundBank = BnkFile.CreateFromBytes(soundBankDataBytes, soundBankPath, false); parsedSoundBanks.Add(parsedSoundBank); } var hircItems = parsedSoundBanks - .SelectMany(soundBank => soundBank.HircChunk.HircItems) + .Where(soundBank => soundBank.HircChunk != null) + .SelectMany(soundBank => soundBank.HircChunk!.HircItems) .ToList(); return hircItems; diff --git a/Editors/Audio/DependencyInjectionContainer.cs b/Editors/Audio/DependencyInjectionContainer.cs index f2ca4bf0d..839dc73d2 100644 --- a/Editors/Audio/DependencyInjectionContainer.cs +++ b/Editors/Audio/DependencyInjectionContainer.cs @@ -25,6 +25,7 @@ using Editors.Audio.Shared.AudioProject.Factories; using Editors.Audio.Shared.Dat; using Editors.Audio.Shared.Storage; +using Editors.Audio.Shared.Storage.CacheDatabase; using Editors.Audio.Shared.Utilities; using Editors.Audio.Shared.Wwise; using Editors.Audio.Shared.Wwise.Generators; @@ -132,6 +133,7 @@ public override void Register(IServiceCollection serviceCollection) // Shared audio stuff serviceCollection.AddScoped(); + serviceCollection.AddScoped(); serviceCollection.AddScoped(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/Editors/Audio/Directory.Build.props b/Editors/Audio/Directory.Build.props new file mode 100644 index 000000000..c60ed847c --- /dev/null +++ b/Editors/Audio/Directory.Build.props @@ -0,0 +1,14 @@ + + + + $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) + + + + + + obj\Test.Audio\ + bin\Test.Audio\ + + + diff --git a/Editors/Audio/Editors.Audio.csproj b/Editors/Audio/Editors.Audio.csproj index b3e40bebf..be879005c 100644 --- a/Editors/Audio/Editors.Audio.csproj +++ b/Editors/Audio/Editors.Audio.csproj @@ -6,15 +6,20 @@ true + + + + + diff --git a/Editors/Audio/Shared/AudioProject/Compiler/AudioProjectCompilerService.cs b/Editors/Audio/Shared/AudioProject/Compiler/AudioProjectCompilerService.cs index 63eda6e19..2a4da1061 100644 --- a/Editors/Audio/Shared/AudioProject/Compiler/AudioProjectCompilerService.cs +++ b/Editors/Audio/Shared/AudioProject/Compiler/AudioProjectCompilerService.cs @@ -3,6 +3,7 @@ using Editors.Audio.Shared.AudioProject.Models; using Editors.Audio.Shared.Dat; using Editors.Audio.Shared.GameInformation.Warhammer3; +using Editors.Audio.Shared.Storage; using Editors.Audio.Shared.Wwise.Generators; using Shared.Core.Misc; using Shared.GameFormats.Wwise; @@ -17,11 +18,13 @@ public interface IAudioProjectCompilerService public class AudioProjectCompilerService( ISoundBankGeneratorService soundBankGeneratorService, IWemGeneratorService wemGeneratorService, - IDatGeneratorService datGeneratorService) : IAudioProjectCompilerService + IDatGeneratorService datGeneratorService, + IAudioRepository audioRepository) : IAudioProjectCompilerService { private readonly ISoundBankGeneratorService _soundBankGeneratorService = soundBankGeneratorService; private readonly IWemGeneratorService _wemGeneratorService = wemGeneratorService; private readonly IDatGeneratorService _datGeneratorService = datGeneratorService; + private readonly IAudioRepository _audioRepository = audioRepository; private readonly ILogger _logger = Logging.Create(); @@ -30,6 +33,7 @@ public void Compile(AudioProjectFile audioProject, string audioProjectFileName, if (audioProject.SoundBanks.Count == 0) return; + _audioRepository.Load([audioProject.Language]); _logger.Here().Information($"Compiling {audioProjectFileName}"); var audioFiles = new List(); @@ -42,17 +46,16 @@ public void Compile(AudioProjectFile audioProject, string audioProjectFileName, GenerateSoundBanks(audioProject); GenerateDatFiles(audioProject, audioProjectNameWithoutExtension); - MemoryOptimiser.Optimise(); } private static void ClearTempAudioFiles() { - if (Directory.Exists(DirectoryHelper.AudioDirectory)) + if (Directory.Exists(DirectoryHelper.AudioTempDirectory)) { - foreach (var file in Directory.GetFiles(DirectoryHelper.AudioDirectory, "*.wav")) + foreach (var file in Directory.GetFiles(DirectoryHelper.AudioTempDirectory, "*.wav")) File.Delete(file); - foreach (var file in Directory.GetFiles(DirectoryHelper.AudioDirectory, "*.wem")) + foreach (var file in Directory.GetFiles(DirectoryHelper.AudioTempDirectory, "*.wem")) File.Delete(file); } } @@ -181,7 +184,7 @@ private static void SetRandomSequenceContainerData(AudioProjectFile audioProject private static void SetSoundData(AudioFile audioFile, SoundBank soundBank) { audioFile.WemPackFileName = $"{audioFile.Id}.wem"; - audioFile.WemDiskFilePath = $"{DirectoryHelper.AudioDirectory}\\{audioFile.WemPackFileName}"; + audioFile.WemDiskFilePath = $"{DirectoryHelper.AudioTempDirectory}\\{audioFile.WemPackFileName}"; if (soundBank.Language == Wh3LanguageInformation.GetLanguageAsString(Wh3Language.Sfx)) audioFile.WemPackFilePath = $"audio\\wwise\\{audioFile.WemPackFileName}"; diff --git a/Editors/Audio/Shared/Storage/AudioRepository.cs b/Editors/Audio/Shared/Storage/AudioRepository.cs index 7d084f05b..167781e7e 100644 --- a/Editors/Audio/Shared/Storage/AudioRepository.cs +++ b/Editors/Audio/Shared/Storage/AudioRepository.cs @@ -1,9 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using System.Security.Cryptography; +using System.Text; using Editors.Audio.Shared.GameInformation.Warhammer3; -using Shared.Core.Misc; +using Editors.Audio.Shared.Storage.CacheDatabase; +using Shared.Core.Events; using Shared.Core.PackFiles; +using Shared.Core.PackFiles.Events; using Shared.Core.PackFiles.Models; using Shared.Core.Settings; using Shared.GameFormats.Wwise.Didx; @@ -22,12 +23,13 @@ public interface IAudioRepository Dictionary> QualifiedStateGroupByStateGroupByDialogueEvent { get; } Dictionary> StatesByStateGroup { get; } - void Load(List languages); + void EnsureGameFilesCache(); + void Load(List languages); void Clear(); - List GetHircsByType() where T : class; - List GetHircsByHircType(AkBkHircType type); + List GetHircs(AkBkHircType type); List GetHircs(uint id); List GetHircs(uint id, string owningFileName); + Dictionary> GetHircs(IReadOnlyCollection ids); string GetNameFromId(uint value); string GetNameFromId(uint value, out bool found); string GetNameFromId(uint? key); @@ -41,112 +43,249 @@ public interface IAudioRepository byte[] FindDataWem(uint dataSoundbankId, int fileOffset, int byteCount); } - public class AudioRepository( - ApplicationSettingsService applicationSettingsService, - BnkLoader bnkLoader, - DatLoader datLoader, - IPackFileService packFileService) : IAudioRepository, IDisposable + internal class AudioRepository : IAudioRepository, IDisposable { - private readonly ApplicationSettingsService _applicationSettingsService = applicationSettingsService; - private readonly BnkLoader _bnkLoader = bnkLoader; - private readonly DatLoader _datLoader = datLoader; - private readonly IPackFileService _packFileService = packFileService; + private readonly ApplicationSettingsService _applicationSettingsService; + private readonly IPackFileService _packFileService; + private readonly IAudioCacheHelper _cacheHelper; + private readonly BnkLoader _bnkLoader; + private readonly IEventHub _eventHub; private readonly List _loadedBnkDataLanguages = []; - private bool _isDatDataLoaded = false; + private readonly List _loadedLayers = []; + private string _loadedFingerprint = ""; + private bool _allCachedHircsLoaded; + private bool _allCachedDidxLoaded; + private Dictionary> _hircsById = []; + private Dictionary> _hircsByType = []; + private Dictionary> _didxAudioListById = []; + + public Dictionary> HircsById => GetAllCachedHircs(); + public Dictionary> DidxAudioListById => GetAllCachedDidx(); + public Dictionary PackFileByBnkName { get; private set; } = []; + public Dictionary NameById { get; private set; } = []; + public Dictionary> StateGroupsByDialogueEvent { get; private set; } = []; + public Dictionary> QualifiedStateGroupByStateGroupByDialogueEvent { get; private set; } = []; + public Dictionary> StatesByStateGroup { get; private set; } = []; + + public AudioRepository( + ApplicationSettingsService applicationSettingsService, + IPackFileService packFileService, + IAudioCacheHelper cacheHelper, + BnkLoader bnkLoader, + IEventHub eventHub) + { + _applicationSettingsService = applicationSettingsService; + _packFileService = packFileService; + _cacheHelper = cacheHelper; + _bnkLoader = bnkLoader; + _eventHub = eventHub; + _eventHub.Register(this, OnPackFileContainerSetAsMainEditable); + } + + private void OnPackFileContainerSetAsMainEditable(PackFileContainerSetAsMainEditableEvent e) + { + if (e.Container == null || e.Container.IsCaPackFile || _loadedLayers.Count == 0) + return; - public Dictionary> HircsById { get; set; } - public Dictionary> DidxAudioListById { get; set; } - public Dictionary PackFileByBnkName { get; set; } - public Dictionary NameById { get; set; } - public Dictionary> StateGroupsByDialogueEvent { get; set; } - public Dictionary> QualifiedStateGroupByStateGroupByDialogueEvent { get; set; } - public Dictionary> StatesByStateGroup { get; set; } + Load([]); + } + + public void EnsureGameFilesCache() + { + var source = CreateGameFilesCacheSource(); + if (source == null) + return; + + using var repository = LoadCachedRepository(source); + } public void Load(List languages) { - var loadedData = false; + var requestedLanguages = _loadedBnkDataLanguages + .Union(languages, StringComparer.OrdinalIgnoreCase) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); var gameInformation = GameInformationDatabase.GetGameById(_applicationSettingsService.CurrentSettings.CurrentGame); - var gameBankGeneratorVersion = gameInformation.BankGeneratorVersion; + if (gameInformation.BankGeneratorVersion == GameBnkVersion.Unsupported) + return; - if (gameBankGeneratorVersion != GameBnkVersion.Unsupported) - { - var loadDatData = !_isDatDataLoaded; - var loadBnkData = !languages.All(language => _loadedBnkDataLanguages.Contains(language, StringComparer.OrdinalIgnoreCase)); - - if (loadDatData || loadBnkData) - MemoryOptimiser.LogMemory("Before loading AudioRepository"); + var cacheSources = CreateCacheSources(); + if (cacheSources.Count == 0) + return; - if (loadDatData) - { - LoadDatData(); - loadedData = true; - } + LoadCacheSources(cacheSources, requestedLanguages); + } - if (loadBnkData) - { - LoadBnkData(languages); - loadedData = true; - } + internal void LoadCacheSources(List cacheSources, List requestedLanguages) + { + var fingerprint = CreateCombinedFingerprint(cacheSources); + if (_loadedLayers.Count != 0 + && fingerprint == _loadedFingerprint + && requestedLanguages.All( + language => _loadedBnkDataLanguages.Contains( + language, + StringComparer.OrdinalIgnoreCase))) + { + return; } - if (loadedData) + var layers = LoadLayers(cacheSources); + try + { + var resolvedBnks = CreateEffectiveBnks(layers, requestedLanguages); + foreach (var bnk in resolvedBnks.Values) + bnk.Layer.ResolvedBnkPaths.Add(bnk.Bnk.Path); + + var datData = MergeDatData(layers.Select(layer => layer.AudioCache.LoadDatData())); + ApplyLoadedLayers( + layers, + resolvedBnks.Keys.ToList(), + datData, + requestedLanguages, + fingerprint); + } + catch { - MemoryOptimiser.Optimise(); - MemoryOptimiser.LogMemory("After loading AudioRepository"); + DisposeLayers(layers); + throw; } + } - private void LoadDatData() + internal List CreateCacheSources() { - var result = _datLoader.LoadDatData(); - NameById = result.NameById ?? []; - StateGroupsByDialogueEvent = result.StateGroupsByDialogueEvent ?? []; - QualifiedStateGroupByStateGroupByDialogueEvent = result.QualifiedStateGroupByStateGroupByDialogueEvent ?? []; - StatesByStateGroup = result.StatesByStateGroup ?? []; - - _isDatDataLoaded = true; + var allContainers = _packFileService.GetAllPackfileContainers(); + var gameFileContainers = allContainers.Where(container => container.IsCaPackFile).ToList(); + if (gameFileContainers.Count == 0) + return []; + + var sources = new List { CreateGameFilesCacheSource(gameFileContainers) }; + + var projectFileContainers = allContainers.Where(container => !container.IsCaPackFile).ToList(); + if (!HasRelevantAudioFiles(projectFileContainers)) + return sources; + + var projectFilesFingerprint = _cacheHelper.ComputeFingerprint(projectFileContainers, "project files"); + var editableContainer = _packFileService.GetEditablePack(); + var cacheOwner = editableContainer ?? projectFileContainers[^1]; + var projectFilesLabel = cacheOwner.Name; + + sources.Add(new AudioCacheSource( + _cacheHelper.GetCacheFilePath(projectFilesLabel, projectFilesFingerprint), + projectFilesFingerprint, + false, + projectFileContainers, + projectFileContainers)); + return sources; } - private void LoadBnkData(List languages) + public void Clear() { - var allLanguages = Wh3LanguageInformation.GetAllLanguages(); - var languageToFilterOut = allLanguages - .Where(language => !languages.Contains(language)) - .ToList(); - var result = _bnkLoader.LoadBnkFiles(languageToFilterOut); - HircsById = result.HircsById ?? []; - DidxAudioListById = result.DidxAudioListById ?? []; - PackFileByBnkName = result.PackFileByBnkName ?? []; + DisposeLayers(_loadedLayers); + _loadedLayers.Clear(); + _loadedBnkDataLanguages.Clear(); + _loadedFingerprint = ""; + _allCachedHircsLoaded = false; + _allCachedDidxLoaded = false; + _hircsById = []; + _hircsByType = []; + _didxAudioListById = []; + PackFileByBnkName = []; + NameById = []; + StateGroupsByDialogueEvent = []; + QualifiedStateGroupByStateGroupByDialogueEvent = []; + StatesByStateGroup = []; - _loadedBnkDataLanguages.AddRange(languages); } - public List GetHircsByType() where T : class + public List GetHircs(AkBkHircType hircType) { - return HircsById.Values - .SelectMany(items => items) - .OfType() - .ToList(); - } + if (_loadedLayers.Count != 0 && !_allCachedHircsLoaded) + { + if (_hircsByType.TryGetValue(hircType, out var cachedHircs)) + return cachedHircs; - public List GetHircsByHircType(AkBkHircType hircType) - { - return HircsById.SelectMany(x => x.Value) + var references = new List(); + foreach (var layer in _loadedLayers) + { + references.AddRange( + layer.AudioCache.FindHircs( + hircType, + layer.ResolvedBnkPaths)); + } + + var hircs = _bnkLoader.LoadHircs(references); + _hircsByType[hircType] = hircs; + return hircs; + } + + return _hircsById + .SelectMany(entry => entry.Value) .Where(hirc => hirc.HircType == hircType) .ToList(); } public List GetHircs(uint id) { - if (HircsById.TryGetValue(id, out var value)) - return value; + if (_hircsById.TryGetValue(id, out var hircs)) + return hircs; + + if (_loadedLayers.Count != 0 && !_allCachedHircsLoaded) + { + var references = new List(); + foreach (var layer in _loadedLayers) + { + references.AddRange( + layer.AudioCache.FindHircs( + id, + layer.ResolvedBnkPaths)); + } + + hircs = _bnkLoader.LoadHircs(references); + _hircsById[id] = hircs; + return hircs; + } + return []; } public List GetHircs(uint id, string owningFileName) => GetHircs(id).Where(x => x.BnkFilePath == owningFileName).ToList(); + public Dictionary> GetHircs(IReadOnlyCollection ids) + { + var resolvedHircsById = new Dictionary>(); + var uncachedIds = new HashSet(); + + foreach (var id in ids) + { + if (_hircsById.TryGetValue(id, out var cachedHircs)) + resolvedHircsById[id] = cachedHircs; + else + uncachedIds.Add(id); + } + + if (uncachedIds.Count != 0 && _loadedLayers.Count != 0 && !_allCachedHircsLoaded) + { + var references = new List(); + foreach (var layer in _loadedLayers) + references.AddRange(layer.AudioCache.FindHircs(uncachedIds, layer.ResolvedBnkPaths)); + + var loadedHircs = _bnkLoader.LoadHircs(references); + foreach (var hircsForId in loadedHircs.GroupBy(hirc => hirc.Id)) + { + var groupedHircs = hircsForId.ToList(); + _hircsById[hircsForId.Key] = groupedHircs; + resolvedHircsById[hircsForId.Key] = groupedHircs; + } + } + + return resolvedHircsById; + } + + public string GetNameFromId(uint value) => GetNameFromId(value, out var _); + public string GetNameFromId(uint value, out bool found) { found = NameById.ContainsKey(value); @@ -155,8 +294,6 @@ public string GetNameFromId(uint value, out bool found) return value.ToString(); } - public string GetNameFromId(uint value) => GetNameFromId(value, out var _); - public string GetNameFromId(uint? key) { if (key.HasValue) @@ -167,27 +304,40 @@ public string GetNameFromId(uint? key) public HashSet GetUsedVanillaHircIdsByLanguageId(uint languageId) { + if (_loadedLayers.Count != 0 && !_allCachedHircsLoaded) + { + var result = new HashSet(); + foreach (var layer in _loadedLayers) + { + result.UnionWith( + layer.AudioCache.FindHircIds( + languageId, + true, + layer.ResolvedBnkPaths)); + } + return result; + } + return HircsById - .SelectMany(hircLookupEntry => hircLookupEntry.Value - .Where(hirc => hirc.LanguageId == languageId && hirc.IsCAHircItem == true) - .Select(_ => hircLookupEntry.Key)) + .SelectMany( + entry => entry.Value + .Where(hirc => hirc.LanguageId == languageId && hirc.IsCA == true) + .Select(_ => entry.Key)) .ToHashSet(); } public HashSet GetUsedVanillaSourceIdsByLanguageId(uint languageId) { - return HircsById - .SelectMany(hircLookupEntry => hircLookupEntry.Value - .Where(hirc => hirc.LanguageId == languageId && hirc is ICAkSound && hirc.IsCAHircItem == true) - .Select(hirc => ((ICAkSound)hirc).GetSourceId())) + return GetHircs(AkBkHircType.Sound) + .Where(hirc => hirc.LanguageId == languageId && hirc is ICAkSound && hirc.IsCA == true) + .Select(hirc => ((ICAkSound)hirc).GetSourceId()) .ToHashSet(); } public Dictionary>> GetVanillaDialogueEventsByBnkByLanguage() { - return GetHircsByType() - .Select(hirc => hirc as HircItem) - .Where(hirc => hirc.IsCAHircItem) + return GetHircs(AkBkHircType.Dialogue_Event) + .Where(hirc => hirc.IsCA) .GroupBy(hirc => GetNameFromId(hirc.LanguageId)) .ToDictionary( languageGroup => languageGroup.Key, @@ -201,7 +351,7 @@ public Dictionary>> GetModdedHircsByBn { return HircsById .SelectMany(hirc => hirc.Value) - .Where(hirc => hirc.IsCAHircItem == false) + .Where(hirc => hirc.IsCA == false) .GroupBy(hirc => GetNameFromId(hirc.LanguageId)) .ToDictionary( languageGroup => languageGroup.Key, @@ -213,9 +363,8 @@ public Dictionary>> GetModdedHircsByBn public Dictionary> GetModdedDialogueEventsByLanguage(List moddedSoundBanks) { - return GetHircsByType() - .Select(hirc => hirc as HircItem) - .Where(hirc => hirc.IsCAHircItem == false && moddedSoundBanks.Contains(hirc.BnkFilePath)) + return GetHircs(AkBkHircType.Dialogue_Event) + .Where(hirc => hirc.IsCA == false && moddedSoundBanks.Contains(hirc.BnkFilePath)) .GroupBy(hirc => GetNameFromId(hirc.LanguageId)) .ToDictionary(group => group.Key, group => group.ToList()); } @@ -223,57 +372,14 @@ public Dictionary> GetModdedDialogueEventsByLanguage(List public List GetModdedSoundBankFilePaths(string bnkNameSubstring) { return HircsById - .SelectMany(hircDictionaryEntry => hircDictionaryEntry.Value) - .Where(hirc => hirc.IsCAHircItem == false && hirc.BnkFilePath.Contains(bnkNameSubstring)) - .Select(hirc => hirc.BnkFilePath ) + .SelectMany(hircDictionaryEntry => hircDictionaryEntry.Value) + .Where(hirc => hirc.IsCA == false && hirc.BnkFilePath.Contains(bnkNameSubstring)) + .Select(hirc => hirc.BnkFilePath ) .Distinct(StringComparer.OrdinalIgnoreCase) .OrderBy(bnkFilePath => bnkFilePath, StringComparer.OrdinalIgnoreCase) .ToList(); } - public void Clear() - { - MemoryOptimiser.LogMemory("Before clearing AudioRepository"); - - if (HircsById != null) - { - foreach (var list in HircsById.Values) - { - list?.Clear(); - list?.TrimExcess(); - } - HircsById.Clear(); - HircsById = null; - } - - if (DidxAudioListById != null) - { - foreach (var list in DidxAudioListById.Values) - { - list?.Clear(); - list?.TrimExcess(); - } - DidxAudioListById.Clear(); - DidxAudioListById = null; - } - - _loadedBnkDataLanguages?.Clear(); - _isDatDataLoaded = false; - PackFileByBnkName?.Clear(); - PackFileByBnkName = null; - NameById?.Clear(); - NameById = null; - StateGroupsByDialogueEvent?.Clear(); - StateGroupsByDialogueEvent = null; - QualifiedStateGroupByStateGroupByDialogueEvent?.Clear(); - QualifiedStateGroupByStateGroupByDialogueEvent = null; - StatesByStateGroup?.Clear(); - StatesByStateGroup = null; - - MemoryOptimiser.Optimise(); - MemoryOptimiser.LogMemory("After clearing AudioRepository"); - } - public PackFile FindWem(string wemId) { var wemFile = _packFileService.FindFile($"audio\\wwise\\{wemId}.wem"); @@ -305,6 +411,223 @@ public byte[] FindDataWem(uint dataSoundbankId, int fileOffset, int byteCount) return byteChunk.ReadBytes(byteCount); } - public void Dispose() => Clear(); + private AudioCacheSource CreateGameFilesCacheSource() + { + var gameFileContainers = _packFileService + .GetAllPackfileContainers() + .Where(container => container.IsCaPackFile) + .ToList(); + return gameFileContainers.Count == 0 ? null : CreateGameFilesCacheSource(gameFileContainers); + } + + private AudioCacheSource CreateGameFilesCacheSource(List gameFileContainers) + { + var fingerprint = _cacheHelper.ComputeFingerprint(gameFileContainers, "game files"); + var label = gameFileContainers[0].Name; + return new AudioCacheSource(_cacheHelper.GetCacheFilePath(label, fingerprint), fingerprint, true, gameFileContainers, gameFileContainers); + } + + private AudioCache LoadCachedRepository(AudioCacheSource source) + { + return _cacheHelper.TryLoadFromCache(source.CacheFilePath, source.Fingerprint) ?? _cacheHelper.SaveAndLoadCache(source); + } + + private List LoadLayers(List sources) + { + var layers = new List(); + try + { + foreach (var source in sources) + layers.Add(new LoadedLayer(LoadCachedRepository(source))); + return layers; + } + catch + { + DisposeLayers(layers); + throw; + } + } + + private void ApplyLoadedLayers(List layers, List bnkPaths, CachedAudioDatData datData, List languages, string fingerprint) + { + DisposeLayers(_loadedLayers); + _loadedLayers.Clear(); + _loadedLayers.AddRange(layers); + _hircsById = []; + _hircsByType = []; + _didxAudioListById = []; + _allCachedHircsLoaded = false; + _allCachedDidxLoaded = false; + NameById = datData.NameById; + StateGroupsByDialogueEvent = datData.StateGroupsByDialogueEvent; + + // Add qualifiers to State Groups as some events have the same State Group twice e.g. VO_Actor. + QualifiedStateGroupByStateGroupByDialogueEvent = DatLoader.BuildDialogueEventsWithStateGroupsWithQualifiersAndStateGroups(StateGroupsByDialogueEvent); + StatesByStateGroup = datData.StatesByStateGroup; + _loadedBnkDataLanguages.Clear(); + _loadedBnkDataLanguages.AddRange(languages); + _loadedFingerprint = fingerprint; + SetCurrentBnkFiles(bnkPaths); + } + + internal static CachedAudioDatData MergeDatData(IEnumerable layers) + { + var result = new CachedAudioDatData(); + foreach (var layer in layers) + { + foreach (var (id, name) in layer.NameById) + result.NameById[id] = name; + + AppendLists(result.StateGroupsByDialogueEvent, layer.StateGroupsByDialogueEvent); + AppendLists(result.StatesByStateGroup, layer.StatesByStateGroup); + } + + return result; + } + + private static void AppendLists(Dictionary> target, Dictionary> source) + { + foreach (var (key, values) in source) + { + if (!target.TryGetValue(key, out var targetValues)) + { + targetValues = []; + target[key] = targetValues; + } + + targetValues.AddRange(values); + } + } + + private void SetCurrentBnkFiles(List bnkPaths) + { + PackFileByBnkName = []; + foreach (var bnkPath in bnkPaths) + { + var bnk = _packFileService.FindFile(bnkPath); + if (bnk != null) + PackFileByBnkName.TryAdd(bnk.Name, bnk); + } + } + + private static Dictionary CreateEffectiveBnks(List layers, List languages) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var layer in layers) + { + foreach (var bnk in layer.AudioCache.GetBnks()) + { + if (IsLanguageIncluded(bnk.Path, languages)) + result[bnk.Path] = new ResolvedBnk(layer, bnk); + } + } + return result; + } + + private void EnsureAllCachedHircsLoaded() + { + if (_loadedLayers.Count == 0 || _allCachedHircsLoaded) + return; + + var references = new List(); + foreach (var layer in _loadedLayers) + references.AddRange(layer.AudioCache.FindAllHircs(layer.ResolvedBnkPaths)); + + _hircsById = _bnkLoader + .LoadHircs(references) + .GroupBy(hirc => hirc.Id) + .ToDictionary(group => group.Key, group => group.ToList()); + _hircsByType.Clear(); + _allCachedHircsLoaded = true; + } + + private Dictionary> GetAllCachedHircs() + { + EnsureAllCachedHircsLoaded(); + return _hircsById; + } + + private void EnsureAllCachedDidxLoaded() + { + if (_loadedLayers.Count == 0 || _allCachedDidxLoaded) + return; + + var didxById = new Dictionary>(); + foreach (var layer in _loadedLayers) + { + var references = layer.AudioCache.FindDidx( + layer.ResolvedBnkPaths); + foreach (var reference in references) + { + var didx = _bnkLoader.LoadDidx(reference); + if (didx == null) + continue; + + if (!didxById.TryGetValue(didx.Id, out var entries)) + { + entries = []; + didxById[didx.Id] = entries; + } + entries.Add(didx); + } + } + + _didxAudioListById = didxById; + _allCachedDidxLoaded = true; + } + + private Dictionary> GetAllCachedDidx() + { + EnsureAllCachedDidxLoaded(); + return _didxAudioListById; + } + + private static string CreateCombinedFingerprint(List sources) + { + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("|", sources.Select(source => source.Fingerprint))))); + } + + private static bool HasRelevantAudioFiles(List containers) + { + return containers.Any(container => container.SearchFiles(null, [".bnk", ".dat", ".wwiseids"]).Count != 0); + } + + private static bool IsLanguageIncluded(string bnkPath, List languages) + { + if (languages.Count == 0) + return true; + + var normalisedPath = bnkPath.Replace('/', '\\'); + var localisedLanguages = Wh3LanguageInformation.GetAllLanguages().Where(language => !language.Equals("sfx", StringComparison.OrdinalIgnoreCase)); + foreach (var language in localisedLanguages) + { + if (!normalisedPath.Contains($"\\{language}\\", StringComparison.OrdinalIgnoreCase)) + continue; + + return languages.Contains(language, StringComparer.OrdinalIgnoreCase); + } + + return true; + } + + private static void DisposeLayers(List layers) + { + foreach (var layer in layers) + layer.AudioCache.Dispose(); + } + + public void Dispose() + { + _eventHub.UnRegister(this); + Clear(); + } + + private sealed class LoadedLayer(AudioCache audioCache) + { + public AudioCache AudioCache { get; } = audioCache; + public HashSet ResolvedBnkPaths { get; } = new(StringComparer.OrdinalIgnoreCase); + } + + private sealed record ResolvedBnk(LoadedLayer Layer, AudioCache.CachedAudioBnk Bnk); } } diff --git a/Editors/Audio/Shared/Storage/BnkLoader.cs b/Editors/Audio/Shared/Storage/BnkLoader.cs index 0f68fa965..c919da8fe 100644 --- a/Editors/Audio/Shared/Storage/BnkLoader.cs +++ b/Editors/Audio/Shared/Storage/BnkLoader.cs @@ -1,122 +1,274 @@ -using System.Text; +using System.IO; +using System.Text; +using Shared.ByteParsing; using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; -using Shared.Core.PackFiles.Utility; +using Shared.Core.PackFiles.Models.FileSources; using Shared.GameFormats.Wwise; using Shared.GameFormats.Wwise.Didx; +using Shared.GameFormats.Wwise.Enums; using Shared.GameFormats.Wwise.Hirc; namespace Editors.Audio.Shared.Storage { + internal sealed record BnkFileReference(string Path, PackFile File, bool IsCA); + + internal sealed record BnkHircReference( + uint Id, + AkBkHircType HircType, + string BnkPath, + long Offset, + int Length, + uint IndexInBnk, + uint BankGeneratorVersion, + uint LanguageId, + bool IsCA); + + internal sealed record BnkDidxReference(uint Id, string BnkPath, uint LanguageId, long Offset, int Length); + public class BnkLoader(IPackFileService packFileService) { - public class Result - { - public Dictionary> HircsById { get; internal set; } = []; - public Dictionary> DidxAudioListById { get; internal set; } = []; - public Dictionary PackFileByBnkName { get; internal set; } = []; - } - private readonly IPackFileService _packFileService = packFileService; - readonly ILogger _logger = Logging.Create(); + private readonly ILogger _logger = Logging.Create(); - public ParsedBnkFile LoadBnkFile(PackFile bnkFile, string bnkFilePath, bool isCAHircItem, bool printData = false) + public BnkFile LoadBnkFile(PackFile bnkFile, string bnkFilePath, bool isCA, bool printData = false) { - var soundDb = BnkParser.Parse(bnkFile, bnkFilePath, isCAHircItem); - if (printData) - PrintHircList(soundDb.HircChunk.HircItems, bnkFilePath); - return soundDb; + var bnk = BnkFile.CreateFromBytes(bnkFile.DataSource.ReadData(), bnkFilePath, isCA); + if (printData && bnk.HircChunk != null) + PrintHircList(bnk.HircChunk.HircItems, bnkFilePath); + return bnk; } - public Result LoadBnkFiles(List languageToFilterOut) + internal static List FindBnkFiles(List containers) { - var bankFiles = PackFileServiceUtility.FindAllWithExtentionIncludePaths(_packFileService, ".bnk"); - var bankFilesAsDictionary = bankFiles.GroupBy(f => f.FileName).ToDictionary(g => g.Key, g => g.Last().Pack); + var resolvedBnks = new Dictionary(StringComparer.OrdinalIgnoreCase); - var removeFilter = new List() { "media", "init.bnk", "animation_blood_data.bnk" }; - removeFilter.AddRange(languageToFilterOut); + foreach (var container in containers) + { + foreach (var (path, file) in container.SearchFiles(null, [".bnk"])) + { + if (IsWantedBnk(path)) + resolvedBnks[path] = new BnkFileReference(path, file, container.IsCaPackFile); + } + } - var wantedBnkFiles = PackFileUtil.FilterUnvantedFiles(bankFilesAsDictionary, removeFilter.ToArray(), out var removedFiles); ; - _logger.Here().Information($"Parsing game sounds. {bankFiles.Count} bnk files found. {wantedBnkFiles.Count} after filtering"); + return resolvedBnks.Values.ToList(); + } - var parsedBnks = new List(); - var bnksWithUnknownHircs = new List(); - var failedBnks = new List<(string bnkFile, string Error)>(); - var result = new Result(); - var counter = 1; + internal BnkFile.Index LoadIndex(PackFile bnk, string bnkPath) + { + var source = bnk.DataSource; + var decodedSize = GetDecodedSize(source); + if (TryGetDirectReadLocation(bnk, bnkPath, out var filePath, out var fileOffset)) + return BnkFile.BuildIndex(bnkPath, decodedSize, (offset, length) => ReadFileRange(filePath, checked(fileOffset + offset), length)); - Parallel.ForEach(wantedBnkFiles, bnkFile => + var decodedData = source.ReadData(); + return BnkFile.BuildIndex(bnkPath, decodedData.Length, (offset, length) => ReadByteArrayRange(decodedData, offset, length)); + } + + internal List LoadHircs(List references) + { + var result = new List(references.Count); + foreach (var bnkReferences in references.GroupBy(x => x.BnkPath, StringComparer.OrdinalIgnoreCase)) { - var filePath = bnkFile.Key; - _logger.Here().Information($"{counter++}/{wantedBnkFiles.Count} - {filePath}"); + var bnk = FindBnk(bnkReferences.Key); + if (bnk == null) + continue; - var packFile = bnkFile.Value; - var packFileContainer = _packFileService.GetPackFileContainer(packFile); - result.PackFileByBnkName.TryAdd(packFile.Name, packFile); + var orderedReferences = bnkReferences.OrderBy(x => x.Offset).ToList(); + var rangeOffset = orderedReferences[0].Offset; + var rangeEnd = orderedReferences.Max(x => checked(x.Offset + x.Length)); + var rangeLength = checked(rangeEnd - rangeOffset); + if (rangeLength > int.MaxValue) + { + _logger.Here().Warning($"HIRC range in '{bnkReferences.Key}' is too large to load"); + continue; + } try { - var parsedBnk = LoadBnkFile(packFile, filePath, packFileContainer.IsCaPackFile); - if (parsedBnk.HircChunk.HircItems.Any(hicItem => hicItem is UnknownHircItem == true || hicItem.HasError)) - bnksWithUnknownHircs.Add(filePath); - - parsedBnks.Add(parsedBnk); + var bnkRange = ReadData(bnk, bnkReferences.Key, rangeOffset, (int)rangeLength); + foreach (var reference in orderedReferences) + { + var hirc = LoadHirc(bnkRange, rangeOffset, reference); + if (hirc != null) + result.Add(hirc); + } } - catch (Exception e) + catch (Exception exception) { - failedBnks.Add((filePath, e.Message)); + _logger.Here().Warning($"Failed to load HIRC range from '{bnkReferences.Key}': {exception.Message}"); } - }); + } - var allHircItems = parsedBnks.SelectMany(x => x.HircChunk.HircItems); - PrintHircList(allHircItems, "All"); - if (failedBnks.Count != 0) - _logger.Here().Error($"{failedBnks.Count} banks failed: {string.Join("\n", failedBnks)}"); + return result; + } - result.HircsById = parsedBnks - .Where(parsedBnk => parsedBnk.HircChunk is not null) - .SelectMany(parsedBnk => parsedBnk.HircChunk.HircItems) - .GroupBy(item => item.Id) - .ToDictionary(group => group.Key, group => group.ToList()); + internal DidxAudio LoadDidx(BnkDidxReference reference) + { + try + { + var bnk = FindBnk(reference.BnkPath); + if (bnk == null) + return null; + + return new DidxAudio + { + Id = reference.Id, + ByteArray = ReadData(bnk, reference.BnkPath, reference.Offset, reference.Length), + OwnerFilePath = reference.BnkPath, + LanguageId = reference.LanguageId + }; + } + catch (Exception exception) + { + _logger.Here().Warning($"Failed to read embedded WEM {reference.Id} from '{reference.BnkPath}' at offset {reference.Offset}: {exception.Message}"); + return null; + } + } + private HircItem LoadHirc(byte[] bnkRange, long rangeOffset, BnkHircReference reference) + { + try + { + return ParseHirc(bnkRange, rangeOffset, reference); + } + catch (Exception exception) + { + _logger.Here().Warning($"Failed to read HIRC {reference.Id} from '{reference.BnkPath}' at offset {reference.Offset}: {exception.Message}"); + return null; + } + } - result.DidxAudioListById = parsedBnks - .Where(parsedBnk => parsedBnk.DataChunk is not null && parsedBnk.DidxChunk is not null) - .SelectMany(parsedBnk => - parsedBnk.DidxChunk.MediaList.Select(didx => new DidxAudio() + private PackFile FindBnk(string bnkPath) + { + var bnk = _packFileService.FindFile(bnkPath); + if (bnk == null) + _logger.Here().Warning($"Audio cache references missing sound bank '{bnkPath}'"); + return bnk; + } + + private static HircItem ParseHirc(byte[] bnkRange, long rangeOffset, BnkHircReference reference) + { + var relativeOffset = checked(reference.Offset - rangeOffset); + if (relativeOffset < 0 || relativeOffset > int.MaxValue || reference.Length > bnkRange.Length - relativeOffset) + throw new InvalidDataException( $"HIRC range in '{reference.BnkPath}' is outside the supplied bank data."); + + var hirc = HircItem.ReadData( + reference.BnkPath, + new ByteChunk(bnkRange, (int)relativeOffset), + reference.BankGeneratorVersion, + reference.LanguageId, + reference.IsCA, + reference.IndexInBnk, + reference.Length); + hirc.IndexInFile = reference.IndexInBnk; + hirc.BnkFilePath = reference.BnkPath; + hirc.LanguageId = reference.LanguageId; + hirc.IsCA = reference.IsCA; + hirc.ByteIndexInFile = checked((uint)reference.Offset); + + if (hirc.Id != reference.Id || hirc.HircType != reference.HircType) + throw new InvalidDataException($"HIRC index mismatch in '{reference.BnkPath}' at offset {reference.Offset}."); + + return hirc; + } + + private static bool IsWantedBnk(string path) + { + var normalizedPath = path.Replace('/', '\\'); + return !normalizedPath.Contains(@"\media\", StringComparison.OrdinalIgnoreCase) + && !normalizedPath.EndsWith(@"\init.bnk", StringComparison.OrdinalIgnoreCase) + && !normalizedPath.EndsWith(@"\animation_blood_data.bnk", StringComparison.OrdinalIgnoreCase); + } + + private byte[] ReadData(PackFile bnk, string bnkPath, long offset, int length) + { + var source = bnk.DataSource; + var decodedSize = GetDecodedSize(source); + if (offset < 0 || length < 0 || offset > decodedSize || length > decodedSize - offset) + throw new InvalidDataException($"The requested BNK range ({offset:N0} + {length:N0}) exceeds the decoded file size ({decodedSize:N0})."); + + if (TryGetDirectReadLocation(bnk, bnkPath, out var filePath, out var fileOffset)) + return ReadFileRange(filePath, checked(fileOffset + offset), length); + + return ReadByteArrayRange(source.ReadData(), offset, length); + } + + private bool TryGetDirectReadLocation(PackFile bnk, string bnkPath, out string filePath, out long fileOffset) + { + if (bnk.DataSource is PackedFileSource { IsCompressed: false, IsEncrypted: false } packedSource) + { + filePath = packedSource.Parent.FilePath; + fileOffset = packedSource.Offset; + return true; + } + + if (bnk.DataSource is FileSystemSource) + { + var container = _packFileService.GetPackFileContainer(bnk); + if (container?.ContainerType == PackFileContainerType.SystemFolder && !string.IsNullOrWhiteSpace(container.SystemFilePath)) + { + var systemFilePath = Path.Combine(container.SystemFilePath, bnkPath); + if (File.Exists(systemFilePath)) { - Id = didx.Id, - ByteArray = parsedBnk.DataChunk.GetBytesFromBuffer((int)didx.Offset, (int)didx.Size), - OwnerFilePath = parsedBnk.BkhdChunk.OwnerFilePath, - LanguageId = parsedBnk.BkhdChunk.AkBankHeader.LanguageId - })) - .GroupBy(didxAudio => didxAudio.Id) - .ToDictionary(group => group.Key, group => group.ToList()); + filePath = systemFilePath; + fileOffset = 0; + return true; + } + } + } + + filePath = ""; + fileOffset = 0; + return false; + } + + private static byte[] ReadByteArrayRange(byte[] data, long offset, int length) + { + var result = new byte[length]; + Array.Copy(data, offset, result, 0, length); + return result; + } + private static long GetDecodedSize(IDataSource source) + { + return source is PackedFileSource { IsCompressed: true } packedSource ? packedSource.UncompressedSize : source.Size; + } + + private static byte[] ReadFileRange(string filePath, long offset, int length) + { + var result = new byte[length]; + using var stream = File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + stream.Seek(offset, SeekOrigin.Begin); + stream.ReadExactly(result); return result; } - void PrintHircList(IEnumerable hircItems, string header) + private void PrintHircList(List hircItems, string header) { var stringBuilder = new StringBuilder(); stringBuilder.AppendLine($"\n Result: {header}"); - var unknownHirc = hircItems.Where(hircItem => hircItem is UnknownHircItem).Count(); - var errorHirc = hircItems.Where(hircItem => hircItem.HasError).Count(); - stringBuilder.AppendLine($"\t Total Hirc Items: {hircItems.Count()} Unknown: {unknownHirc} Decoding Errors:{errorHirc}"); + var unknownHircs = hircItems.Count(hircItem => hircItem is UnknownHircItem); + var errorHircs = hircItems.Count(hircItem => hircItem.HasError); + stringBuilder.AppendLine($"\t Total Hirc Items: {hircItems.Count} Unknown: {unknownHircs} Decoding Errors:{errorHircs}"); - var grouped = hircItems.GroupBy(hircItem => hircItem.HircType); - var groupedWithError = grouped.Where(groupedHircItems => groupedHircItems.Any(y => y is UnknownHircItem == true || y.HasError)); - var groupedWithoutError = grouped.Where(groupedHircItems => groupedHircItems.Any(y => y is UnknownHircItem == false && y.HasError == false)); + var groupedHircs = hircItems.GroupBy(hircItem => hircItem.HircType); + var groupedWithErrors = groupedHircs.Where(group => group.Any(hircItem => hircItem is UnknownHircItem || hircItem.HasError)); + var groupedWithoutErrors = groupedHircs.Where(group => group.Any(hircItem => hircItem is not UnknownHircItem && !hircItem.HasError)); stringBuilder.AppendLine("\t\t Succeeded:"); - foreach (var group in groupedWithoutError) + foreach (var group in groupedWithoutErrors) stringBuilder.AppendLine($"\t\t\t {group.Key}: Count: {group.Count()}"); - if (groupedWithError.Any()) + if (groupedWithErrors.Any()) { stringBuilder.AppendLine("\t\t Failed:"); - foreach (var group in groupedWithError) - stringBuilder.AppendLine($"\t\t\t {group.Key}: {group.Where(x => x is UnknownHircItem == true || x.HasError).Count()}/{group.Count()} Failed"); + foreach (var group in groupedWithErrors) + { + var errorCount = group.Count(hircItem => hircItem is UnknownHircItem || hircItem.HasError); + stringBuilder.AppendLine($"\t\t\t {group.Key}: {errorCount}/{group.Count()} Failed"); + } } _logger.Here().Information(stringBuilder.ToString()); diff --git a/Editors/Audio/Shared/Storage/CacheDatabase/AudioCache.cs b/Editors/Audio/Shared/Storage/CacheDatabase/AudioCache.cs new file mode 100644 index 000000000..2176b539b --- /dev/null +++ b/Editors/Audio/Shared/Storage/CacheDatabase/AudioCache.cs @@ -0,0 +1,468 @@ +using System.Collections.Concurrent; +using System.IO; +using System.Text.Json; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Shared.Core.PackFiles.Models; +using Shared.GameFormats.Wwise; +using Shared.GameFormats.Wwise.Enums; + +namespace Editors.Audio.Shared.Storage.CacheDatabase +{ + internal sealed class AudioCache : IDisposable + { + private static readonly ILogger s_logger = Logging.CreateStatic(typeof(AudioCache)); + private static readonly JsonSerializerOptions s_jsonOptions = new(); + private const int CurrentSchemaVersion = 2; + + private readonly DbContextOptions _dbOptions; + private readonly Lock _dbLock = new(); + private AudioCacheDbContext _db; + + public string DbFilePath { get; } + + public AudioCache(string dbFilePath) + { + DbFilePath = dbFilePath; + _dbOptions = new DbContextOptionsBuilder().UseSqlite($"Data Source={dbFilePath};Pooling=False").Options; + _db = CreateDbContext(); + } + + public AudioCache(DbContextOptions dbOptions) + { + DbFilePath = TryResolveFileDbPath(dbOptions); + _dbOptions = dbOptions; + _db = CreateDbContext(); + } + + public void Save(string fingerprint, bool isGameFiles, List bnkContainers, BnkLoader bnkLoader, DatLoader.Result datData) + { + s_logger.Here().Information($"Saving {(isGameFiles ? "game files" : "project files")} audio cache"); + + using (var db = new AudioCacheDbContext(_dbOptions)) + { + db.Database.EnsureDeleted(); + db.Database.EnsureCreated(); + } + + var (connection, shouldDisposeConnection) = GetSqliteConnection(_dbOptions); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + + try + { + using var transaction = connection.BeginTransaction(); + InsertCacheInfo(connection, transaction, fingerprint); + var (bnkCount, hircCount) = IndexAndInsertBnks(bnkContainers, isGameFiles, bnkLoader, connection, transaction); + + InsertDatData(connection, transaction, datData); + + transaction.Commit(); + s_logger.Here().Information($"Saved audio cache '{DbFilePath}' with {bnkCount} banks and {hircCount} HIRC references"); + } + finally + { + if (shouldDisposeConnection) + connection.Dispose(); + } + + _db.Dispose(); + _db = CreateDbContext(); + } + + public static AudioCache CreateFromFingerPrint(string dbFilePath, string expectedFingerprint) + { + if (!File.Exists(dbFilePath)) + { + s_logger.Here().Information($"No audio cache file found at '{dbFilePath}'"); + return null; + } + + var dbOptions = new DbContextOptionsBuilder().UseSqlite($"Data Source={dbFilePath};Pooling=False").Options; + return CreateFromFingerPrint(dbOptions, expectedFingerprint); + } + + public static AudioCache CreateFromFingerPrint(DbContextOptions dbOptions, string expectedFingerprint) + { + using var db = new AudioCacheDbContext(dbOptions); + try + { + db.Database.EnsureCreated(); + } + catch (Exception exception) + { + s_logger.Here().Warning($"Failed to open audio cache database: {exception.Message}"); + return null; + } + + AudioCacheInfoEntity cacheInfo; + try + { + cacheInfo = db.CacheInfo.FirstOrDefault(); + } + catch (Exception exception) + { + s_logger.Here().Warning($"Failed to read audio cache info: {exception.Message}"); + return null; + } + + if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) + { + s_logger.Here().Information($"Audio cache invalid - schema:{cacheInfo?.SchemaVersion} (expected {CurrentSchemaVersion}), fingerprint match:{cacheInfo?.Fingerprint == expectedFingerprint}"); + return null; + } + + var repository = new AudioCache(dbOptions); + s_logger.Here().Information($"Loaded audio repository from cache '{repository.DbFilePath}'"); + return repository; + } + + internal List GetBnks() + { + lock (_dbLock) + { + return _db.Bnks + .Select(x => new CachedAudioBnk( + x.Path, + (uint)x.BankGeneratorVersion, + (uint)x.LanguageId, + x.IsCA)) + .ToList(); + } + } + + internal List FindHircs(uint id, IReadOnlySet resolvedBnkPaths) + { + List references; + lock (_dbLock) + { + references = ( + from hirc in _db.Hircs + join bnk in _db.Bnks on hirc.SoundBankId equals bnk.Id + where hirc.HircId == id + select CreateHircReference(hirc, bnk)) + .ToList(); + } + + return references.Where(x => resolvedBnkPaths.Contains(x.BnkPath)).ToList(); + } + + internal List FindHircs(IReadOnlyCollection ids, IReadOnlySet resolvedBnkPaths) + { + List references; + lock (_dbLock) + { + references = ( + from hirc in _db.Hircs + join bnk in _db.Bnks on hirc.SoundBankId equals bnk.Id + where ids.Contains((uint)hirc.HircId) + select CreateHircReference(hirc, bnk)) + .ToList(); + } + + return references.Where(x => resolvedBnkPaths.Contains(x.BnkPath)).ToList(); + } + + internal List FindHircs(AkBkHircType hircType, IReadOnlySet resolvedBnkPaths) + { + List references; + lock (_dbLock) + { + references = ( + from hirc in _db.Hircs + join bnk in _db.Bnks on hirc.SoundBankId equals bnk.Id + where hirc.HircType == (int)hircType + select CreateHircReference(hirc, bnk)) + .ToList(); + } + + return references.Where(x => resolvedBnkPaths.Contains(x.BnkPath)).ToList(); + } + + internal HashSet FindHircIds(uint languageId, bool isCA, IReadOnlySet resolvedBnkPaths) + { + List references; + lock (_dbLock) + { + references = ( + from hirc in _db.Hircs + join bnk in _db.Bnks on hirc.SoundBankId equals bnk.Id + where bnk.LanguageId == languageId && bnk.IsCA == isCA + select new CachedHircIdReference((uint)hirc.HircId, bnk.Path)) + .Distinct() + .ToList(); + } + + return references + .Where(x => resolvedBnkPaths.Contains(x.BnkPath)) + .Select(x => x.Id) + .ToHashSet(); + } + + internal List FindAllHircs(IReadOnlySet resolvedBnkPaths) + { + List references; + lock (_dbLock) + references = CreateHircQuery().ToList(); + + return references.Where(x => resolvedBnkPaths.Contains(x.BnkPath)).ToList(); + } + + internal List FindDidx(IReadOnlySet resolvedBnkPaths) + { + List references; + lock (_dbLock) + { + references = ( + from didx in _db.Didx + join bnk in _db.Bnks on didx.SoundBankId equals bnk.Id + select new BnkDidxReference( + (uint)didx.SourceId, + bnk.Path, + (uint)bnk.LanguageId, + didx.Offset, + didx.Length)) + .ToList(); + } + + return references.Where(x => resolvedBnkPaths.Contains(x.BnkPath)).ToList(); + } + + internal CachedAudioDatData LoadDatData() + { + Dictionary data; + lock (_dbLock) + data = _db.DatData.ToDictionary(x => x.Name, x => x.Data); + + return new CachedAudioDatData + { + NameById = Deserialize>(data, nameof(CachedAudioDatData.NameById)), + StateGroupsByDialogueEvent = Deserialize>>(data, nameof(CachedAudioDatData.StateGroupsByDialogueEvent)), + StatesByStateGroup = Deserialize>>(data, nameof(CachedAudioDatData.StatesByStateGroup)) + }; + } + + public void Dispose() => _db.Dispose(); + + private IQueryable CreateHircQuery() + { + return + from hirc in _db.Hircs + join bnk in _db.Bnks on hirc.SoundBankId equals bnk.Id + select CreateHircReference(hirc, bnk); + } + + private static BnkHircReference CreateHircReference(CachedHircEntity hirc, CachedAudioBnkEntity bnk) + { + return new( + (uint)hirc.HircId, + (AkBkHircType)hirc.HircType, + bnk.Path, + hirc.Offset, + hirc.Length, + (uint)hirc.IndexInBnk, + (uint)bnk.BankGeneratorVersion, + (uint)bnk.LanguageId, + bnk.IsCA); + } + + private static (int BnkCount, long HircCount) IndexAndInsertBnks( + List containers, + bool isGameFiles, + BnkLoader bnkLoader, + SqliteConnection connection, + SqliteTransaction transaction) + { + var effectiveBnks = BnkLoader.FindBnkFiles(containers); + var failedBnks = new ConcurrentBag<(string Path, string Error)>(); + var writeLock = new object(); + var bnkCount = 0; + long hircCount = 0; + + Parallel.ForEach(effectiveBnks, bnk => + { + try + { + var index = bnkLoader.LoadIndex(bnk.File, bnk.Path); + lock (writeLock) + { + var bnkId = InsertBnk( + connection, + transaction, + bnk.Path, + bnk.IsCA, + index); + InsertHircs(connection, transaction, index, bnkId); + InsertDidx(connection, transaction, index, bnkId); + bnkCount++; + hircCount += index.HircEntries.Count; + } + } + catch (Exception exception) + { + failedBnks.Add((bnk.Path, exception.Message)); + } + }); + + if (!failedBnks.IsEmpty) + s_logger.Here().Warning($"{failedBnks.Count} sound banks could not be indexed: {string.Join(Environment.NewLine, failedBnks.Select(x => $"{x.Path}: {x.Error}"))}"); + + return (bnkCount, hircCount); + } + + private AudioCacheDbContext CreateDbContext() + { + var db = new AudioCacheDbContext(_dbOptions); + db.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + return db; + } + + private static (SqliteConnection Connection, bool ShouldDisposeConnection) GetSqliteConnection(DbContextOptions dbOptions) + { + var relationalOptions = dbOptions.Extensions.OfType().FirstOrDefault(); + + if (relationalOptions?.Connection is SqliteConnection sqliteConnection) + return (sqliteConnection, false); + + var connectionString = relationalOptions?.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + throw new InvalidOperationException("Unable to resolve SQLite connection from audio cache options."); + + return (new SqliteConnection(connectionString), true); + } + + private static string TryResolveFileDbPath(DbContextOptions dbOptions) + { + var relationalOptions = dbOptions.Extensions.OfType().FirstOrDefault(); + + var connectionString = relationalOptions?.Connection?.ConnectionString ?? relationalOptions?.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + return null; + + var builder = new SqliteConnectionStringBuilder(connectionString); + if (builder.Mode == SqliteOpenMode.Memory) + return null; + if (string.Equals(builder.DataSource, ":memory:", StringComparison.OrdinalIgnoreCase)) + return null; + + return string.IsNullOrWhiteSpace(builder.DataSource) ? null : builder.DataSource; + } + + private static void InsertCacheInfo(SqliteConnection connection, SqliteTransaction transaction, string fingerprint) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + INSERT INTO CacheInfo (SchemaVersion, Fingerprint) + VALUES ($schemaVersion, $fingerprint) + """; + command.Parameters.AddWithValue("$schemaVersion", CurrentSchemaVersion); + command.Parameters.AddWithValue("$fingerprint", fingerprint); + command.ExecuteNonQuery(); + } + + private static long InsertBnk(SqliteConnection connection, SqliteTransaction transaction, string path, bool isCA, BnkFile.Index index) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + INSERT INTO Bnks (Path, BankGeneratorVersion, LanguageId, IsCA) + VALUES ($path, $version, $languageId, $isCa) + RETURNING Id + """; + command.Parameters.AddWithValue("$path", path); + command.Parameters.AddWithValue("$version", (long)index.BankGeneratorVersion); + command.Parameters.AddWithValue("$languageId", (long)index.LanguageId); + command.Parameters.AddWithValue("$isCa", isCA); + return (long)command.ExecuteScalar()!; + } + + private static void InsertHircs(SqliteConnection connection, SqliteTransaction transaction, BnkFile.Index index, long bnkId) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + INSERT INTO Hircs (HircId, HircType, SoundBankId, Offset, Length, IndexInBnk) + VALUES ($id, $type, $bnkId, $offset, $length, $index) + """; + var id = command.Parameters.Add("$id", SqliteType.Integer); + var type = command.Parameters.Add("$type", SqliteType.Integer); + command.Parameters.AddWithValue("$bnkId", bnkId); + var offset = command.Parameters.Add("$offset", SqliteType.Integer); + var length = command.Parameters.Add("$length", SqliteType.Integer); + var hircIndex = command.Parameters.Add("$index", SqliteType.Integer); + command.Prepare(); + + foreach (var hirc in index.HircEntries) + { + id.Value = (long)hirc.Header.Id; + type.Value = (int)hirc.Header.HircType; + offset.Value = hirc.Offset; + length.Value = hirc.Length; + hircIndex.Value = (long)hirc.Index; + command.ExecuteNonQuery(); + } + } + + private static void InsertDidx(SqliteConnection connection, SqliteTransaction transaction, BnkFile.Index index, long bnkId) + { + if (!index.DataOffset.HasValue) + return; + + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = """ + INSERT INTO Didx (SourceId, SoundBankId, Offset, Length) + VALUES ($id, $bnkId, $offset, $length) + """; + var id = command.Parameters.Add("$id", SqliteType.Integer); + command.Parameters.AddWithValue("$bnkId", bnkId); + var offset = command.Parameters.Add("$offset", SqliteType.Integer); + var length = command.Parameters.Add("$length", SqliteType.Integer); + command.Prepare(); + + foreach (var didx in index.DidxEntries) + { + id.Value = (long)didx.Id; + offset.Value = checked(index.DataOffset.Value + didx.Offset); + length.Value = checked((int)didx.Size); + command.ExecuteNonQuery(); + } + } + + private static void InsertDatData(SqliteConnection connection, SqliteTransaction transaction, DatLoader.Result result) + { + InsertJson(connection, transaction, nameof(CachedAudioDatData.NameById), result.NameById); + InsertJson(connection, transaction, nameof(CachedAudioDatData.StateGroupsByDialogueEvent), result.StateGroupsByDialogueEvent); + InsertJson(connection, transaction, nameof(CachedAudioDatData.StatesByStateGroup), result.StatesByStateGroup); + } + + private static void InsertJson(SqliteConnection connection, SqliteTransaction transaction, string name, T value) + { + using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = "INSERT INTO DatData (Name, Data) VALUES ($name, $data)"; + command.Parameters.AddWithValue("$name", name); + command.Parameters.Add("$data", SqliteType.Blob).Value = + JsonSerializer.SerializeToUtf8Bytes(value, s_jsonOptions); + command.ExecuteNonQuery(); + } + + private static T Deserialize(Dictionary data, string key) where T : new() + { + return data.TryGetValue(key, out var bytes) ? JsonSerializer.Deserialize(bytes, s_jsonOptions) ?? new T() : new T(); + } + + internal sealed record CachedAudioBnk(string Path, uint BankGeneratorVersion, uint LanguageId, bool IsCA); + + private sealed record CachedHircIdReference(uint Id, string BnkPath); + } + + internal sealed class CachedAudioDatData + { + public Dictionary NameById { get; set; } = []; + public Dictionary> StateGroupsByDialogueEvent { get; set; } = []; + public Dictionary> StatesByStateGroup { get; set; } = []; + } +} diff --git a/Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheDbContext.cs b/Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheDbContext.cs new file mode 100644 index 000000000..c18a3d40a --- /dev/null +++ b/Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheDbContext.cs @@ -0,0 +1,94 @@ +using Microsoft.EntityFrameworkCore; + +namespace Editors.Audio.Shared.Storage.CacheDatabase +{ + internal class AudioCacheDbContext(DbContextOptions options) : DbContext(options) + { + public DbSet CacheInfo { get; set; } + public DbSet Bnks { get; set; } + public DbSet Hircs { get; set; } + public DbSet Didx { get; set; } + public DbSet DatData { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("CacheInfo"); + entity.HasKey(x => x.Id); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("Bnks"); + entity.HasKey(x => x.Id); + entity.HasIndex(x => x.Path).IsUnique(); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("Hircs"); + entity.HasKey(x => x.CacheId); + entity.HasIndex(x => x.HircId); + entity.HasIndex(x => x.SoundBankId); + entity.HasIndex(x => x.HircType); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("Didx"); + entity.HasKey(x => x.CacheId); + entity.HasIndex(x => x.SourceId); + entity.HasIndex(x => x.SoundBankId); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("DatData"); + entity.HasKey(x => x.Name); + }); + } + } + + internal class AudioCacheInfoEntity + { + public int Id { get; set; } + public int SchemaVersion { get; set; } + public string Fingerprint { get; set; } = ""; + } + + internal class CachedAudioBnkEntity + { + public long Id { get; set; } + public string Path { get; set; } = ""; + public long BankGeneratorVersion { get; set; } + public long LanguageId { get; set; } + public bool IsCA { get; set; } + } + + internal class CachedHircEntity + { + public long CacheId { get; set; } + public long HircId { get; set; } + public int HircType { get; set; } + public long SoundBankId { get; set; } + public long Offset { get; set; } + public int Length { get; set; } + public long IndexInBnk { get; set; } + } + + internal class CachedDidxEntity + { + public long CacheId { get; set; } + public long SourceId { get; set; } + public long SoundBankId { get; set; } + public long Offset { get; set; } + public int Length { get; set; } + } + + internal class CachedDatDataEntity + { + public string Name { get; set; } = ""; + public byte[] Data { get; set; } = []; + } +} diff --git a/Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheHelper.cs b/Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheHelper.cs new file mode 100644 index 000000000..8da6d8d74 --- /dev/null +++ b/Editors/Audio/Shared/Storage/CacheDatabase/AudioCacheHelper.cs @@ -0,0 +1,166 @@ +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Shared.Core.Misc; +using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Models.FileSources; +using Shared.Core.Settings; + +namespace Editors.Audio.Shared.Storage.CacheDatabase +{ + internal interface IAudioCacheHelper + { + string ComputeFingerprint(List containers, string cacheKind); + string GetCacheFilePath(string label, string fingerprint); + AudioCache TryLoadFromCache(string cacheFilePath, string fingerprint); + AudioCache SaveAndLoadCache(AudioCacheSource source); + } + + internal sealed record AudioCacheSource(string CacheFilePath, string Fingerprint, bool IsGameFiles, List BnkContainers, List DatContainers); + + internal class AudioCacheHelper(ApplicationSettingsService applicationSettingsService, DatLoader datLoader, BnkLoader bnkLoader) : IAudioCacheHelper + { + private readonly ApplicationSettingsService _applicationSettingsService = applicationSettingsService; + private readonly DatLoader _datLoader = datLoader; + private readonly BnkLoader _bnkLoader = bnkLoader; + private readonly ILogger _logger = Logging.Create(); + + public string GetCacheFilePath(string label, string fingerprint) + { + var safeLabel = string.Join("_", label.Split(Path.GetInvalidFileNameChars())); + return Path.Combine(DirectoryHelper.CacheDirectory, $"CachedAudioData_{safeLabel}_{fingerprint}.db"); + } + + public string ComputeFingerprint(List containers, string cacheKind) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var fingerprintedFiles = new HashSet(StringComparer.OrdinalIgnoreCase); + Append(hash, $"game:{_applicationSettingsService.CurrentSettings.CurrentGame};kind:{cacheKind};"); + + for (var containerIndex = 0; containerIndex < containers.Count; containerIndex++) + { + var container = containers[containerIndex]; + var relevantFiles = container.SearchFiles(null, [".bnk", ".dat", ".wwiseids"]); + foreach (var (path, file) in relevantFiles.OrderBy(x => x.Path, StringComparer.OrdinalIgnoreCase)) + { + Append(hash, $"{containerIndex}|{container.IsCaPackFile}|{path}|"); + AppendDataSourceFingerprint(hash, container, path, file.DataSource, fingerprintedFiles); + } + } + + return Convert.ToHexString(hash.GetHashAndReset()); + } + + public AudioCache SaveAndLoadCache(AudioCacheSource source) + { + Directory.CreateDirectory(Path.GetDirectoryName(source.CacheFilePath)!); + + _logger.Here().Information($"Building processed DAT data for the {(source.IsGameFiles ? "game files" : "project files")} audio cache"); + var datData = _datLoader.LoadDatData(source.DatContainers); + + using (var repository = new AudioCache(source.CacheFilePath)) + { + repository.Save(source.Fingerprint, source.IsGameFiles, source.BnkContainers, _bnkLoader, datData); + } + + var loaded = AudioCache.CreateFromFingerPrint(source.CacheFilePath, source.Fingerprint); + if (loaded == null) + throw new InvalidDataException($"Failed to load audio repository after saving. CacheFile: {source.CacheFilePath}"); + + DeleteStaleCaches(source); + return loaded; + } + + public AudioCache TryLoadFromCache(string cacheFilePath, string fingerprint) + { + if (!File.Exists(cacheFilePath)) + { + _logger.Here().Information($"Audio cache file does not exist: {cacheFilePath}"); + return null; + } + + try + { + _logger.Here().Information($"Attempting to load audio cache from: {cacheFilePath} with fingerprint: {fingerprint}"); + var result = AudioCache.CreateFromFingerPrint(cacheFilePath, fingerprint); + if (result == null) + { + _logger.Here().Information($"Audio cache load returned null (fingerprint/schema mismatch) for: {cacheFilePath}"); + } + return result; + } + catch (Exception exception) + { + _logger.Here().Warning($"Failed to load audio cache '{cacheFilePath}': {exception.Message}"); + return null; + } + } + + private void AppendDataSourceFingerprint(IncrementalHash hash, IPackFileContainer container, string relativePath, IDataSource dataSource, HashSet fingerprintedFiles) + { + if (dataSource is PackedFileSource packedSource) + { + AppendFileFingerprint(hash, packedSource.Parent.FilePath, fingerprintedFiles); + Append(hash, $"packed:{packedSource.Offset}|{packedSource.Size}|{packedSource.IsEncrypted}|{packedSource.IsCompressed}|{packedSource.CompressionFormat}|{packedSource.UncompressedSize};"); + } + else if (dataSource is FileSystemSource && container.ContainerType == PackFileContainerType.SystemFolder && !string.IsNullOrWhiteSpace(container.SystemFilePath)) + AppendFileFingerprint(hash, Path.Combine(container.SystemFilePath, relativePath), fingerprintedFiles); + else + { + Append(hash, $"memory:{dataSource.Size}|"); + hash.AppendData(SHA256.HashData(dataSource.ReadData())); + } + } + + private void AppendFileFingerprint(IncrementalHash hash, string path, HashSet fingerprintedFiles) + { + var fullPath = Path.GetFullPath(path); + if (!fingerprintedFiles.Add(fullPath)) + return; + + if (!File.Exists(fullPath)) + { + _logger.Here().Warning($"Audio cache fingerprint could not find file '{fullPath}'"); + return; + } + + var fileInfo = new FileInfo(fullPath); + Append(hash, $"{fullPath}|{fileInfo.Length}|{fileInfo.LastWriteTimeUtc.Ticks};"); + } + + private void DeleteStaleCaches(AudioCacheSource source) + { + var cacheDirectory = Path.GetDirectoryName(source.CacheFilePath); + if (cacheDirectory == null || !Directory.Exists(cacheDirectory)) + return; + + var fileName = Path.GetFileName(source.CacheFilePath); + var fingerprintSuffix = $"_{source.Fingerprint}.db"; + if (!fileName.EndsWith(fingerprintSuffix, StringComparison.OrdinalIgnoreCase)) + return; + + var labelPrefix = fileName[..^fingerprintSuffix.Length]; + var staleCacheFiles = Directory.EnumerateFiles(cacheDirectory, $"{labelPrefix}_*.db"); + + foreach (var cacheFile in staleCacheFiles) + { + if (cacheFile.Equals(source.CacheFilePath, StringComparison.OrdinalIgnoreCase)) + continue; + + try + { + File.Delete(cacheFile); + } + catch (Exception exception) + { + _logger.Here().Warning($"Failed to remove stale audio cache '{cacheFile}': {exception.Message}"); + } + } + } + + private static void Append(IncrementalHash hash, string value) + { + hash.AppendData(Encoding.UTF8.GetBytes(value)); + } + } +} diff --git a/Editors/Audio/Shared/Storage/DatLoader.cs b/Editors/Audio/Shared/Storage/DatLoader.cs index 2968b0c8e..21df864a9 100644 --- a/Editors/Audio/Shared/Storage/DatLoader.cs +++ b/Editors/Audio/Shared/Storage/DatLoader.cs @@ -18,24 +18,20 @@ public class Result { public Dictionary NameById { get; set; } = []; public Dictionary> StateGroupsByDialogueEvent { get; set; } = []; - public Dictionary> QualifiedStateGroupByStateGroupByDialogueEvent { get; set; } = []; public Dictionary> StatesByStateGroup { get; set; } = []; } private readonly IPackFileService _pfs = pfs; private readonly ApplicationSettingsService _applicationSettingsService = applicationSettingsService; - public Result LoadDatData() + public Result LoadDatData(List containers = null) { - var datDb = LoadDatFiles(_pfs, out var _); - var nameLookUp = BuildNameHelper(datDb); + var datDb = LoadDatFiles(containers, out var _); + var nameLookUp = BuildNameHelper(datDb, containers); var unprocessedDialogueEventsWithStateGroups = datDb.DialogueEventsWithStateGroups; var processedDialogueEventsWithStateGroups = ProcessDialogueEvents(unprocessedDialogueEventsWithStateGroups, nameLookUp); - // Add qualifiers to State Groups as some events have the same State Group twice e.g. VO_Actor. - var dialogueEventsWithStateGroupsWithQualifiersAndStateGroups = BuildDialogueEventsWithStateGroupsWithQualifiersAndStateGroups(processedDialogueEventsWithStateGroups); - var stateGroupsWithStates0 = datDb.StateGroupsWithStates0; var stateGroupsWithStates1 = datDb.StateGroupsWithStates1; var unprocessedStateGroupsWithStates = stateGroupsWithStates0.Concat(stateGroupsWithStates1).ToList(); @@ -45,7 +41,6 @@ public Result LoadDatData() { NameById = nameLookUp, StateGroupsByDialogueEvent = processedDialogueEventsWithStateGroups, - QualifiedStateGroupByStateGroupByDialogueEvent = dialogueEventsWithStateGroupsWithQualifiersAndStateGroups, StatesByStateGroup = processedStateGroupsWithStates, }; } @@ -122,20 +117,20 @@ public static Dictionary> BuildDialogueEvents return dialogueEventsWithStateGroupsWithQualifiersAndStateGroups; } - public Dictionary BuildNameHelper(SoundDatFile wh3Db) + public Dictionary BuildNameHelper(SoundDatFile wh3Db, List containers = null) { var nameLookUp = new Dictionary(); var wh3DbNameList = wh3Db.CreateFileNameList(); AddNames(wh3DbNameList, nameLookUp); - var bnkFiles = PackFileServiceUtility.FindAllWithExtention(_pfs, ".bnk"); + var bnkFiles = FindFiles(".bnk", containers); var bnkNames = bnkFiles.Select(x => x.Name.Replace(".bnk", "")).ToArray(); AddNames(bnkNames, nameLookUp); var languages = new List { "sfx", "chinese", "english(uk)", "french(france)", "german", "italian", "polish", "russian", "spanish(spain)" }.ToArray(); AddNames(languages, nameLookUp); - var wwiseIdFiles = PackFileServiceUtility.FindAllWithExtention(_pfs, ".wwiseids"); + var wwiseIdFiles = FindFiles(".wwiseids", containers); foreach (var item in wwiseIdFiles) { var data = Encoding.UTF8.GetString(item.DataSource.ReadData()); @@ -147,10 +142,10 @@ public Dictionary BuildNameHelper(SoundDatFile wh3Db) return nameLookUp; } - private SoundDatFile LoadDatFiles(IPackFileService pfs, out List failedFiles) + private SoundDatFile LoadDatFiles(List containers, out List failedFiles) { - var datFiles = PackFileServiceUtility.FindAllWithExtention(pfs, ".dat"); - datFiles = PackFileUtil.FilterUnvantedFiles(pfs, datFiles, ["bank_splits.dat", "campaign_music.dat", "battle_music.dat", "icudt61l.dat"], out var removedFiles); + var datFiles = FindFiles(".dat", containers); + datFiles = PackFileUtil.FilterUnvantedFiles(_pfs, datFiles, ["bank_splits.dat", "campaign_music.dat", "battle_music.dat", "icudt61l.dat"], out var removedFiles); var failedDatParsing = new List<(string, string)>(); var masterDat = new SoundDatFile(); @@ -164,7 +159,7 @@ private SoundDatFile LoadDatFiles(IPackFileService pfs, out List failedF } catch (Exception e) { - var fullPath = pfs.GetFullPath(datFile); + var fullPath = _pfs.GetFullPath(datFile); failedDatParsing.Add((fullPath, e.Message)); } } @@ -173,6 +168,23 @@ private SoundDatFile LoadDatFiles(IPackFileService pfs, out List failedF return masterDat; } + private List FindFiles(string extension, List containers) + { + if (containers == null) + return PackFileServiceUtility.FindAllWithExtentionIncludePaths(_pfs, extension) + .Select(x => x.Pack) + .ToList(); + + var effectiveFiles = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var container in containers) + { + foreach (var (path, file) in container.SearchFiles(null, [extension])) + effectiveFiles[path] = file; + } + + return effectiveFiles.Values.ToList(); + } + private SoundDatFile LoadDatFile(PackFile datFile) { if (_applicationSettingsService.CurrentSettings.CurrentGame == GameTypeEnum.Attila) diff --git a/Editors/Audio/Shared/Wwise/Generators/Hirc/HircChunkGenerator.cs b/Editors/Audio/Shared/Wwise/Generators/Hirc/HircChunkGenerator.cs index 3c055de83..4fe0e3a45 100644 --- a/Editors/Audio/Shared/Wwise/Generators/Hirc/HircChunkGenerator.cs +++ b/Editors/Audio/Shared/Wwise/Generators/Hirc/HircChunkGenerator.cs @@ -9,7 +9,7 @@ public class HircChunkGenerator { public static HircChunk GenerateHircChunk(List hircItems) { - var chunkSize = HircChunk.ChunkHeaderSize + (uint)(hircItems.Sum(hirc => HircItem.HircHeaderSize + hirc.SectionSize)); + var chunkSize = HircChunk.ChunkHeaderSize + (uint)(hircItems.Sum(hirc => HircHeader.PrefixSize + hirc.SectionSize)); var hircChunk = new HircChunk { ChunkHeader = ChunkHeaderGenerator.GenerateChunkHeader(BankChunkTypes.HIRC, chunkSize), diff --git a/Editors/Audio/Shared/Wwise/Generators/SoundBankGeneratorService.cs b/Editors/Audio/Shared/Wwise/Generators/SoundBankGeneratorService.cs index 297665255..7aee59028 100644 --- a/Editors/Audio/Shared/Wwise/Generators/SoundBankGeneratorService.cs +++ b/Editors/Audio/Shared/Wwise/Generators/SoundBankGeneratorService.cs @@ -88,9 +88,8 @@ public void GenerateDialogueEventsForTestingSoundBank(SoundBank soundBank) var dialogueEventHircs = GenerateDialogueEventHircs(soundBank); hircItems.AddRange(dialogueEventHircs); - var vanillaDialogueEvents = _audioRepository.GetHircsByType() - .Select(hircItem => hircItem as HircItem) - .Where(hircItem => hircItem.IsCAHircItem == true); + var vanillaDialogueEvents = _audioRepository.GetHircs(AkBkHircType.Dialogue_Event) + .Where(hircItem => hircItem.IsCA == true); foreach (var hircItem in hircItems) { @@ -443,7 +442,7 @@ private void WriteSoundBank(uint id, uint languageId, string fileName, string fi var bytes = memStream.ToArray(); var bnkPackFile = new PackFile(fileName, new MemorySource(bytes)); - var reparsedSanityFile = BnkParser.Parse(bnkPackFile, "test\\fakefilename.bnk", true); + var reparsedSanityFile = BnkFile.CreateFromBytes(bnkPackFile.DataSource.ReadData(), "test\\fakefilename.bnk", true); _fileSaveService.Save(filePath, bnkPackFile.DataSource.ReadData(), false); } diff --git a/Editors/Audio/Shared/Wwise/HircExploration/HircTreeChildrenParser.cs b/Editors/Audio/Shared/Wwise/HircExploration/HircTreeChildrenParser.cs index 108bf2e28..9af10b026 100644 --- a/Editors/Audio/Shared/Wwise/HircExploration/HircTreeChildrenParser.cs +++ b/Editors/Audio/Shared/Wwise/HircExploration/HircTreeChildrenParser.cs @@ -87,9 +87,7 @@ private void ProcessAction(HircItem item, HircTreeNode parent) if (action.GetActionType() == AkActionType.SetState) { var stateGroupId = action.GetStateGroupId(); - var musicSwitches = AudioRepository.HircsById - .SelectMany(kvp => kvp.Value) - .Where(hirc => hirc.HircType == AkBkHircType.Music_Switch) + var musicSwitches = AudioRepository.GetHircs(AkBkHircType.Music_Switch) .DistinctBy(hirc => hirc.Id) .Cast() .ToList(); @@ -101,9 +99,7 @@ private void ProcessAction(HircItem item, HircTreeNode parent) ProcessNext(musicSwitch.Id, node); } - var normalSwitches = AudioRepository.HircsById - .SelectMany(kvp => kvp.Value) - .Where(hirc => hirc.HircType == AkBkHircType.SwitchContainer) + var normalSwitches = AudioRepository.GetHircs(AkBkHircType.SwitchContainer) .DistinctBy(hirc => hirc.Id) .Cast() .ToList(); diff --git a/Editors/Audio/Test.Audio.csproj b/Editors/Audio/Test.Audio.csproj new file mode 100644 index 000000000..a124be9ca --- /dev/null +++ b/Editors/Audio/Test.Audio.csproj @@ -0,0 +1,28 @@ + + + + net10.0-windows + latest + enable + enable + false + false + + + + + + + + + + + + + + + + + + + diff --git a/Editors/Reports/Audio/DialogueEventAndEventNamePrinter.cs b/Editors/Reports/Audio/DialogueEventAndEventNamePrinter.cs index 6e98a1420..5649b5aa4 100644 --- a/Editors/Reports/Audio/DialogueEventAndEventNamePrinter.cs +++ b/Editors/Reports/Audio/DialogueEventAndEventNamePrinter.cs @@ -30,7 +30,8 @@ public void Create() public void PrintInfo() { - var itemsToProcess = _audioRepository.GetHircsByType() + var itemsToProcess = _audioRepository.HircsById.Values + .SelectMany(hircs => hircs) .Where(item => item is ICAkDialogueEvent or ICAkEvent) .ToList(); diff --git a/Editors/Reports/Audio/DialogueEventInfoPrinter.cs b/Editors/Reports/Audio/DialogueEventInfoPrinter.cs index 147ca1e52..a5f300cb1 100644 --- a/Editors/Reports/Audio/DialogueEventInfoPrinter.cs +++ b/Editors/Reports/Audio/DialogueEventInfoPrinter.cs @@ -2,6 +2,7 @@ using Editors.Audio.Shared.Storage; using Shared.Core.Events; using Shared.Core.Misc; +using Shared.GameFormats.Wwise.Enums; using Shared.GameFormats.Wwise.Hirc; namespace Editors.Reports.Audio @@ -30,7 +31,7 @@ public void Create() public void PrintDialogueEventInfo() { - var dialogueEvents = _audioRepository.GetHircsByType(); + var dialogueEvents = _audioRepository.GetHircs(AkBkHircType.Dialogue_Event).OfType(); foreach (var dialogueEvent in dialogueEvents) PrintDialogueEventInfo(dialogueEvent); } diff --git a/Shared/GameFiles/Wwise/Bkhd/AkBankHeader.cs b/Shared/GameFiles/Wwise/Bkhd/AkBankHeader.cs index ed64584c6..d3a112ee1 100644 --- a/Shared/GameFiles/Wwise/Bkhd/AkBankHeader.cs +++ b/Shared/GameFiles/Wwise/Bkhd/AkBankHeader.cs @@ -4,6 +4,8 @@ namespace Shared.GameFormats.Wwise.Bkhd { public class AkBankHeader { + public const int MinimumSize = 20; + public uint BankGeneratorVersion { get; set; } public uint SoundBankId { get; set; } public uint LanguageId { get; set; } @@ -13,15 +15,18 @@ public class AkBankHeader public void ReadData(ByteChunk chunk, uint chunkSize) { + if (chunkSize < MinimumSize) + throw new InvalidDataException($"BKHD chunk is only {chunkSize} bytes."); + BankGeneratorVersion = chunk.ReadUInt32(); SoundBankId = chunk.ReadUInt32(); LanguageId = chunk.ReadUInt32(); AltValues = chunk.ReadUInt32(); ProjectId = chunk.ReadUInt32(); - var headerDiff = (int)chunkSize - 20; - if (headerDiff > 0) - Padding = chunk.ReadBytes(headerDiff); + var headerDifference = (int)chunkSize - MinimumSize; + if (headerDifference > 0) + Padding = chunk.ReadBytes(headerDifference); } public byte[] WriteData() diff --git a/Shared/GameFiles/Wwise/BnkFile.cs b/Shared/GameFiles/Wwise/BnkFile.cs new file mode 100644 index 000000000..90891126e --- /dev/null +++ b/Shared/GameFiles/Wwise/BnkFile.cs @@ -0,0 +1,114 @@ +using Shared.ByteParsing; +using Shared.GameFormats.Wwise.Bkhd; +using Shared.GameFormats.Wwise.Data; +using Shared.GameFormats.Wwise.Didx; +using Shared.GameFormats.Wwise.Enums; +using Shared.GameFormats.Wwise.Hirc; +using Shared.GameFormats.Wwise.Stid; + +namespace Shared.GameFormats.Wwise +{ + public class BnkFile + { + public BkhdChunk BkhdChunk { get; set; } = new BkhdChunk(); + public HircChunk? HircChunk { get; set; } + public DidxChunk? DidxChunk { get; set; } + public DataChunk? DataChunk { get; set; } + public StidChunk? StidChunk { get; set; } + + public class Index + { + public uint BankGeneratorVersion { get; set; } + public uint LanguageId { get; set; } + public long? DataOffset { get; set; } + public List HircEntries { get; set; } = []; + public List DidxEntries { get; set; } = []; + } + + public static BnkFile CreateFromBytes(byte[] bnkBytes, string filePath, bool isCA) + { + var bnkFile = new BnkFile(); + bnkFile.ReadData(new ByteChunk(bnkBytes), filePath, isCA); + return bnkFile; + } + + public void ReadData(ByteChunk chunk, string filePath, bool isCA) + { + while (chunk.BytesLeft != 0) + { + var chunkHeader = ChunkHeader.PeekFromBytes(chunk); + var indexBeforeRead = chunk.Index; + var expectedIndexAfterRead = indexBeforeRead + ChunkHeader.ChunkHeaderSize + chunkHeader.ChunkSize; + + if (BankChunkTypes.BKHD == chunkHeader.Tag) + BkhdChunk = BkhdChunk.ReadData(filePath, chunk); + else if (BankChunkTypes.HIRC == chunkHeader.Tag) + HircChunk = HircChunk.ReadData(filePath, chunk, BkhdChunk.AkBankHeader.BankGeneratorVersion, BkhdChunk.AkBankHeader.LanguageId, isCA); + else if (BankChunkTypes.DIDX == chunkHeader.Tag) + DidxChunk = DidxChunk.ReadData(filePath, chunk); + else if (BankChunkTypes.DATA == chunkHeader.Tag) + DataChunk = DataChunk.ReadData(filePath, chunk); + else if (BankChunkTypes.STID == chunkHeader.Tag) + StidChunk = StidChunk.ReadData(filePath, chunk); + else + throw new ArgumentException($"Unknown data block '{chunkHeader.Tag}' while parsing bnk file '{filePath}'"); + + // Verify + var bytesRead = expectedIndexAfterRead - indexBeforeRead; + if (chunk.Index != expectedIndexAfterRead) + throw new Exception($"Error parsing bnk with tag '{chunkHeader.Tag}', incorrect num bytes read. '{bytesRead}' bytes read in this operation"); + } + + if (chunk.BytesLeft != 0) + throw new Exception("Error parsing bnk, bytes left"); + } + + public static Index BuildIndex(string filePath, long decodedSize, Func readData) + { + var result = new Index(); + long chunkOffset = 0; + + while (chunkOffset < decodedSize) + { + if (decodedSize - chunkOffset < ChunkHeader.ChunkHeaderSize) + throw new InvalidDataException($"BNK chunk header extends beyond the end of '{filePath}'."); + + var chunkHeader = ChunkHeader.ReadData(new ByteChunk(readData(chunkOffset, checked((int)ChunkHeader.ChunkHeaderSize)))); + var payloadOffset = checked(chunkOffset + ChunkHeader.ChunkHeaderSize); + var nextChunkOffset = checked(payloadOffset + chunkHeader.ChunkSize); + if (nextChunkOffset > decodedSize) + throw new InvalidDataException($"BNK chunk '{chunkHeader.Tag}' extends beyond the end of '{filePath}'."); + + if (chunkHeader.Tag == BankChunkTypes.BKHD) + { + var totalChunkSize = checked((int)(ChunkHeader.ChunkHeaderSize + chunkHeader.ChunkSize)); + var bkhdChunk = BkhdChunk.ReadData(filePath, new ByteChunk(readData(chunkOffset, totalChunkSize))); + result.BankGeneratorVersion = bkhdChunk.AkBankHeader.BankGeneratorVersion; + result.LanguageId = bkhdChunk.AkBankHeader.LanguageId; + } + else if (chunkHeader.Tag == BankChunkTypes.HIRC) + { + if (chunkHeader.ChunkSize > int.MaxValue) + throw new InvalidDataException($"HIRC chunk is too large to index: {chunkHeader.ChunkSize} bytes."); + + var hircEntries = HircChunk.BuildIndex(payloadOffset, chunkHeader.ChunkSize, new ByteChunk(readData(payloadOffset, (int)chunkHeader.ChunkSize))); + result.HircEntries.AddRange(hircEntries); + } + else if (chunkHeader.Tag == BankChunkTypes.DIDX) + { + if (chunkHeader.ChunkSize > int.MaxValue) + throw new InvalidDataException($"DIDX chunk is too large to index: {chunkHeader.ChunkSize} bytes."); + + var mediaHeaders = DidxChunk.ReadMediaHeaders(new ByteChunk(readData(payloadOffset, (int)chunkHeader.ChunkSize)), chunkHeader.ChunkSize); + result.DidxEntries.AddRange(mediaHeaders); + } + else if (chunkHeader.Tag == BankChunkTypes.DATA) + result.DataOffset = payloadOffset; + + chunkOffset = nextChunkOffset; + } + + return result; + } + } +} diff --git a/Shared/GameFiles/Wwise/BnkParser.cs b/Shared/GameFiles/Wwise/BnkParser.cs deleted file mode 100644 index 73bb090d8..000000000 --- a/Shared/GameFiles/Wwise/BnkParser.cs +++ /dev/null @@ -1,73 +0,0 @@ -using Shared.ByteParsing; -using Shared.Core.PackFiles.Models; -using Shared.GameFormats.Wwise.Bkhd; -using Shared.GameFormats.Wwise.Data; -using Shared.GameFormats.Wwise.Didx; -using Shared.GameFormats.Wwise.Enums; -using Shared.GameFormats.Wwise.Hirc; -using Shared.GameFormats.Wwise.Stid; - -namespace Shared.GameFormats.Wwise -{ - public class BnkParser - { - public static ParsedBnkFile Parse(PackFile packFile, string filePath, bool isCAHircItem) - { - var parsedBnkFile = new ParsedBnkFile(); - var chunk = packFile.DataSource.ReadDataAsChunk(); - - while (chunk.BytesLeft != 0) - { - if (packFile.Name == "init.bnk") - continue; - - var chunkHeader = ChunkHeader.PeekFromBytes(chunk); - var indexBeforeRead = chunk.Index; - var expectedIndexAfterRead = indexBeforeRead + ChunkHeader.ChunkHeaderSize + chunkHeader.ChunkSize; - - if (BankChunkTypes.BKHD == chunkHeader.Tag) - parsedBnkFile.BkhdChunk = LoadBkhdChunk(filePath, chunk); - else if (BankChunkTypes.HIRC == chunkHeader.Tag) - parsedBnkFile.HircChunk = LoadHircChunk(filePath, chunk, chunkHeader.ChunkSize, parsedBnkFile.BkhdChunk.AkBankHeader, isCAHircItem); - else if (BankChunkTypes.DIDX == chunkHeader.Tag) - parsedBnkFile.DidxChunk = LoadDidxChunk(filePath, chunk); - else if (BankChunkTypes.DATA == chunkHeader.Tag) - parsedBnkFile.DataChunk = LoadDataChunk(filePath, chunk); - else if (BankChunkTypes.STID == chunkHeader.Tag) - LoadStidChunk(filePath, chunk); - else - throw new ArgumentException($"Unknown data block '{chunkHeader.Tag}' while parsing bnk file '{filePath}'"); - - // Verify - var bytesRead = expectedIndexAfterRead - indexBeforeRead; - if (chunk.Index != expectedIndexAfterRead) - throw new Exception($"Error parsing bnk with tag '{chunkHeader.Tag}', incorrect num bytes read. '{bytesRead}' bytes read in this operation"); - } - - if (chunk.BytesLeft != 0) - throw new Exception("Error parsing bnk, bytes left"); - - return parsedBnkFile; - } - - private static BkhdChunk LoadBkhdChunk(string fullName, ByteChunk chunk) => BkhdChunk.ReadData(fullName, chunk); - - private static HircChunk LoadHircChunk(string fullName, ByteChunk chunk, uint chunkSize, AkBankHeader akBankHeader, bool isCAHircItem) - { - var bankGeneratorVersion = akBankHeader.BankGeneratorVersion; - var languageId = akBankHeader.LanguageId; - var hircData = HircChunk.ReadData(fullName, chunk, bankGeneratorVersion, languageId, isCAHircItem); - - var expectedHircChunkSize = HircChunk.ChunkHeaderSize + (hircData.HircItems.Sum(hirc => HircItem.HircHeaderSize + hirc.SectionSize)); - var areEqual = expectedHircChunkSize == chunkSize; - if (areEqual == false) - throw new Exception("Error parsing HIRC in bnk, expected and actual not matching"); - - return hircData; - } - - private static DidxChunk LoadDidxChunk(string fullName, ByteChunk chunk) => DidxChunk.ReadData(fullName, chunk); - private static ByteChunk LoadDataChunk(string fullName, ByteChunk chunk) => DataChunk.ReadData(fullName, chunk); - private static void LoadStidChunk(string fullName, ByteChunk chunk) => StidChunk.ReadData(fullName, chunk); - } -} diff --git a/Shared/GameFiles/Wwise/Data/DataChunk.cs b/Shared/GameFiles/Wwise/Data/DataChunk.cs index 784f7d480..02cdcc93d 100644 --- a/Shared/GameFiles/Wwise/Data/DataChunk.cs +++ b/Shared/GameFiles/Wwise/Data/DataChunk.cs @@ -1,16 +1,17 @@ -using System.Text; -using Shared.ByteParsing; +using Shared.ByteParsing; namespace Shared.GameFormats.Wwise.Data { public class DataChunk { - public static ByteChunk ReadData(string fileName, ByteChunk chunk) + public ChunkHeader ChunkHeader { get; set; } = new ChunkHeader(); + public ByteChunk Data { get; set; } = new ByteChunk([]); + + public static DataChunk ReadData(string fileName, ByteChunk chunk) { - var tag = Encoding.UTF8.GetString(chunk.ReadBytes(4)); - var chunkSize = chunk.ReadUInt32(); - var buffer = chunk.CreateSub((int)chunkSize); - return buffer; + var dataChunk = new DataChunk { ChunkHeader = ChunkHeader.ReadData(chunk) }; + dataChunk.Data = chunk.CreateSub((int)dataChunk.ChunkHeader.ChunkSize); + return dataChunk; } } } diff --git a/Shared/GameFiles/Wwise/Didx/DidxChunk.cs b/Shared/GameFiles/Wwise/Didx/DidxChunk.cs index 5dcec2b15..c5bb05c28 100644 --- a/Shared/GameFiles/Wwise/Didx/DidxChunk.cs +++ b/Shared/GameFiles/Wwise/Didx/DidxChunk.cs @@ -1,21 +1,30 @@ -using System.Text; -using Shared.ByteParsing; +using Shared.ByteParsing; namespace Shared.GameFormats.Wwise.Didx { public partial class DidxChunk { + public ChunkHeader ChunkHeader { get; set; } = new ChunkHeader(); public List MediaList { get; set; } = []; public static DidxChunk ReadData(string fileName, ByteChunk chunk) { - var tag = Encoding.UTF8.GetString(chunk.ReadBytes(4)); - var chunkSize = chunk.ReadUInt32(); - var numItems = chunkSize / MediaHeader.ByteSize; - var mediaList = Enumerable.Range(0, (int)numItems) - .Select(item => MediaHeader.ReadData(chunk)) - .ToList(); - return new DidxChunk { MediaList = mediaList }; + var didxChunk = new DidxChunk { ChunkHeader = ChunkHeader.ReadData(chunk) }; + didxChunk.MediaList = ReadMediaHeaders(chunk, didxChunk.ChunkHeader.ChunkSize); + return didxChunk; + } + + public static List ReadMediaHeaders(ByteChunk chunk, uint chunkSize) + { + if (chunkSize % MediaHeader.ByteSize != 0) + throw new InvalidDataException($"DIDX chunk size {chunkSize} is not a multiple of {MediaHeader.ByteSize}."); + + var items = chunkSize / MediaHeader.ByteSize; + var mediaHeaders = new List((int)items); + for (var itemIndex = 0; itemIndex < items; itemIndex++) + mediaHeaders.Add(MediaHeader.ReadData(chunk)); + + return mediaHeaders; } } } diff --git a/Shared/GameFiles/Wwise/Hirc/HircChunk.cs b/Shared/GameFiles/Wwise/Hirc/HircChunk.cs index 3bd134b9c..ef0450bfa 100644 --- a/Shared/GameFiles/Wwise/Hirc/HircChunk.cs +++ b/Shared/GameFiles/Wwise/Hirc/HircChunk.cs @@ -1,5 +1,4 @@ using Shared.ByteParsing; -using Shared.GameFormats.Wwise.Enums; namespace Shared.GameFormats.Wwise.Hirc { @@ -10,11 +9,8 @@ public class HircChunk public uint NumHircItems { get; set; } public List HircItems { get; set; } = []; - public static HircChunk ReadData(string filePath, ByteChunk chunk, uint bankGeneratorVersion, uint languageId, bool isCAHircItem) + public static HircChunk ReadData(string filePath, ByteChunk chunk, uint bankGeneratorVersion, uint languageId, bool isCA) { - var failedItems = new List(); - var factory = HircFactory.CreateFactory(bankGeneratorVersion); - var hircChunk = new HircChunk { ChunkHeader = ChunkHeader.ReadData(chunk), @@ -22,38 +18,59 @@ public static HircChunk ReadData(string filePath, ByteChunk chunk, uint bankGene }; for (uint itemIndex = 0; itemIndex < hircChunk.NumHircItems; itemIndex++) + hircChunk.HircItems.Add( + HircItem.ReadData( + filePath, + chunk, + bankGeneratorVersion, + languageId, + isCA, + itemIndex)); + + var expectedChunkSize = ChunkHeaderSize + hircChunk.HircItems.Sum(hirc => HircHeader.PrefixSize + hirc.SectionSize); + if (expectedChunkSize != hircChunk.ChunkHeader.ChunkSize) + throw new Exception("Error parsing HIRC in bnk, expected and actual not matching"); + + return hircChunk; + } + + public static List BuildIndex(long payloadOffset, uint chunkSize, ByteChunk chunk) + { + if (chunkSize < sizeof(uint)) + throw new InvalidDataException($"HIRC chunk is only {chunkSize} bytes."); + + var result = new List(); + var hircCount = chunk.ReadUInt32(); + + for (uint itemIndex = 0; itemIndex < hircCount; itemIndex++) { - var hircType = (AkBkHircType)chunk.PeakByte(); - - var start = chunk.Index; - try - { - var hircItem = factory.CreateInstance(hircType); - hircItem.IndexInFile = itemIndex; - hircItem.ByteIndexInFile = itemIndex; - hircItem.BnkFilePath = filePath; - hircItem.LanguageId = languageId; - hircItem.IsCAHircItem = isCAHircItem; - hircItem.ReadHirc(chunk); - hircChunk.HircItems.Add(hircItem); - } - catch (Exception e) - { - failedItems.Add(itemIndex); - chunk.Index = start; - - var unknownHirc = new UnknownHircItem + if (chunk.BytesLeft < HircHeader.Size) + throw new InvalidDataException($"HIRC item {itemIndex} does not contain a complete header."); + + var itemOffsetInChunk = chunk.Index; + var header = HircHeader.ReadData(chunk); + if (header.SectionSize < sizeof(uint)) + throw new InvalidDataException($"HIRC item {itemIndex} has an invalid section size of {header.SectionSize}."); + + var hircLength = checked(HircHeader.PrefixSize + header.SectionSize); + if (hircLength > int.MaxValue || hircLength - HircHeader.Size > chunk.BytesLeft) + throw new InvalidDataException($"HIRC item {itemIndex} extends beyond its HIRC chunk."); + + result.Add( + new HircIndexEntry { - ErrorMsg = e.Message, - ByteIndexInFile = itemIndex, - BnkFilePath = filePath - }; - unknownHirc.ReadHirc(chunk); - hircChunk.HircItems.Add(unknownHirc); - } + Header = header, + Offset = payloadOffset + itemOffsetInChunk, + Length = (int)hircLength, + Index = itemIndex + }); + chunk.Advance((int)(hircLength - HircHeader.Size)); } - return hircChunk; + if (chunk.BytesLeft != 0) + throw new InvalidDataException($"HIRC index left {chunk.BytesLeft} unread bytes in the chunk."); + + return result; } public static byte[] WriteData(HircChunk hircChunk, uint gameBankGeneratorVersion) diff --git a/Shared/GameFiles/Wwise/Hirc/HircHeader.cs b/Shared/GameFiles/Wwise/Hirc/HircHeader.cs new file mode 100644 index 000000000..759162e04 --- /dev/null +++ b/Shared/GameFiles/Wwise/Hirc/HircHeader.cs @@ -0,0 +1,39 @@ +using Shared.ByteParsing; +using Shared.GameFormats.Wwise.Enums; + +namespace Shared.GameFormats.Wwise.Hirc +{ + public class HircHeader + { + public const uint Size = 9; + public const uint PrefixSize = 5; + + public AkBkHircType HircType { get; set; } + public uint SectionSize { get; set; } + public uint Id { get; set; } + + public static HircHeader ReadData(ByteChunk chunk) + { + return new HircHeader + { + HircType = (AkBkHircType)chunk.ReadByte(), + SectionSize = chunk.ReadUInt32(), + Id = chunk.ReadUInt32() + }; + } + + public static byte[] WriteData(HircHeader header) + { + using var memStream = new MemoryStream(); + memStream.Write(ByteParsers.Byte.EncodeValue((byte)header.HircType, out _)); + memStream.Write(ByteParsers.UInt32.EncodeValue(header.SectionSize, out _)); + memStream.Write(ByteParsers.UInt32.EncodeValue(header.Id, out _)); + var byteArray = memStream.ToArray(); + + // Reload the object to ensure sanity + ReadData(new ByteChunk(byteArray)); + + return byteArray; + } + } +} diff --git a/Shared/GameFiles/Wwise/Hirc/HircIndexEntry.cs b/Shared/GameFiles/Wwise/Hirc/HircIndexEntry.cs new file mode 100644 index 000000000..5047bb9d4 --- /dev/null +++ b/Shared/GameFiles/Wwise/Hirc/HircIndexEntry.cs @@ -0,0 +1,10 @@ +namespace Shared.GameFormats.Wwise.Hirc +{ + public class HircIndexEntry + { + public required HircHeader Header { get; set; } + public long Offset { get; set; } + public int Length { get; set; } + public uint Index { get; set; } + } +} diff --git a/Shared/GameFiles/Wwise/Hirc/HircItem.cs b/Shared/GameFiles/Wwise/Hirc/HircItem.cs index b7ec95105..4b80ec689 100644 --- a/Shared/GameFiles/Wwise/Hirc/HircItem.cs +++ b/Shared/GameFiles/Wwise/Hirc/HircItem.cs @@ -7,18 +7,65 @@ public abstract class HircItem { readonly ILogger _logger = Logging.Create(); - public static uint HircHeaderSize { get => 5; } public string BnkFilePath { get; set; } = "Not Set"; - public bool IsCAHircItem { get; set; } + public bool IsCA { get; set; } public uint LanguageId { get; set; } public uint ByteIndexInFile { get; set; } public uint IndexInFile { get; set; } public bool HasError { get; set; } = true; public bool IsTarget { get; set; } public List? HircChildren { get; set; } - public AkBkHircType HircType { get; set; } - public uint SectionSize { get; set; } - public uint Id { get; set; } + public HircHeader Header { get; set; } = new HircHeader(); + public AkBkHircType HircType { get => Header.HircType; set => Header.HircType = value; } + public uint SectionSize { get => Header.SectionSize; set => Header.SectionSize = value; } + public uint Id { get => Header.Id; set => Header.Id = value; } + + public static HircItem ReadData( + string filePath, + ByteChunk chunk, + uint bankGeneratorVersion, + uint languageId, + bool isCA, + uint itemIndex, + int? expectedLength = null) + { + if (expectedLength.HasValue && expectedLength.Value < HircHeader.Size) + throw new InvalidDataException($"HIRC item {itemIndex} is only {expectedLength.Value} bytes."); + + var itemStartIndex = chunk.Index; + var hircType = (AkBkHircType)chunk.PeakByte(); + var factory = HircFactory.CreateFactory(bankGeneratorVersion); + HircItem hircItem; + + try + { + hircItem = factory.CreateInstance(hircType); + hircItem.IndexInFile = itemIndex; + hircItem.ByteIndexInFile = itemIndex; + hircItem.BnkFilePath = filePath; + hircItem.LanguageId = languageId; + hircItem.IsCA = isCA; + hircItem.ReadHirc(chunk); + } + catch (Exception exception) + { + chunk.Index = itemStartIndex; + + hircItem = new UnknownHircItem + { + ErrorMsg = exception.Message, + ByteIndexInFile = itemIndex, + BnkFilePath = filePath + }; + hircItem.ReadHirc(chunk); + } + + var bytesRead = chunk.Index - itemStartIndex; + if (expectedLength.HasValue && bytesRead != expectedLength.Value) + throw new InvalidDataException($"HIRC item {itemIndex} expected {expectedLength.Value} bytes but read {bytesRead}."); + + return hircItem; + } public void ReadHirc(ByteChunk chunk) { @@ -27,13 +74,11 @@ public void ReadHirc(ByteChunk chunk) var indexBeforeRead = chunk.Index; ByteIndexInFile = (uint)indexBeforeRead; - HircType = (AkBkHircType)chunk.ReadByte(); - SectionSize = chunk.ReadUInt32(); - Id = chunk.ReadUInt32(); + Header = HircHeader.ReadData(chunk); ReadData(chunk); var currentIndex = chunk.Index; - var indexAfterRead = (int)(indexBeforeRead + HircHeaderSize + SectionSize); + var indexAfterRead = (int)(indexBeforeRead + HircHeader.PrefixSize + SectionSize); chunk.Index = indexAfterRead; HasError = false; } @@ -48,9 +93,7 @@ public void ReadHirc(ByteChunk chunk) protected MemoryStream WriteHeader() { var memStream = new MemoryStream(); - memStream.Write(ByteParsers.Byte.EncodeValue((byte)HircType, out _)); - memStream.Write(ByteParsers.UInt32.EncodeValue(SectionSize, out _)); - memStream.Write(ByteParsers.UInt32.EncodeValue(Id, out _)); + memStream.Write(HircHeader.WriteData(Header)); return memStream; } diff --git a/Shared/GameFiles/Wwise/ParsedBnkFile.cs b/Shared/GameFiles/Wwise/ParsedBnkFile.cs deleted file mode 100644 index 515355d47..000000000 --- a/Shared/GameFiles/Wwise/ParsedBnkFile.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Shared.ByteParsing; -using Shared.GameFormats.Wwise.Bkhd; -using Shared.GameFormats.Wwise.Didx; -using Shared.GameFormats.Wwise.Hirc; - -namespace Shared.GameFormats.Wwise -{ - public class ParsedBnkFile - { - public BkhdChunk BkhdChunk { get; internal set; } = new BkhdChunk(); - public HircChunk HircChunk { get; internal set; } = new HircChunk(); - public DidxChunk DidxChunk { get; internal set; } - public ByteChunk DataChunk { get; internal set; } - } -} diff --git a/Shared/GameFiles/Wwise/Stid/StidChunk.cs b/Shared/GameFiles/Wwise/Stid/StidChunk.cs index cce601d24..8b74bf206 100644 --- a/Shared/GameFiles/Wwise/Stid/StidChunk.cs +++ b/Shared/GameFiles/Wwise/Stid/StidChunk.cs @@ -4,11 +4,14 @@ namespace Shared.GameFormats.Wwise.Stid { public class StidChunk { - public static ChunkHeader ReadData(string fileName, ByteChunk chunk) + public ChunkHeader ChunkHeader { get; set; } = new ChunkHeader(); + public ByteChunk Data { get; set; } = new ByteChunk([]); + + public static StidChunk ReadData(string fileName, ByteChunk chunk) { - var chunkHeader = ChunkHeader.ReadData(chunk); - chunk.Index += (int)chunkHeader.ChunkSize; - return chunkHeader; + var stidChunk = new StidChunk { ChunkHeader = ChunkHeader.ReadData(chunk) }; + stidChunk.Data = chunk.CreateSub((int)stidChunk.ChunkHeader.ChunkSize); + return stidChunk; } } } diff --git a/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs b/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs index 6e3a5f767..4778ae12b 100644 --- a/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs +++ b/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs @@ -33,7 +33,7 @@ public class DirectoryHelper public static string ReportsDirectory { get { return ApplicationDirectory + "\\Reports"; } } public static string Applications { get { return ApplicationDirectory + "\\Applications"; } } public static string Temp { get { return ApplicationDirectory + "\\Temp"; } } - public static string AudioDirectory { get { return Temp + "\\Audio"; } } + public static string AudioTempDirectory { get { return Temp + "\\Audio"; } } public static string UpdateDirectory { get { return Temp + "\\Update"; } } public static string CacheDirectory { get => ApplicationDirectory + "\\Cache"; } public static string AnimationIndexMappingDirectory { get { return ApplicationDirectory + "\\Animation\\BoneIndexMapping"; } } diff --git a/Shared/SharedCore/Shared.Core/Misc/MemoryOptimiser.cs b/Shared/SharedCore/Shared.Core/Misc/MemoryOptimiser.cs deleted file mode 100644 index b72b2c36b..000000000 --- a/Shared/SharedCore/Shared.Core/Misc/MemoryOptimiser.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Diagnostics; -using System.Runtime; -using System.Runtime.InteropServices; - -namespace Shared.Core.Misc -{ - public class MemoryOptimiser - { - static readonly ILogger s_logger = Logging.Create(); - - public static void Optimise() - { - - RunGarbageCollection(); - - using var process = Process.GetCurrentProcess(); - try - { - EmptyWorkingSet(process.Handle); - } - catch { } - - } - - [DllImport("psapi.dll")] private static extern bool EmptyWorkingSet(nint hProcess); - - public static void RunGarbageCollection() - { - GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; - GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true, compacting: true); - } - - public static void LogMemory(string label) - { - var process = Process.GetCurrentProcess(); - var memoryInfo = GC.GetGCMemoryInfo(); - var managed = GC.GetTotalMemory(forceFullCollection: true); - s_logger.Here().Information($"{label} | Managed={managed / 1024 / 1024} MB, HeapSize={memoryInfo.HeapSizeBytes / 1024 / 1024} MB, " + - $"TotalCommitted={memoryInfo.TotalCommittedBytes / 1024 / 1024} MB, WorkingSet={process.WorkingSet64 / 1024 / 1024} MB"); - } - } -} From c631693ba9b79912f1ce240c745e498f82fa0169 Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Sun, 2 Aug 2026 21:49:49 +0100 Subject: [PATCH 2/6] Added breadth first search to hirc graph --- .../HircExploration/HircTreeBaseParser.cs | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/Editors/Audio/Shared/Wwise/HircExploration/HircTreeBaseParser.cs b/Editors/Audio/Shared/Wwise/HircExploration/HircTreeBaseParser.cs index d955f55ef..d8cf3ef35 100644 --- a/Editors/Audio/Shared/Wwise/HircExploration/HircTreeBaseParser.cs +++ b/Editors/Audio/Shared/Wwise/HircExploration/HircTreeBaseParser.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Editors.Audio.AudioExplorer; +using Editors.Audio.AudioExplorer; using Editors.Audio.Shared.Storage; using Shared.GameFormats.Wwise.Enums; using Shared.GameFormats.Wwise.Hirc; @@ -10,13 +7,19 @@ namespace Editors.Audio.Shared.Wwise.HircExploration { public abstract class HircTreeBaseParser(IAudioRepository audioRepository) { + private sealed record PendingHircNode(uint HircId, HircTreeNode Parent); + public readonly IAudioRepository AudioRepository = audioRepository; public readonly Dictionary> HircProcessChildMap = []; + private readonly List _breadthFirstSearchFrontier = []; + public HircTreeNode BuildHierarchy(HircItem item) { var root = new HircTreeNode(); ProcessHircObject(item, root); + RunBreadthFirstSearch(); + var actualRoot = root.Children.FirstOrDefault(); actualRoot.Parent = null; root.Children = null; @@ -26,17 +29,17 @@ public HircTreeNode BuildHierarchy(HircItem item) public List BuildHierarchyAsFlatList(HircItem item) { var rootNode = BuildHierarchy(item); - var flatList = GetHircParents(rootNode); + var flatList = RunDepthFirstSearch(rootNode); return flatList; } - private static List GetHircParents(HircTreeNode root) + private static List RunDepthFirstSearch(HircTreeNode root) { var childData = new List(); if (root.Children != null) { foreach (var child in root.Children) - childData.AddRange(GetHircParents(child)); + childData.AddRange(RunDepthFirstSearch(child)); } childData.Add(root); @@ -59,12 +62,7 @@ protected void ProcessNext(uint hircId, HircTreeNode parent) if (hircId == 0) return; - var hircs = AudioRepository.GetHircs(hircId); - var hirc = hircs.FirstOrDefault(); - if (hirc == null) - parent.Children.Add(new HircTreeNode() { DisplayName = $"Error: Unable to find Hirc with ID {hircId}" }); - else - ProcessHircObject(hirc, parent); + _breadthFirstSearchFrontier.Add(new PendingHircNode(hircId, parent)); } protected void ProcessNext(List ids, HircTreeNode parent) @@ -73,6 +71,26 @@ protected void ProcessNext(List ids, HircTreeNode parent) ProcessNext(id, parent); } + private void RunBreadthFirstSearch() + { + while (_breadthFirstSearchFrontier.Count != 0) + { + var currentDepth = new List(_breadthFirstSearchFrontier); + _breadthFirstSearchFrontier.Clear(); + + var hircIdsAtDepth = currentDepth.Select(pendingNode => pendingNode.HircId).Distinct().ToList(); + var hircsById = AudioRepository.GetHircs(hircIdsAtDepth); + + foreach (var pendingNode in currentDepth) + { + if (hircsById.TryGetValue(pendingNode.HircId, out var hircs) && hircs.Count != 0) + ProcessHircObject(hircs[0], pendingNode.Parent); + else + pendingNode.Parent.Children.Add(new HircTreeNode() { DisplayName = $"Error: Unable to find Hirc with ID {pendingNode.HircId}" }); + } + } + } + protected virtual string GetDisplayId(uint id, string fileName, bool hidenNameIfMissing) { var name = AudioRepository.GetNameFromId(id, out var found); From c443856a413dd23de571f174cf265b5e5ee5522e Mon Sep 17 00:00:00 2001 From: Pear-231 <61670316+Pear-231@users.noreply.github.com> Date: Mon, 3 Aug 2026 00:34:00 +0100 Subject: [PATCH 3/6] Added hirc tree lazy loading --- .../AudioExplorer/AudioExplorerView.xaml | 7 +- .../AudioExplorer/AudioExplorerView.xaml.cs | 9 +++ .../AudioExplorer/AudioExplorerViewModel.cs | 59 ++++++-------- Editors/Audio/AudioExplorer/HircTreeNode.cs | 25 +++++- .../HircExploration/HircTreeBaseParser.cs | 78 ++++++++++++++++++- .../HircExploration/HircTreeChildrenParser.cs | 8 +- 6 files changed, 136 insertions(+), 50 deletions(-) diff --git a/Editors/Audio/AudioExplorer/AudioExplorerView.xaml b/Editors/Audio/AudioExplorer/AudioExplorerView.xaml index 6f1e4b44c..d343ac27c 100644 --- a/Editors/Audio/AudioExplorer/AudioExplorerView.xaml +++ b/Editors/Audio/AudioExplorer/AudioExplorerView.xaml @@ -197,14 +197,15 @@ ItemsSource="{Binding TreeList, UpdateSourceTrigger=PropertyChanged}" VirtualizingPanel.IsVirtualizing="True" VirtualizingPanel.VirtualizationMode="Recycling" - VirtualizingPanel.IsVirtualizingWhenGrouping="True"> + VirtualizingPanel.IsVirtualizingWhenGrouping="True" + PreviewMouseDoubleClick="OnNodeDoubleClick">