tools/ue_audio_extract: UE3 XMA2 audio extraction, re-encode and repack pipeline

tools/ue_audio_extract/ExtractManual.cs

@@ -0,0 +1,45 @@
+using System;
+using System.IO;
+using UELib;
+using UELib.Core;
+
+/// <summary>
+/// Inspects a (decompressed) Army of Two package and lists every
+/// USoundNodeWave export with the raw BulkData layout pointers.
+/// It reads the bulk bytes directly from the file instead of calling
+/// UBulkData.LoadData: UELib's load path rewinds the stream based on its
+/// table offsets in ways that produce stale/duplicated buffers in this
+/// build, while StorageOffset/StorageSize in the export deserialization
+/// are exact.
+/// </summary>
+class ExtractManual {
+    static void Main(string[] args) {
+        // args: <assembled.pkg> <outDir> <outCsv>
+        string outDir = args[1];
+        Directory.CreateDirectory(outDir);
+        string pkgBase = Path.GetFileNameWithoutExtension(args[0]);
+        byte[] fileBytes = File.ReadAllBytes(args[0]);
+        using (var pkg = UnrealLoader.LoadFullPackage(args[0], FileAccess.Read)) {
+            var t = typeof(USoundNodeWave);
+            var fX360 = t.GetField("CompressedXbox360Data");
+            using (var csv = new StreamWriter(args[2])) {
+                csv.WriteLine("wave,ElementCount,StorageOffset,StorageSize,bulk_file");
+                foreach (var exp in pkg.Exports) {
+                    if (exp.Class == null || exp.Class.ObjectName.ToString() != "SoundNodeWave") continue;
+                    var wave = pkg.IndexToObject<USoundNodeWave>(exp.Index + 1);
+                    if (wave == null) continue;
+                    try { wave.Load(); } catch { }
+                    var bd = (UBulkData<byte>)fX360.GetValue(wave);
+                    long off = bd.StorageOffset, sz = bd.StorageSize;
+                    if (off <= 0 || sz <= 0 || off + sz > fileBytes.Length) continue;
+                    string nm = wave.Name.ToString().Replace('/', '_') + "__" + pkgBase + ".bulk";
+                    byte[] outBuf = new byte[sz];
+                    Array.Copy(fileBytes, off, outBuf, 0, sz);
+                    File.WriteAllBytes(Path.Combine(outDir, nm), outBuf);
+                    csv.WriteLine(wave.Name + "," + bd.ElementCount + "," + off + "," + sz + "," + nm);
+                    Console.WriteLine("WAVE " + wave.Name + " elems=" + bd.ElementCount + " off=" + off + " size=" + sz);
+                }
+            }
+        }
+    }
+}

tools/ue_audio_extract/README.md

@@ -0,0 +1,104 @@
+# UE3 audio extraction / re-encode / repack
+
+Tools to pull the `SoundNodeWave` audio of Army of Two (Xbox 360, UE3 v445)
+out of `assets/AO2Game/CookedXenon/*.xxx` packages, decode the XMA2 payload,
+and splice replacement audio (e.g. a pt-BR dub) back into the same bulk
+slots, so the game plays new text-to-speech in place of the original English
+voice.
+
+## Why this exists
+
+- UE3 Xbox-360 packages here are **LZO chunk compressed** (`CompressionFlags=2`).
+  UE Explorer / UELib returns early from `Deserialize` when compressed chunks
+  exist ("We can't continue without decompressing"), so Names/Exports/Imports
+  come back as 0 entries. That's UELib behavior, not an API mistake.
+- The sound bulk element of each wave holds a **108-byte custom header**
+  followed by the raw XMA2 payload (`CompressedXbox360Data`).
+
+## Layout of the compressed package (445/79, big-endian)
+
+```
+[0:4]   Signature (0x9e2a83c1)
+[4:8]   Version / Licensee
+[8:12]  HeaderSize
+...     table offsets: NameCount@101, ExportCount@Imports in ExportOffset, ...
+[~89]   CompressionFlags = 2, chunk count (e.g. 7), entries:
+        UncompressedOffset, CompressedOffset, UncompressedSize,
+        CompressedSize (UELib ReadArray order; see ue_package.py)
+Data (uncompressed logical start): offset 101 (name table)
+```
+
+**CompressedChunkHeader** at CompressedOffset: Tag=UE sig (4B), ChunkSize=0x20000
+(4B), Summary (CompSize, UncompSize), N block pairs (compSize, uncompSize);
+compressed payload starts right after (16 + 8*N bytes). Each block is an
+independent LZO1X stream — decompression with the game's own decoder
+(see `aot/src/cyrillic_fonts.cpp`; we include a verbatim copy in `lzodec.cpp`).
+
+## Bulk element layout (SoundNodeWave audio, 56 + 4*N bytes of header)
+
+```
+u32BE headerLen      (44)
+u32BE payloadOffset  (52)
+u32BE dataSize       — the XMA2 stream length
+u32BE magic          (0x04010000)
+u32BE 384            (encoder delay/rounding constant)
+u32BE numSamples(A)  (~16384/sample-ish; 129408 in the pilot)
+u32BE ~24000         sample rate (UE3 Xbox VO 24 kHz mono)
+u32BE 0x10D60000     constant
+u32BE ~6593
+u32BE 2048           packet size (fixed)
+u32BE numSamples     (decodable samples)
+u32BE numSamples-518
+u32BE nPackets
+u32BE 0x01000000     marker
+nPackets * u32BE     cumulative samples per packet (seek table; last == numSamples)
+```
+
+Audio payload starts at `StorageOffset + 56 + 4*nPackets`, length `dataSize`,
+multiple of 2048 (13 blocks for the Clyde pilot = 26624 bytes).
+Sample rate is **24000 Hz mono** (validated by pitch, RMS and decode).
+
+## XMA2 packets
+
+2048-byte blocks: big-endian header dword = frames starting in the packet
+(bits 26..31), first-frame bit offset (15 bits), metadata (3), skip count
+(8). Unused tail of a block is 0xFF padding. FFmpeg 9.x (`-f wav` with
+`WAVE_FORMAT_XMA2`, or vgmstream with a TXTH) decodes this, as does the
+game's own XAudio2/XMA decoder (`sdk/src/audio/xma_context.cpp`).
+
+## Components
+
+| File | Purpose |
+| --- | --- |
+| `lzodec.cpp` | One-shot LZXP/LZO block decoder CLI (verbatim LZO1X from the port's fonts path). |
+| `ExtractManual.cs` + `chunk_dump.cs` | C# UELib dumpers (need `Eliot.UELib.dll`). They parse the summary/chunk table and list (wave, off, size) from the deserialized properties rather than calling `UBulkData.LoadData` (which translates offsets through the compressed-chunk stream and yields wrong buffers). |
+| `ue_package.py` | Pure-Python helpers: chunk Table via UELib, decompress via `lzodec`, assemble plain packages. |
+| `ue_audio_extract.py` | CLI: `.xxx` → per-wave `.xma2` + TXTH stubs (vgmstream decodes them). |
+| `xma2_reencode.py` | CLI: encode WAVs → xma2 packets (external `xma2enc_cli.exe`, birabittoh/xma2enc), rebuild the bulk header (dataSize, numSamples, packet count, seek table), splice at the wave's StorageOffset, verify the export still parses. |
+
+## Rebuild notes
+
+```
+REM fullscreen build on Windows (MSVC)
+cl lzodec.cpp /EHsc /O2 /Fe:lzodec.exe
+
+REM .NET tools (csc.exe via the VS Build Tools toolchain)
+csc.exe /nologo /out:chunk_dump.exe /r:Eliot.UELib.dll chunk_dump.cs
+csc.exe /nologo /out:ExtractManual.exe /r:Eliot.UELib.dll ExtractManual.cs
+```
+
+`xma2enc_cli.exe` is built from https://github.com/birabittoh/xma2enc
+(WMA-Pro bitstream + XMA2 packet layer). Repack bitrates are auto-tuned so
+the re-encoded blob fits the original bulk StorageSize (the export record
+and the package's byte budget don't change).
+
+## Validated results
+
+- Full game inventory: 140 `*_LOC_int.xxx` packages → 9753 `SoundNodeWave`
+  exports → decoded to 9753 WAVs (RMS ~5-7k, voice-grade, 24000 Hz).
+- 902 mission lines matched 1:1 nome→linha `*_sound_vo_main.int` (en+pt) and
+  cloned in pt-BR (`language="Portuguese"`, voice clones from each line's
+  English reference); pilot of 5 lines approved by the author (intonation
+  preserved, Portuguese accent).
+- Repack of the market package: boot into the militarybase with the repacked
+  `01_map_market_sound_LOC_int.xxx`, log free of XMA/audio errors.

tools/ue_audio_extract/lzodec.cpp

@@ -0,0 +1,240 @@
+// lzodec.cpp — UE3/Xenon LZXP/LZO chunk decompressor (one-shot CLI tool).
+//
+// Reads one compressed chunk (or the whole stream of chunks in a package)
+// from a CookedXenon *.xxx file and writes the decompressed logical bytes
+// to stdout/file. The block decoder is the LZO1X implementation shipped in
+// this project's own renderer/fonts path (`aot/src/cyrillic_fonts.cpp`),
+// derived verbatim from the port codebase (same decoder used for the
+// Russian fonts' page blobs). UE3 packages in this game are compressed with
+// LZO chunks (CompressionFlags=2): UELib stops after the summary, so this
+// tool is required before UELib can see the tables.
+//
+// Block layout inside a compressed chunk:
+//   u32 Tag (UE signature, big-endian)
+//   u32 ChunkSize (block uncompressed size, 0x20000)
+//   u32 CompTotalBytes, UncompTotalBytes
+//   blocks[]: pairs (compSize, uncompSize), big-endian
+//   compressed data follows (each block is an independent LZO1X stream)
+//
+// usage: lzodec <in.xxx> <out.bin> [<HdrOff>]
+//   HdrOff: offset of the chunk header; if omitted, the first UE signature
+//   beyond the summary is used.
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstdint>
+#include <cstring>
+#include <vector>
+
+static uint32_t U(const uint8_t* d, long off) {
+  return ((uint32_t)d[off] << 24) | ((uint32_t)d[off + 1] << 16) | ((uint32_t)d[off + 2] << 8) | d[off + 3];
+}
+
+// LZO1X block decoder (verbatim from aot/src/cyrillic_fonts.cpp).
+static bool Lzo1xDecompress(const uint8_t *in, size_t in_len, uint8_t *out, size_t out_len)
+{
+  const uint8_t *ip = in, *ip_end = in + in_len;
+  uint8_t *op = out, *op_end = out + out_len;
+  size_t t = 0;
+  const uint8_t *m_pos = nullptr;
+  auto need_in = [&](size_t n) { return size_t(ip_end - ip) >= n; };
+  auto need_out = [&](size_t n) { return size_t(op_end - op) >= n; };
+  auto long_length = [&](size_t base) -> bool {
+    while (need_in(1) && *ip == 0)
+    {
+      t += 255;
+      ++ip;
+    }
+    if (!need_in(1))
+      return false;
+    t += base + *ip++;
+    return true;
+  };
+  auto copy_literals = [&](size_t n) -> bool {
+    if (!need_in(n) || !need_out(n))
+      return false;
+    std::memcpy(op, ip, n);
+    op += n;
+    ip += n;
+    return true;
+  };
+  auto copy_match = [&](size_t n) -> bool {
+    if (m_pos < out || m_pos >= op || !need_out(n))
+      return false;
+    for (size_t i = 0; i < n; ++i)
+      *op++ = *m_pos++;
+    return true;
+  };
+
+  if (!need_in(1))
+    return false;
+  bool skip_to_first = false, skip_to_next = false;
+  if (*ip > 17)
+  {
+    t = *ip++ - 17;
+    if (t < 4)
+    {
+      skip_to_next = true;
+    }
+    else
+    {
+      if (!copy_literals(t))
+        return false;
+      skip_to_first = true;
+    }
+  }
+  for (;;)
+  {
+    if (!skip_to_first && !skip_to_next)
+    {
+      if (!need_in(1))
+        return false;
+      t = *ip++;
+      if (t < 16)
+      {
+        if (t == 0 && !long_length(15))
+          return false;
+        if (!copy_literals(t + 3))
+          return false;
+        skip_to_first = true;
+      }
+    }
+    if (skip_to_first)
+    {
+      skip_to_first = false;
+      if (!need_in(1))
+        return false;
+      t = *ip++;
+      if (t < 16)
+      {
+        if (!need_in(1))
+          return false;
+        m_pos = op - (1 + 0x0800) - (t >> 2) - (size_t(*ip++) << 2);
+        if (!copy_match(3))
+          return false;
+        goto match_done;
+      }
+    }
+    for (;;)
+    {
+      if (skip_to_next)
+      {
+        skip_to_next = false;
+        goto match_next;
+      }
+      if (t >= 64)
+      {
+        if (!need_in(1))
+          return false;
+        m_pos = op - 1 - ((t >> 2) & 7) - (size_t(*ip++) << 3);
+        t = (t >> 5) - 1;
+        if (!copy_match(t + 2))
+          return false;
+      }
+      else if (t >= 32)
+      {
+        t &= 31;
+        if (t == 0 && !long_length(31))
+          return false;
+        if (!need_in(2))
+          return false;
+        m_pos = op - 1 - ((ip[0] >> 2) + (size_t(ip[1]) << 6));
+        ip += 2;
+        if (!copy_match(t + 2))
+          return false;
+      }
+      else if (t >= 16)
+      {
+        m_pos = op - ((t & 8) << 11);
+        t &= 7;
+        if (t == 0 && !long_length(7))
+          return false;
+        if (!need_in(2))
+          return false;
+        m_pos -= (ip[0] >> 2) + (size_t(ip[1]) << 6);
+        ip += 2;
+        if (m_pos == op)
+          return op == op_end;
+        m_pos -= 0x4000;
+        if (!copy_match(t + 2))
+          return false;
+      }
+      else
+      {
+        if (!need_in(1))
+          return false;
+        m_pos = op - 1 - (t >> 2) - (size_t(*ip++) << 2);
+        if (!copy_match(2))
+          return false;
+      }
+    match_done:
+      t = ip[-2] & 3;
+      if (t == 0)
+        break;
+    match_next:
+      if (!copy_literals(t))
+        return false;
+      if (!need_in(1))
+        return false;
+      t = *ip++;
+    }
+  }
+}
+
+int main(int argc, char** argv)
+{
+  if (argc < 3) {
+    std::fprintf(stderr, "usage: lzodec <in.xxx> <out.bin> [<HdrOff>]\n");
+    return 2;
+  }
+  const char* inpath = argv[1];
+  const char* outpath = argv[2];
+  long hdr = argc > 3 ? atol(argv[3]) : -1;
+
+  FILE* fi = std::fopen(inpath, "rb");
+  if (!fi) { std::perror("open"); return 1; }
+  std::fseek(fi, 0, SEEK_END);
+  long fsize = std::ftell(fi);
+  std::fseek(fi, 0, SEEK_SET);
+  std::vector<uint8_t> d(fsize);
+  if (std::fread(d.data(), 1, fsize, fi) != (size_t)fsize) return 1;
+  std::fclose(fi);
+
+  if (hdr < 0) {
+    hdr = -1;
+    for (long i = 64; i + 4 <= fsize; ++i) {
+      if (d[i] == 0x9e && d[i + 1] == 0x2a && d[i + 2] == 0x83 && d[i + 3] == 0xc1) { hdr = i; break; }
+    }
+    if (hdr < 0) { std::fprintf(stderr, "no chunk header\n"); return 1; }
+  }
+  std::fprintf(stderr, "hdr=%ld\n", hdr);
+  uint32_t chunkSize = U(d.data(), hdr + 4);
+  uint32_t total = U(d.data(), hdr + 12);
+  std::fprintf(stderr, "chunkSize=%u totalUncomp=%u\n", chunkSize, total);
+
+  std::vector<std::pair<uint32_t, uint32_t>> blocks;
+  long hb = hdr + 16;
+  for (uint32_t sum = 0; sum < total && blocks.size() < 256; hb += 8) {
+    uint32_t cs = U(d.data(), hb), us = U(d.data(), hb + 4);
+    blocks.emplace_back(cs, us);
+    sum += us;
+  }
+  std::fprintf(stderr, "blocks=%zu\n", blocks.size());
+
+  std::vector<uint8_t> out(total);
+  long at = hb, written = 0;
+  for (size_t b = 0; b < blocks.size(); ++b) {
+    uint32_t cs = blocks[b].first, us = blocks[b].second;
+    bool ok = Lzo1xDecompress(d.data() + at, cs, out.data() + written, us);
+    std::fprintf(stderr, "block %zu comp=%u uncomp=%u ok=%d\n", b, cs, us, ok ? 1 : 0);
+    if (!ok) return 1;
+    at += cs;
+    written += us;
+  }
+  FILE* fo = std::fopen(outpath, "wb");
+  if (!fo) { std::perror("out"); return 1; }
+  std::fwrite(out.data(), 1, out.size(), fo);
+  std::fclose(fo);
+  std::fprintf(stderr, "OK wrote %zu bytes\n", out.size());
+  return 0;
+}