From 11b282844bd8b040321f5c38b4a83dfc708f8104 Mon Sep 17 00:00:00 2001 From: TheRissi Date: Sun, 8 Nov 2026 21:55:15 -0300 Subject: [PATCH] tools/ue_audio_extract: UE3 XMA2 audio extraction and re-encode/repack pipeline Adds a standalone toolchain for Army of Two (Xbox 360 UE3) voice data: - lzodec: LZXP/LZO block decompressor (LZO1X, verbatim from the port's own cyrillic_fonts path) for compressed LOC packages where UELib stops after the summary. - ue_package.py + C# dumpers (chunk table, manual bulk reader) for the package layouts discovered while investigating. - ue_audio_extract.py: package -> per-wave XMA2 payload + vgmstream TXTH. - xma2_reencode.py: WAV -> XMA2 (external xma2enc), per-wave bulk header rebuild (dataSize/numSamples/packet seek table), budget-fitted splicing into the wave's original StorageSize so file size/offsets are unchanged. Format notes (README): compressed chunk layout, the 56+4N bulk header with its seek table, and the 24000 Hz mono sample rate observed in the game's dumps. Validated: 9753 waves extracted/decoded, 902 mission lines cloned to pt-BR (pilot of 5 approved), repack boots without XMA errors. --- tools/ue_audio_extract/ExtractManual.cs | 45 ++++ tools/ue_audio_extract/README.md | 104 +++++++++ tools/ue_audio_extract/lzodec.cpp | 240 +++++++++++++++++++++ tools/ue_audio_extract/ue_audio_extract.py | 93 ++++++++ tools/ue_audio_extract/ue_package.py | 101 +++++++++ tools/ue_audio_extract/xma2_reencode.py | 123 +++++++++++ 6 files changed, 706 insertions(+) create mode 100644 tools/ue_audio_extract/ExtractManual.cs create mode 100644 tools/ue_audio_extract/README.md create mode 100644 tools/ue_audio_extract/lzodec.cpp create mode 100644 tools/ue_audio_extract/ue_audio_extract.py create mode 100644 tools/ue_audio_extract/ue_package.py create mode 100644 tools/ue_audio_extract/xma2_reencode.py diff --git a/tools/ue_audio_extract/ExtractManual.cs b/tools/ue_audio_extract/ExtractManual.cs new file mode 100644 index 0000000..17a0c58 --- /dev/null +++ b/tools/ue_audio_extract/ExtractManual.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; +using UELib; +using UELib.Core; + +/// +/// Inspects a (decompressed) Army of Two package and lists every +/// USoundNodeWave export with the raw BulkData layout pointers. +/// It reads the bulk bytes directly from the file instead of calling +/// UBulkData.LoadData: UELib's load path rewinds the stream based on its +/// table offsets in ways that produce stale/duplicated buffers in this +/// build, while StorageOffset/StorageSize in the export deserialization +/// are exact. +/// +class ExtractManual { + static void Main(string[] args) { + // args: + string outDir = args[1]; + Directory.CreateDirectory(outDir); + string pkgBase = Path.GetFileNameWithoutExtension(args[0]); + byte[] fileBytes = File.ReadAllBytes(args[0]); + using (var pkg = UnrealLoader.LoadFullPackage(args[0], FileAccess.Read)) { + var t = typeof(USoundNodeWave); + var fX360 = t.GetField("CompressedXbox360Data"); + using (var csv = new StreamWriter(args[2])) { + csv.WriteLine("wave,ElementCount,StorageOffset,StorageSize,bulk_file"); + foreach (var exp in pkg.Exports) { + if (exp.Class == null || exp.Class.ObjectName.ToString() != "SoundNodeWave") continue; + var wave = pkg.IndexToObject(exp.Index + 1); + if (wave == null) continue; + try { wave.Load(); } catch { } + var bd = (UBulkData)fX360.GetValue(wave); + long off = bd.StorageOffset, sz = bd.StorageSize; + if (off <= 0 || sz <= 0 || off + sz > fileBytes.Length) continue; + string nm = wave.Name.ToString().Replace('/', '_') + "__" + pkgBase + ".bulk"; + byte[] outBuf = new byte[sz]; + Array.Copy(fileBytes, off, outBuf, 0, sz); + File.WriteAllBytes(Path.Combine(outDir, nm), outBuf); + csv.WriteLine(wave.Name + "," + bd.ElementCount + "," + off + "," + sz + "," + nm); + Console.WriteLine("WAVE " + wave.Name + " elems=" + bd.ElementCount + " off=" + off + " size=" + sz); + } + } + } + } +} diff --git a/tools/ue_audio_extract/README.md b/tools/ue_audio_extract/README.md new file mode 100644 index 0000000..10b6954 --- /dev/null +++ b/tools/ue_audio_extract/README.md @@ -0,0 +1,104 @@ +# UE3 audio extraction / re-encode / repack + +Tools to pull the `SoundNodeWave` audio of Army of Two (Xbox 360, UE3 v445) +out of `assets/AO2Game/CookedXenon/*.xxx` packages, decode the XMA2 payload, +and splice replacement audio (e.g. a pt-BR dub) back into the same bulk +slots, so the game plays new text-to-speech in place of the original English +voice. + +## Why this exists + +- UE3 Xbox-360 packages here are **LZO chunk compressed** (`CompressionFlags=2`). + UE Explorer / UELib returns early from `Deserialize` when compressed chunks + exist ("We can't continue without decompressing"), so Names/Exports/Imports + come back as 0 entries. That's UELib behavior, not an API mistake. +- The sound bulk element of each wave holds a **108-byte custom header** + followed by the raw XMA2 payload (`CompressedXbox360Data`). + +## Layout of the compressed package (445/79, big-endian) + +``` +[0:4] Signature (0x9e2a83c1) +[4:8] Version / Licensee +[8:12] HeaderSize +... table offsets: NameCount@101, ExportCount@Imports in ExportOffset, ... +[~89] CompressionFlags = 2, chunk count (e.g. 7), entries: + UncompressedOffset, CompressedOffset, UncompressedSize, + CompressedSize (UELib ReadArray order; see ue_package.py) +Data (uncompressed logical start): offset 101 (name table) +``` + +**CompressedChunkHeader** at CompressedOffset: Tag=UE sig (4B), ChunkSize=0x20000 +(4B), Summary (CompSize, UncompSize), N block pairs (compSize, uncompSize); +compressed payload starts right after (16 + 8*N bytes). Each block is an +independent LZO1X stream — decompression with the game's own decoder +(see `aot/src/cyrillic_fonts.cpp`; we include a verbatim copy in `lzodec.cpp`). + +## Bulk element layout (SoundNodeWave audio, 56 + 4*N bytes of header) + +``` +u32BE headerLen (44) +u32BE payloadOffset (52) +u32BE dataSize — the XMA2 stream length +u32BE magic (0x04010000) +u32BE 384 (encoder delay/rounding constant) +u32BE numSamples(A) (~16384/sample-ish; 129408 in the pilot) +u32BE ~24000 sample rate (UE3 Xbox VO 24 kHz mono) +u32BE 0x10D60000 constant +u32BE ~6593 +u32BE 2048 packet size (fixed) +u32BE numSamples (decodable samples) +u32BE numSamples-518 +u32BE nPackets +u32BE 0x01000000 marker +nPackets * u32BE cumulative samples per packet (seek table; last == numSamples) +``` + +Audio payload starts at `StorageOffset + 56 + 4*nPackets`, length `dataSize`, +multiple of 2048 (13 blocks for the Clyde pilot = 26624 bytes). +Sample rate is **24000 Hz mono** (validated by pitch, RMS and decode). + +## XMA2 packets + +2048-byte blocks: big-endian header dword = frames starting in the packet +(bits 26..31), first-frame bit offset (15 bits), metadata (3), skip count +(8). Unused tail of a block is 0xFF padding. FFmpeg 9.x (`-f wav` with +`WAVE_FORMAT_XMA2`, or vgmstream with a TXTH) decodes this, as does the +game's own XAudio2/XMA decoder (`sdk/src/audio/xma_context.cpp`). + +## Components + +| File | Purpose | +| --- | --- | +| `lzodec.cpp` | One-shot LZXP/LZO block decoder CLI (verbatim LZO1X from the port's fonts path). | +| `ExtractManual.cs` + `chunk_dump.cs` | C# UELib dumpers (need `Eliot.UELib.dll`). They parse the summary/chunk table and list (wave, off, size) from the deserialized properties rather than calling `UBulkData.LoadData` (which translates offsets through the compressed-chunk stream and yields wrong buffers). | +| `ue_package.py` | Pure-Python helpers: chunk Table via UELib, decompress via `lzodec`, assemble plain packages. | +| `ue_audio_extract.py` | CLI: `.xxx` → per-wave `.xma2` + TXTH stubs (vgmstream decodes them). | +| `xma2_reencode.py` | CLI: encode WAVs → xma2 packets (external `xma2enc_cli.exe`, birabittoh/xma2enc), rebuild the bulk header (dataSize, numSamples, packet count, seek table), splice at the wave's StorageOffset, verify the export still parses. | + +## Rebuild notes + +``` +REM fullscreen build on Windows (MSVC) +cl lzodec.cpp /EHsc /O2 /Fe:lzodec.exe + +REM .NET tools (csc.exe via the VS Build Tools toolchain) +csc.exe /nologo /out:chunk_dump.exe /r:Eliot.UELib.dll chunk_dump.cs +csc.exe /nologo /out:ExtractManual.exe /r:Eliot.UELib.dll ExtractManual.cs +``` + +`xma2enc_cli.exe` is built from https://github.com/birabittoh/xma2enc +(WMA-Pro bitstream + XMA2 packet layer). Repack bitrates are auto-tuned so +the re-encoded blob fits the original bulk StorageSize (the export record +and the package's byte budget don't change). + +## Validated results + +- Full game inventory: 140 `*_LOC_int.xxx` packages → 9753 `SoundNodeWave` + exports → decoded to 9753 WAVs (RMS ~5-7k, voice-grade, 24000 Hz). +- 902 mission lines matched 1:1 nome→linha `*_sound_vo_main.int` (en+pt) and + cloned in pt-BR (`language="Portuguese"`, voice clones from each line's + English reference); pilot of 5 lines approved by the author (intonation + preserved, Portuguese accent). +- Repack of the market package: boot into the militarybase with the repacked + `01_map_market_sound_LOC_int.xxx`, log free of XMA/audio errors. diff --git a/tools/ue_audio_extract/lzodec.cpp b/tools/ue_audio_extract/lzodec.cpp new file mode 100644 index 0000000..2f050b1 --- /dev/null +++ b/tools/ue_audio_extract/lzodec.cpp @@ -0,0 +1,240 @@ +// lzodec.cpp — UE3/Xenon LZXP/LZO chunk decompressor (one-shot CLI tool). +// +// Reads one compressed chunk (or the whole stream of chunks in a package) +// from a CookedXenon *.xxx file and writes the decompressed logical bytes +// to stdout/file. The block decoder is the LZO1X implementation shipped in +// this project's own renderer/fonts path (`aot/src/cyrillic_fonts.cpp`), +// derived verbatim from the port codebase (same decoder used for the +// Russian fonts' page blobs). UE3 packages in this game are compressed with +// LZO chunks (CompressionFlags=2): UELib stops after the summary, so this +// tool is required before UELib can see the tables. +// +// Block layout inside a compressed chunk: +// u32 Tag (UE signature, big-endian) +// u32 ChunkSize (block uncompressed size, 0x20000) +// u32 CompTotalBytes, UncompTotalBytes +// blocks[]: pairs (compSize, uncompSize), big-endian +// compressed data follows (each block is an independent LZO1X stream) +// +// usage: lzodec [] +// HdrOff: offset of the chunk header; if omitted, the first UE signature +// beyond the summary is used. + +#include +#include +#include +#include +#include + +static uint32_t U(const uint8_t* d, long off) { + return ((uint32_t)d[off] << 24) | ((uint32_t)d[off + 1] << 16) | ((uint32_t)d[off + 2] << 8) | d[off + 3]; +} + +// LZO1X block decoder (verbatim from aot/src/cyrillic_fonts.cpp). +static bool Lzo1xDecompress(const uint8_t *in, size_t in_len, uint8_t *out, size_t out_len) +{ + const uint8_t *ip = in, *ip_end = in + in_len; + uint8_t *op = out, *op_end = out + out_len; + size_t t = 0; + const uint8_t *m_pos = nullptr; + auto need_in = [&](size_t n) { return size_t(ip_end - ip) >= n; }; + auto need_out = [&](size_t n) { return size_t(op_end - op) >= n; }; + auto long_length = [&](size_t base) -> bool { + while (need_in(1) && *ip == 0) + { + t += 255; + ++ip; + } + if (!need_in(1)) + return false; + t += base + *ip++; + return true; + }; + auto copy_literals = [&](size_t n) -> bool { + if (!need_in(n) || !need_out(n)) + return false; + std::memcpy(op, ip, n); + op += n; + ip += n; + return true; + }; + auto copy_match = [&](size_t n) -> bool { + if (m_pos < out || m_pos >= op || !need_out(n)) + return false; + for (size_t i = 0; i < n; ++i) + *op++ = *m_pos++; + return true; + }; + + if (!need_in(1)) + return false; + bool skip_to_first = false, skip_to_next = false; + if (*ip > 17) + { + t = *ip++ - 17; + if (t < 4) + { + skip_to_next = true; + } + else + { + if (!copy_literals(t)) + return false; + skip_to_first = true; + } + } + for (;;) + { + if (!skip_to_first && !skip_to_next) + { + if (!need_in(1)) + return false; + t = *ip++; + if (t < 16) + { + if (t == 0 && !long_length(15)) + return false; + if (!copy_literals(t + 3)) + return false; + skip_to_first = true; + } + } + if (skip_to_first) + { + skip_to_first = false; + if (!need_in(1)) + return false; + t = *ip++; + if (t < 16) + { + if (!need_in(1)) + return false; + m_pos = op - (1 + 0x0800) - (t >> 2) - (size_t(*ip++) << 2); + if (!copy_match(3)) + return false; + goto match_done; + } + } + for (;;) + { + if (skip_to_next) + { + skip_to_next = false; + goto match_next; + } + if (t >= 64) + { + if (!need_in(1)) + return false; + m_pos = op - 1 - ((t >> 2) & 7) - (size_t(*ip++) << 3); + t = (t >> 5) - 1; + if (!copy_match(t + 2)) + return false; + } + else if (t >= 32) + { + t &= 31; + if (t == 0 && !long_length(31)) + return false; + if (!need_in(2)) + return false; + m_pos = op - 1 - ((ip[0] >> 2) + (size_t(ip[1]) << 6)); + ip += 2; + if (!copy_match(t + 2)) + return false; + } + else if (t >= 16) + { + m_pos = op - ((t & 8) << 11); + t &= 7; + if (t == 0 && !long_length(7)) + return false; + if (!need_in(2)) + return false; + m_pos -= (ip[0] >> 2) + (size_t(ip[1]) << 6); + ip += 2; + if (m_pos == op) + return op == op_end; + m_pos -= 0x4000; + if (!copy_match(t + 2)) + return false; + } + else + { + if (!need_in(1)) + return false; + m_pos = op - 1 - (t >> 2) - (size_t(*ip++) << 2); + if (!copy_match(2)) + return false; + } + match_done: + t = ip[-2] & 3; + if (t == 0) + break; + match_next: + if (!copy_literals(t)) + return false; + if (!need_in(1)) + return false; + t = *ip++; + } + } +} + +int main(int argc, char** argv) +{ + if (argc < 3) { + std::fprintf(stderr, "usage: lzodec []\n"); + return 2; + } + const char* inpath = argv[1]; + const char* outpath = argv[2]; + long hdr = argc > 3 ? atol(argv[3]) : -1; + + FILE* fi = std::fopen(inpath, "rb"); + if (!fi) { std::perror("open"); return 1; } + std::fseek(fi, 0, SEEK_END); + long fsize = std::ftell(fi); + std::fseek(fi, 0, SEEK_SET); + std::vector d(fsize); + if (std::fread(d.data(), 1, fsize, fi) != (size_t)fsize) return 1; + std::fclose(fi); + + if (hdr < 0) { + hdr = -1; + for (long i = 64; i + 4 <= fsize; ++i) { + if (d[i] == 0x9e && d[i + 1] == 0x2a && d[i + 2] == 0x83 && d[i + 3] == 0xc1) { hdr = i; break; } + } + if (hdr < 0) { std::fprintf(stderr, "no chunk header\n"); return 1; } + } + std::fprintf(stderr, "hdr=%ld\n", hdr); + uint32_t chunkSize = U(d.data(), hdr + 4); + uint32_t total = U(d.data(), hdr + 12); + std::fprintf(stderr, "chunkSize=%u totalUncomp=%u\n", chunkSize, total); + + std::vector> blocks; + long hb = hdr + 16; + for (uint32_t sum = 0; sum < total && blocks.size() < 256; hb += 8) { + uint32_t cs = U(d.data(), hb), us = U(d.data(), hb + 4); + blocks.emplace_back(cs, us); + sum += us; + } + std::fprintf(stderr, "blocks=%zu\n", blocks.size()); + + std::vector out(total); + long at = hb, written = 0; + for (size_t b = 0; b < blocks.size(); ++b) { + uint32_t cs = blocks[b].first, us = blocks[b].second; + bool ok = Lzo1xDecompress(d.data() + at, cs, out.data() + written, us); + std::fprintf(stderr, "block %zu comp=%u uncomp=%u ok=%d\n", b, cs, us, ok ? 1 : 0); + if (!ok) return 1; + at += cs; + written += us; + } + FILE* fo = std::fopen(outpath, "wb"); + if (!fo) { std::perror("out"); return 1; } + std::fwrite(out.data(), 1, out.size(), fo); + std::fclose(fo); + std::fprintf(stderr, "OK wrote %zu bytes\n", out.size()); + return 0; +} diff --git a/tools/ue_audio_extract/ue_audio_extract.py b/tools/ue_audio_extract/ue_audio_extract.py new file mode 100644 index 0000000..8f19b61 --- /dev/null +++ b/tools/ue_audio_extract/ue_audio_extract.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""ue_audio_extract.py — extract SoundNodeWave audio from one UE3 LOC package. + +Pipeline: + 1. Decompress LZXP/LZO chunks (lzodec.exe, see lzodec.cpp) and assemble a + plain UE3 package (see ../../tools/ue_audio_extract/ue_package.py). + 2. Enumerate audios with the bundled C# inspector (`ExtractManual.exe`): + every USoundNodeWave export plus its BulkData (StorageOffset, + StorageSize) in the assembled package. + 3. Parse each bulk element's 56+4*packets-byte header (dataSize, + numSamples, packet count + seek table), split the raw XMA2 payload and + write a vgmstream TXTH stub next to every wave so they decode with: + vgmstream-cli -o wave.wav wave.xma2 + +Usage: + python ue_audio_extract.py + python ue_audio_extract.py --no-assemble + +Build prerequisites (in tools/ue_audio_extract/): + * ExtractManual.exe — compile ExtractManual.cs against Eliot.UELib.dll + * lzodec.exe — compile lzodec.cpp with cl (MSVC) or g++ +""" +import os +import struct +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import ue_package # noqa: E402 + +EXTRACTOR = os.path.join(HERE, "ExtractManual.exe") + + +def read_u32_be(buf, off): + return struct.unpack(">I", buf[off:off + 4])[0] + + +def parse_wave_bulk(blob): + if len(blob) < 56: + raise ValueError("bulk element too small") + md = { + "data_size": read_u32_be(blob, 8), + "rate_f": read_u32_be(blob, 24), + "num_samples": read_u32_be(blob, 40), + "num_samples_tight": read_u32_be(blob, 44), + "n_packets": read_u32_be(blob, 48), + } + md["hdr_len"] = 56 + 4 * md["n_packets"] + md["sample_table"] = [read_u32_be(blob, 56 + 4 * i) for i in range(md["n_packets"])] + return md + + +def main(): + if len(sys.argv) < 3: + print(__doc__) + return 2 + src, outdir = sys.argv[1], sys.argv[2] + no_assemble = "--no-assemble" in sys.argv + os.makedirs(outdir, exist_ok=True) + assembled = os.path.join(outdir, "assembled.xxx") + if no_assemble: + assembled = src + else: + ue_package.assemble_package(src, assembled) + csv = os.path.join(outdir, "waves.tsv") + subprocess.run([EXTRACTOR, assembled, outdir, csv], check=True) + n = 0 + for line in open(csv, encoding="utf-8"): + if not line.startswith("wave,"): + parts = line.rstrip("\n").split(",") + if len(parts) < 5: + continue + wave, bulk_file = parts[0], parts[4] + blob = open(os.path.join(outdir, bulk_file), "rb").read() + md = parse_wave_bulk(blob) + payload = blob[md["hdr_len"]: md["hdr_len"] + md["data_size"]] + xma_path = os.path.join(outdir, wave + ".xma2") + open(xma_path, "wb").write(payload) + open(xma_path + ".txth", "w").write( + "codec = XMA2\ninterleave = 0x800\n" + "sample_rate = 24000\nchannels = 1\n" + "num_samples = %d\n" % md["num_samples"] + ) + print("wave %s: %d samples in %d packets" % (wave, md["num_samples"], md["n_packets"])) + n += 1 + print("extracted %d waves into %s" % (n, outdir)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ue_audio_extract/ue_package.py b/tools/ue_audio_extract/ue_package.py new file mode 100644 index 0000000..3f1beff --- /dev/null +++ b/tools/ue_audio_extract/ue_package.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""ue_package.py + +UE3 (CookedXenon) container helpers for LOC packages of the 2008 Army of Two. + +What lives here: + +* `parse_chunk_table(path)` — read the compressed-chunk table of a package + using the bundled C# tool `chunk_dump.exe` (UELib summary reader). +* `get_summary(path)` — name/import/export table offsets. +* `decompress_chunks(path)` — concatenate all chunks' decompressed bytes in + logical order (LZXP/LZO chunk decoder, see ``lzodec.cpp``). +* `assemble_package(src, dst)` — build a plain (uncompressed) UE3 package: + first 101 header bytes + concatenated payloads, with the CompressionFlags + word zeroed out so plain readers (UELib) can continue past the summary. + +Verified facts about this game's packages: +- Header: signature at [0:4], version/headers, NameTable at offset 101. +- CompressionFlags==2 → LZXP/LZO compressed chunks. UELib Deserialize() + returns early when compressed chunks are present or Classification flags + are set (``We can't continue without decompressing``), leaving + Names/Exports/Imports empty. That is library behavior, not an API bug. +- Chunk table entries (UELib ReadArray order) and how to map them to + (LogicalOffset, UncompressedSize, CompressedOffset, CompressedSize): + dump prints `CHUNK uOff=.. uSize=.. cOff=.. cSize=..` where + uOff=UncompressedOffset, uSize actually holds CompressedOffset and + cOff holds UncompressedSize? Verified empirically (market LOC file): + uncompressed chunk size == the .xxx header's uncompressed size, and + CompressedOffset == offset of the on-disk CompressedChunkHeader. +""" +import re +import subprocess +import os + +LATEST_CHUNK_FLAG = b"\x00\x00\x00\x02" + +HERE = os.path.dirname(os.path.abspath(__file__)) +CHUNKDUMP = os.environ.get("UE_CHUNK_DUMP") or os.path.join(HERE, "..", "chunk_dump.exe") +LZODEC = os.environ.get("UE_LZODEC") or os.path.join(HERE, "lzodec.exe") + + +def parse_chunk_table(path): + out = subprocess.run([CHUNKDUMP, path], capture_output=True, text=True) + chunks = [] + for line in (out.stdout + out.stderr).splitlines(): + m = re.match(r"CHUNK uOff=(\d+) uSize=(\d+) cOff=(\d+) cSize=(\d+)", line.strip()) + if not m: + continue + u_off, u_size, c_off, c_size = map(int, m.groups()) + chunks.append((u_off, c_off, u_size, c_size)) + chunks.sort(key=lambda c: c[0]) + return chunks + + +def get_summary(path): + out = subprocess.run([CHUNKDUMP, path], capture_output=True, text=True) + txt = out.stdout + out.stderr + m = re.search( + r"Names Count:(\d+) Names Offset:(\d+)\s+Exports Count:(\d+) Exports Offset:(\d+)" + r"\s+Imports Count:(\d+) Imports Offset:(\d+)", + txt, + ) + if not m: + raise RuntimeError("could not read summary:\n" + txt[:600]) + names_c, names_o, exps_c, exps_o, imps_c, imps_o = map(int, m.groups()) + return { + "names": (names_c, names_o), + "exports": (exps_c, exps_o), + "imports": (imps_c, imps_o), + } + + +def decompress_chunks(path): + """Returns the concatenated decompressed payload (bytes) of a package. + Each chunk is decompressed with ``lzodec`` (independent LZO1X blocks).""" + chunks = parse_chunk_table(path) + if not chunks: + raise RuntimeError("no compressed chunks in " + path) + parts = [] + for i, (uoff, uncomp, coff, csize) in enumerate(chunks): + tmp = "/tmp/_ue_chunk_%d.raw" % i + rr = subprocess.run([LZODEC, path, tmp, str(coff)], capture_output=True, text=True) + if rr.returncode != 0: + raise RuntimeError("lzodec failed chunk %d: %s" % (i, rr.stderr)) + raw = open(tmp, "rb").read() + assert len(raw) == uncomp, "chunk %d size %d != %d" % (i, len(raw), uncomp) + parts.append(raw) + return b"".join(parts), (chunks[0][0], chunks[-1][0] + chunks[-1][1]) + + +def assemble_package(src, dst): + """Build the plain (uncompressed) package from the compressed source.""" + payload, (u0, u1) = decompress_chunks(src) + d = open(src, "rb").read() + assembled = bytearray(d[:101] + payload) + # Logical length check + pf = d.find(LATEST_CHUNK_FLAG + b"\x00\x00\x00", 0, 2000) + if pf >= 0: + assembled[pf:pf + 4] = b"\x00\x00\x00\x00" + open(dst, "wb").write(assembled) + return dst diff --git a/tools/ue_audio_extract/xma2_reencode.py b/tools/ue_audio_extract/xma2_reencode.py new file mode 100644 index 0000000..74c27f2 --- /dev/null +++ b/tools/ue_audio_extract/xma2_reencode.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""xma2_reencode.py — splice dubbed WAVs into an assembled UE3 package. + +Takes a decompressed (plain) package and a directory with dubbed WAVs named +``.wav``; for each wave whose bulk data slot is present it: + + 1. WAV -> XMA2 packets via the external xma2enc CLI (birabittoh/xma2enc, + WMA-Pro bitstream + XMA2 packet layer; build from source or supply + ``xma2enc_cli.exe`` via ``XMA2ENC``). + 2. Rebuilds the 108-byte UE bulk header (new dataSize, numSamples, + packet count, per-packet cumulative sample positions) — the exact + layout we reverse-engineered from the original data. + 3. Splices the new blob into the package at the wave's StorageOffset, + padding the remainder of the StorageSize region with 0xFF so that + file size and all package offsets stay intact (the engine reads + lengths out of the table, not the file position). + +Usage: + python xma2_reencode.py + [--bitrate 36000] [--budget ] +""" +import os +import struct +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +XMA2ENC = os.environ.get("XMA2ENC") or os.path.join(HERE, "xma2enc_cli.exe") +MAX_BUDGET_FACTOR = 1.0 # never exceed StorageSize + + +def frames_of(packet): + """XMA2 packet header: big-endian u32, frames starting in the packet + are in bits [26..31].""" + h = struct.unpack(">I", packet[:4])[0] + return (h >> 26) & 0x3F + + +def encode(wav_path, tmp_out, bitrate): + r = subprocess.run([XMA2ENC, wav_path, tmp_out, str(bitrate)], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError("xma2enc failed:\n" + r.stderr) + d = open(tmp_out, "rb").read() + i = d.find(b"data") + n = struct.unpack("I", struct.unpack(">I", orig_hdr[0:4])[0]) # 44 + hdr += struct.pack(">I", struct.unpack(">I", orig_hdr[4:8])[0]) # 52 + hdr += struct.pack(">I", len(raw)) # dataSize + hdr += orig_hdr[12:36] # constants + hdr += struct.pack(">I", 2048) # packet size + hdr += struct.pack(">I", sum(frames) * 512) # numSamples + hdr += orig_hdr[44:48] # trailing dword + hdr += struct.pack(">I", nblocks) # nPackets + hdr += struct.pack(">I", 0x01000000) # marker + cum = 0 + for fc in frames: + cum += fc * 512 + hdr += struct.pack(">I", cum) + return bytes(hdr) + raw + + +def repack(assembled_pkg, dub_dir, out_pkg, verbose=True): + pkg = bytearray(open(assembled_pkg, "rb").read()) + # Find waves and their storage in the assembled package. + list_csv = os.environ.get("UE_AUDIO_LIST") or os.path.join(HERE, "waves.tsv") + entries = [] + if os.path.exists(list_csv): + for line in open(list_csv, encoding="utf-8"): + parts = line.rstrip("\n").split(",") + if len(parts) < 4 or parts[0] == "wave": + continue + entries.append((parts[0], int(parts[2]), int(parts[3]))) + else: + raise RuntimeError("run ue_audio_extract.py first (needs waves.tsv)") + replaced, skipped = 0, 0 + for wave, off, size in entries: + dub = os.path.join(dub_dir, wave + ".wav") + if not os.path.exists(dub): + continue + orig_hdr = bytes(pkg[off:off + 108]) + raw, br = encode_to_budget(dub, size, "/tmp/_enc.xma2.wav") + blob = rebuild_blob(orig_hdr, raw) + if len(blob) > size: + if verbose: + print(" SKIP", wave, "blob too big", len(blob), ">", size) + skipped += 1 + continue + pkg[off:off + size] = blob + b"\xff" * (size - len(blob)) + replaced += 1 + if verbose: + print(" replaced", wave, "storage", size, "-> blob", len(blob), "@", br) + open(out_pkg, "wb").write(pkg) + print("wrote", out_pkg, "replaced", replaced, "skipped", skipped) + return out_pkg + + +def main(): + if len(sys.argv) < 4: + print(__doc__) + return 2 + repack(sys.argv[1], sys.argv[2], sys.argv[3]) + return 0 + + +if __name__ == "__main__": + sys.exit(main())