From 02764032753b6de5bb2b305112fe1a145b1b7944 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:55:24 +0900 Subject: [PATCH 1/2] test(cli): add pinned TypeScript differential and chaos coverage --- crates/bamts-cli/tests/README.md | 32 +++ crates/bamts-cli/tests/cli.rs | 102 ++++++-- crates/bamts-cli/tests/cli/chaos.rs | 243 ++++++++++++++++++ .../tests/fixtures/runtime-workload.ts | 48 ++++ 4 files changed, 407 insertions(+), 18 deletions(-) create mode 100644 crates/bamts-cli/tests/README.md create mode 100644 crates/bamts-cli/tests/cli/chaos.rs create mode 100644 crates/bamts-cli/tests/fixtures/runtime-workload.ts diff --git a/crates/bamts-cli/tests/README.md b/crates/bamts-cli/tests/README.md new file mode 100644 index 00000000..cc348525 --- /dev/null +++ b/crates/bamts-cli/tests/README.md @@ -0,0 +1,32 @@ +# CLI execution and chaos tests + +Use the repository-pinned Node 24.18.0 on `PATH`, then install the locked TypeScript 7.0.2 oracle from the repository root: + +```sh +npm ci +cargo test --locked -p bamts-cli +``` + +Run the bounded chaos suite alone: + +```sh +cargo test --locked -p bamts-cli --test cli chaos:: +``` + +The oracle helpers verify both versions, even when a single differential test is selected. They compile the original `.ts` files with `tsc --strict`, execute the emitted JavaScript in Node, and compare it with the original TypeScript executed through real `bamts --api` processes in JIT and AOT modes. Compiler internals and transport dispatch are not mocked. + +## Coverage + +`runtime-workload.ts` exercises typed record processing, loop closure capture, destructuring defaults, Map iteration, generators, labeled control flow with `finally`, and asynchronous completion. `optional-chain-continuations.ts` covers skipped computed keys and arguments, optional-call continuations, non-null assertions, grouped method receivers, and deletion. + +The generated-program test uses eight fixed seeds and sixteen bounded functions per seed. Each function executes arithmetic through `continue` and `finally`. Expected results are calculated independently in Rust and checked against Node before comparison with JIT and AOT. Failures print the seed and complete generated source. No source depends on wall time or host randomness. + +The transport test interleaves 96 malformed JSON/request frames with valid version requests across eight sessions. Seeded fragments split headers and bodies into 1–17-byte writes. Every malformed frame must produce the expected typed error, and every subsequent request must receive its own response in order. + +Additional tests cover exact header limits, unterminated headers with the peer still connected, duplicate lengths, truncated frames, LSP edit recovery, 512 KiB TypeScript output, and simultaneous 256 KiB stdout/stderr capture. The process harness drains both pipes while the child runs, retains a 120-second child deadline, and kills/reaps a timed-out child. The capture regression has a five-second deadline; the watchdog regression uses 200 milliseconds. + +## Limits + +These tests provide bounded regression evidence, not exhaustive fuzzing, formal gate admission, full TypeScript compatibility, or performance benchmark evidence. Socket-specific process tests are Unix-only. The deterministic framing unit tests remain portable. The watchdog controls the direct child, not arbitrary descendant process trees. + +The checker still diagnoses some valid optional-chain indexed accesses with `BAMTS-C064`; the optional-chain fixture separately establishes corrected execution semantics. No reference implementation source is copied into these fixtures. diff --git a/crates/bamts-cli/tests/cli.rs b/crates/bamts-cli/tests/cli.rs index 79e55388..ce4f4582 100644 --- a/crates/bamts-cli/tests/cli.rs +++ b/crates/bamts-cli/tests/cli.rs @@ -1,5 +1,5 @@ use std::fs; -use std::io::Write; +use std::io::{Read, Write}; #[cfg(unix)] use std::os::unix::io::OwnedFd; #[cfg(unix)] @@ -73,6 +73,39 @@ vm.runInThisContext("import('node:util').then(function(ns) { process.stdout.writ "#; static NEXT_DIRECTORY: AtomicU32 = AtomicU32::new(0); +#[path = "cli/chaos.rs"] +mod chaos; + +#[test] +fn process_capture_drains_stdout_and_stderr_before_waiting() { + let child = Command::new("node") + .args([ + "-e", + "process.stdout.write('x'.repeat(262144)); process.stderr.write('y'.repeat(262144));", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Node is required for CLI E2E oracle tests"); + let output = + wait_for_output_with_timeout(child, "large two-pipe output", Duration::from_secs(5)); + assert_success(&output, "large two-pipe output"); + assert_eq!(output.stdout, vec![b'x'; 262144]); + assert_eq!(output.stderr, vec![b'y'; 262144]); +} + +#[test] +#[should_panic(expected = "watchdog probe exceeded")] +fn process_capture_still_terminates_and_reaps_a_hung_child() { + let child = Command::new("node") + .args(["-e", "setInterval(() => {}, 1000);"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Node watchdog probe starts"); + wait_for_output_with_timeout(child, "watchdog probe", Duration::from_millis(200)); +} + #[test] fn optional_chain_continuations_skip_side_effects_and_preserve_receivers() { let project = ScratchDirectory::new(); @@ -1669,24 +1702,57 @@ fn assert_execution_success(output: &ExecutionOutput, command: &str) { ); } -fn wait_for_output(mut child: Child, command: &str) -> Output { - let deadline = Instant::now() + Duration::from_secs(120); - loop { - match child.try_wait() { - Ok(Some(_)) => return child.wait_with_output().expect("finished child output"), - Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), - Ok(None) => { - child.kill().expect("timed-out child is killed"); - let output = child.wait_with_output().expect("timed-out child output"); - panic!( - "{command} exceeded 120 seconds\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); +fn wait_for_output(child: Child, command: &str) -> Output { + wait_for_output_with_timeout(child, command, Duration::from_secs(120)) +} + +fn wait_for_output_with_timeout(mut child: Child, command: &str, timeout: Duration) -> Output { + let deadline = Instant::now() + timeout; + drop(child.stdin.take()); + thread::scope(|scope| { + let stdout = child + .stdout + .take() + .map(|pipe| scope.spawn(move || read_output(pipe))); + let stderr = child + .stderr + .take() + .map(|pipe| scope.spawn(move || read_output(pipe))); + let (status, timed_out) = loop { + match child.try_wait() { + Ok(Some(status)) => break (status, false), + Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Ok(None) => { + child.kill().expect("timed-out child is killed"); + break (child.wait().expect("timed-out child is reaped"), true); + } + Err(error) => { + child.kill().expect("failed child wait is terminated"); + child.wait().expect("failed child wait is reaped"); + panic!("could not wait for {command}: {error}"); + } } - Err(error) => panic!("could not wait for {command}: {error}"), - } - } + }; + let output = Output { + status, + stdout: stdout.map_or_else(Vec::new, |reader| reader.join().expect("stdout reader")), + stderr: stderr.map_or_else(Vec::new, |reader| reader.join().expect("stderr reader")), + }; + assert!( + !timed_out, + "{command} exceeded {timeout:?}\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output + }) +} + +fn read_output(mut pipe: impl Read) -> Vec { + let mut bytes = Vec::new(); + pipe.read_to_end(&mut bytes) + .expect("child pipe is readable"); + bytes } fn framed(payload: &[u8]) -> Vec { diff --git a/crates/bamts-cli/tests/cli/chaos.rs b/crates/bamts-cli/tests/cli/chaos.rs new file mode 100644 index 00000000..a68de76a --- /dev/null +++ b/crates/bamts-cli/tests/cli/chaos.rs @@ -0,0 +1,243 @@ +//! Reproducible real-process tests. No random seed is drawn from the host. + +use super::*; +use std::fmt::Write as _; + +const SEEDS: [u32; 8] = [0, 1, 2, 7, 42, 0x5eed, 0xdead_beef, u32::MAX]; + +fn reference_output(project: &ScratchDirectory, entrypoint: &str) -> Vec { + verify_oracle_versions(); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let compiler = root.join("node_modules/typescript/bin/tsc"); + assert!( + compiler.is_file(), + "run npm ci at the repository root before oracle E2E tests" + ); + let mut compile = Command::new("node"); + compile + .arg(compiler) + .args([ + "--ignoreConfig", + "--strict", + "--target", + "es2022", + "--module", + "commonjs", + "--outDir", + "oracle", + entrypoint, + ]) + .current_dir(&project.path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = wait_for_output( + compile.spawn().expect("TypeScript oracle starts"), + "TypeScript oracle", + ); + assert_success(&output, "TypeScript oracle"); + let javascript = project + .path + .join("oracle") + .join(Path::new(entrypoint).with_extension("js")); + let child = Command::new("node") + .arg(javascript) + .current_dir(&project.path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Node oracle starts"); + let output = wait_for_output(child, "Node oracle"); + assert_success(&output, "Node oracle"); + assert!( + output.stderr.is_empty(), + "Node oracle stderr: {}", + stderr(&output) + ); + output.stdout +} + +#[test] +fn oracle_versions_match_the_repository_pins() { + verify_oracle_versions(); +} + +fn verify_oracle_versions() { + static VERIFIED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); + VERIFIED.get_or_init(|| { + let child = Command::new("node") + .arg("--version") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Node oracle is installed"); + let node = wait_for_output(child, "Node version"); + assert_success(&node, "Node version"); + assert_eq!( + stdout(&node).trim(), + "v24.18.0", + "select the repository-pinned Node on PATH" + ); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let child = Command::new("node") + .arg(root.join("node_modules/typescript/bin/tsc")) + .arg("--version") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("TypeScript oracle is installed"); + let typescript = wait_for_output(child, "TypeScript version"); + assert_success(&typescript, "TypeScript version"); + assert_eq!(stdout(&typescript).trim(), "Version 7.0.2"); + }); +} + +#[test] +fn real_typescript_workloads_match_the_pinned_oracle_in_both_modes() { + for (name, source) in [ + ( + "runtime-workload", + include_str!("../fixtures/runtime-workload.ts"), + ), + ( + "optional-chain-continuations", + include_str!("../fixtures/optional-chain-continuations.ts"), + ), + ] { + let project = ScratchDirectory::new(); + project.write("main.ts", source); + let expected = reference_output(&project, "main.ts"); + for mode in ["jit", "aot"] { + let actual = project.execute(mode, "main.ts", &[]); + assert_execution_success(&actual, mode); + assert_eq!(actual.stdout, expected, "{name}, {mode}"); + } + } +} + +#[test] +fn seeded_typescript_control_flow_matches_independent_arithmetic_and_node() { + for seed in SEEDS { + let mut state = seed; + let mut source = + String::from("declare const process: { stdout: { write(text: string): void } };\n"); + let mut expected = String::new(); + for case in 0..16 { + let mut next = || { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state + }; + let initial = next() % 997; + let factor = next() % 8 + 2; + let addend = next() % 31 + 1; + let divisor = next() % 6 + 2; + let iterations = next() % 24 + 8; + writeln!(source, "function case{case}(): number {{ let value: number = {initial}; for (let i = 0; i < {iterations}; i++) {{ try {{ if (i % {divisor} === 0) {{ continue; }} value = (value * {factor} + {addend}) % 10007; }} finally {{ value = (value + 7) % 10007; }} }} return value; }} process.stdout.write('{seed}:{case}:' + case{case}() + '\\n');").expect("write generated TypeScript"); + let mut value = initial; + for index in 0..iterations { + if index % divisor != 0 { + value = (value * factor + addend) % 10007; + } + value = (value + 7) % 10007; + } + writeln!(expected, "{seed}:{case}:{value}").expect("write independent result"); + } + let project = ScratchDirectory::new(); + project.write("main.ts", &source); + assert_eq!( + reference_output(&project, "main.ts"), + expected.as_bytes(), + "oracle seed={seed}\n{source}" + ); + for mode in ["jit", "aot"] { + let actual = project.execute(mode, "main.ts", &[]); + assert_execution_success(&actual, mode); + assert_eq!( + actual.stdout, + expected.as_bytes(), + "seed={seed}, mode={mode}\n{source}" + ); + } + } +} + +#[test] +fn large_typescript_output_survives_real_api_backpressure() { + let project = ScratchDirectory::new(); + project.write( + "main.ts", + "for (let i = 0; i < 4096; i++) { process.stdout.write('0123456789abcdef'.repeat(8)); }\n", + ); + let expected = "0123456789abcdef".repeat(8 * 4096); + for mode in ["jit", "aot"] { + let output = project.execute(mode, "main.ts", &[]); + assert_execution_success(&output, mode); + assert_eq!( + output.stdout, + expected.as_bytes(), + "{mode}: 512 KiB response must not fill a pipe and hang" + ); + } +} + +#[test] +fn seeded_malformed_json_and_fragmentation_preserve_api_recovery() { + let malformed: [(&[u8], i64); 6] = [ + (b"{", -32700), + (b"\xff", -32700), + (b"null", -32600), + (b"[]", -32600), + (br#"{"method":1}"#, -32600), + (br#"{"id":true,"method":"compiler/version"}"#, -32600), + ]; + for seed in SEEDS { + let project = ScratchDirectory::new(); + let mut state = seed; + let mut input = Vec::new(); + let mut errors = Vec::new(); + for id in 1..=12 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let (payload, code) = malformed[(state as usize) % malformed.len()]; + input.extend(framed(payload)); + input.extend(framed( + &serde_json::to_vec(&serde_json::json!({ + "jsonrpc":"2.0", "id":id, "method":"compiler/version" + })) + .expect("version request serializes"), + )); + errors.push(code); + } + let mut child = project + .command() + .arg("--api") + .current_dir(&project.path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("real API child starts"); + let mut stdin = child.stdin.take().expect("piped stdin"); + let mut offset = 0; + while offset < input.len() { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let end = (offset + (state as usize % 17) + 1).min(input.len()); + stdin + .write_all(&input[offset..end]) + .expect("fragment is written"); + offset = end; + } + drop(stdin); + let output = wait_for_output(child, "seeded malformed API frames"); + assert_success(&output, "seeded malformed API frames"); + let responses = decode_frames(&output.stdout); + assert_eq!(responses.len(), 24, "seed={seed}: {responses:?}"); + for (index, code) in errors.iter().enumerate() { + assert_eq!(responses[index * 2]["id"], serde_json::Value::Null); + assert_eq!(responses[index * 2]["error"]["code"], *code, "seed={seed}"); + assert_eq!(responses[index * 2 + 1]["id"], index + 1, "seed={seed}"); + assert!( + responses[index * 2 + 1]["result"]["version"].is_string(), + "seed={seed}" + ); + } + } +} diff --git a/crates/bamts-cli/tests/fixtures/runtime-workload.ts b/crates/bamts-cli/tests/fixtures/runtime-workload.ts new file mode 100644 index 00000000..bce07e01 --- /dev/null +++ b/crates/bamts-cli/tests/fixtures/runtime-workload.ts @@ -0,0 +1,48 @@ +declare const process: { stdout: { write(text: string): void } }; + +interface Row { readonly id: number; readonly score: number; } +const rows: readonly Row[] = [{ id: 1, score: 5 }, { id: 2, score: 9 }, { id: 3, score: 3 }]; +const total = rows.filter(row => row.score > 3).reduce((sum, row) => sum + row.score, 0); +process.stdout.write(String(total) + "\n"); + +const callbacks: (() => number)[] = []; +for (let i = 0; i < 5; i++) { callbacks.push(() => i); } +process.stdout.write(callbacks.map(callback => callback()).join(",") + "\n"); + +let a = 1; +let b = 2; +[a, b] = [b, a]; +let defaults = 0; +function nextDefault(): number { defaults++; return defaults; } +const { x = nextDefault(), y = nextDefault() } = { x: 7, y: undefined }; +process.stdout.write([a, b, x, y, defaults].join(",") + "\n"); + +const counts = new Map(); +for (const value of [3, 1, 4, 1, 5]) { counts.set(value, (counts.get(value) ?? 0) + 1); } +const sorted = Array.from(counts.keys()).sort((left, right) => left - right); +process.stdout.write(sorted.map(key => key + ":" + counts.get(key)).join(",") + "\n"); + +function* squares(limit: number): Generator { + for (let i = 0; i < limit; i++) { yield i * i; } +} +let sum = 0; +for (const value of squares(8)) { sum += value; } +process.stdout.write(String(sum) + "\n"); + +function cleanup(limit: number): number { + let result = 0; + outer: for (let i = 0; i < limit; i++) { + try { + if (i % 3 === 0) { continue outer; } + if (i === 7) { break outer; } + result += i; + } finally { result += 10; } + } + return result; +} +process.stdout.write(String(cleanup(20)) + "\n"); + +async function increment(value: number): Promise { + return (await Promise.resolve(value)) + 1; +} +increment(41).then(value => process.stdout.write(String(value) + "\n")); From 5a952f78b4be1dc47c1527274ea047b0d94f1d56 Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:19:33 +0900 Subject: [PATCH 2/2] fix(ci): provision pinned TypeScript oracle --- .github/workflows/cli-release-build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-release-build.yml b/.github/workflows/cli-release-build.yml index d4d3cd44..558ec0d7 100644 --- a/.github/workflows/cli-release-build.yml +++ b/.github/workflows/cli-release-build.yml @@ -99,7 +99,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 24 + node-version: 24.18.0 package-manager-cache: false - name: Install pinned Rust toolchain shell: pwsh @@ -111,6 +111,10 @@ jobs: - name: Install Rust target shell: pwsh run: rustup target add '${{ matrix.target }}' + - name: Install pinned TypeScript oracle + if: matrix.label == 'win32-x64' + shell: pwsh + run: npm ci - name: Test Windows cache guard if: matrix.label == 'win32-x64' shell: pwsh