From 1e3764963150fbbea7b2b76e514af8db58f0b407 Mon Sep 17 00:00:00 2001 From: James Braid Date: Mon, 2 Nov 2026 13:34:34 -0700 Subject: [PATCH 1/3] cpu: report Invalid and Overflow as the hardware does The Octane PROM's ide FPU test checks results the host FPU gets wrong for MIPS: infinity minus infinity, zero times infinity, 0/0 and infinity/infinity are Invalid Operations that deliver the default NaN (0x7FBFFFFF), 0/0 is not a divide by zero, and MAX+MAX is Overflow with Inexact. iris returned the host's NaN with no flags. 12 of the 14 subtests passed. Classify those results on every CPU model, in both engines, at no cost to an ordinary result. ADD, SUB, MUL and DIV test the result's magnitude once. That test replaces the old denormal-result test, and the top of the range, which is where Invalid and Overflow live, goes to a cold classifier shared by both engines. The operand flags those ops used to compute up front are gone: an ordinary result proves there were none, and the classifier finds a divide by zero itself. Compiled code reaches the classifier through one call per instruction. The same call takes a denormal, infinite or NaN operand and a denormal result, where the interpreter's operand checks and denormal rule run, so the instruction carries one call and one trap check, not a copy of those rules at each of two exits. Zero, as an operand or a result, is common in real code and stays inline. Generated code for the page holding Whetstone and its transcendental kernel comes to 247 bytes per guest instruction, against 404 before this series. SQRT, RECIP, RSQRT, the multiply-add family and CVT.S.D swap the host's NaN for the default NaN and add V. They do it the same way round: one compare of the result's magnitude, and only an infinity or a NaN goes on to classify the operands. In compiled RECIP, RSQRT and the multiply-add family the compare shares the branch these ops already took on a stale Cause, so the ordinary path gains no branch. A finite result cannot come from a NaN operand, a zero divisor or a negative square root, so the signalling-NaN tests on up to three operands no longer run for ordinary arithmetic. CVT.S.D checks its source for a signalling NaN only after the conversion has already left the exact path, since a NaN is never equal to itself. Host instructions per FP op for this and the next commit together, perf stat instructions:u, ordinary operands, ship profile, against the commit before this series (slope over a 120-op region, three alternating rounds, counts do not move between rounds). R10000 numbers; R5000 and R4400 differ by a constant: compiled add.d, mul.d, div.d 36 -> 35 compiled sqrt.s, cvt.s.d no change interpreter add.d, mul.d, div.d 197-200 -> 171 interpreter madd/msub/nmadd/nmsub 168-181 -> 153-159 interpreter recip.s, rsqrt.s 145 -> 136, 153 -> 137 interpreter sqrt.s, cvt.s.d no change Every measured op, in both engines, on the R4400, R5000 and R10000, is within 0.1 instruction of the old count or cheaper. The compiled madd/msub/nmadd/nmsub, recip and rsqrt were recounted from the generated code of a 120-op run of each (R10000, host instructions on the ordinary path, before -> after): madd.d 52 -> 24, recip.s 39 -> 24, rsqrt.s 48 -> 27, with the same three conditional branches per op as before. The tests that check results run on the ordinary passthrough models in both engines, and compiled code is checked against the interpreter for each rule. This commit delivers the IEEE 754-2008 default NaN, 0x7FC00000. The Octane's R10000 delivers 0x7FBFFFFF; the next commit moves the NaN encoding, and with it the ide test's NaN checks pass. --- CHANGELOG.md | 20 + .../jitv2/fpu-flags-are-computed-not-read.md | 49 +- src/cpu/jitv2/codegen.rs | 439 ++++++++++++------ src/cpu/jitv2/equiv_test.rs | 335 +++++++++++++ src/cpu/mips_core.rs | 12 + src/cpu/mips_exec.rs | 438 ++++++++++++++--- src/cpu/mips_exec_test.rs | 293 ++++++++++++ 7 files changed, 1378 insertions(+), 208 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c62ec1a..366855d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,26 @@ hashes are given where a change is easiest to understand by reading the commit. bus's devices; and a `Jitv2` joins its workers before it frees the pages they read. The Indy, Indigo2 and O2 still leak their tables. +### cpu: FPU Invalid and Overflow + +- **Invalid operations raise Invalid.** Infinity minus infinity, zero times + infinity, 0/0 and infinity/infinity set Cause and Flag V and, untrapped, + deliver the default quiet NaN (0x7FC00000, 0x7FF80000_00000000) instead of + the host's NaN. 0/0 raises Invalid rather than divide-by-zero, and + infinity/0 raises nothing. +- **Overflow raises Overflow and Inexact** in ADD, SUB, MUL and DIV. The result + was already rounded for the FCSR rounding mode (an infinity, or the largest + finite number when the mode rounds toward zero); only the flags were + missing. With Overflow enabled it now traps. +- SQRT, RECIP, RSQRT, MADD, MSUB, NMADD, NMSUB and CVT.S.D deliver the default + NaN for an Invalid Operation (the square root of a negative number, + infinity times zero, a signalling NaN operand) and raise V where they raised + nothing. They still compute no Overflow or Inexact. +- Applies to every CPU model. An ordinary result costs no more host + instructions than before in either engine: one range compare on the result + replaces the denormal test, and the multiply-add family, RECIP and RSQRT now + classify their operands only when the result is an infinity or a NaN. + ### Disk write cache - A disk now has a `write_cache` setting in its `[scsi.N]` table (and a diff --git a/clanker/jitv2/fpu-flags-are-computed-not-read.md b/clanker/jitv2/fpu-flags-are-computed-not-read.md index ff6396e1f..d23c73bcc 100644 --- a/clanker/jitv2/fpu-flags-are-computed-not-read.md +++ b/clanker/jitv2/fpu-flags-are-computed-not-read.md @@ -20,10 +20,51 @@ formulation both engines can implement identically. For plain ADD/SUB/MUL/DIV/SQRT/RECIP/RSQRT and the MADD family, `flags` is **0** unless a specific condition applies (signalling-NaN operand -> FV, -divide-by-zero -> FZ). In particular: +divide-by-zero -> FZ, and the result rules below). In +particular: -- **Inexact (FI) is never set by arithmetic.** -- **Overflow (FO) is never set by arithmetic.** +- **Inexact (FI) is never set by arithmetic on its own.** It is set only as + part of an Overflow. +- **Overflow (FO) and Invalid (FV) are computed for ADD/SUB/MUL/DIV only**, + from the result: one range test on the result's magnitude, which replaced + the denormal-result test on the same path, sends zero, denormals, ±MAX, + ±infinity and NaN to a cold side. There `fpu_binop_top_flags_s/d` + (`mips_exec.rs`) classifies the top of the range: a NaN from non-NaN + operands is Invalid (inf-inf, 0*inf, 0/0, inf/inf) and delivers the + default NaN 0x7FC00000 / 0x7FF80000_00000000; an infinity, or a ±MAX whose + exact value reaches 2^(emax+1), from finite operands is Overflow+Inexact. + The JIT reaches the same function through `fpu_binop_commit_fn`, one + call that also does the operand checks and the denormal-result rule for + a denormal, infinite or NaN operand or a denormal or top-of-range result + (zero operands and results stay inline), so the engines cannot disagree + and each ADD/SUB/MUL/DIV carries one call, not a copy of the rules. 0/0 is Invalid, not divide-by-zero; FZ needs a + finite nonzero dividend. The Octane PROM's IDE `fpu` test checks both + (its "invalid result" and "overflow result" cases). SQRT, RECIP, + RSQRT, the MADD, MSUB, NMADD and NMSUB families and CVT.S.D follow the + Invalid rule only (next bullet) and still compute no Overflow or Inexact: + an overflowing result from any of them is delivered as the host rounds + it, with FCSR untouched. CVT.S.D has its own overflow check but no + Inexact-with-Overflow. +- **Invalid results of the operations above deliver the default NaN.** + The host's NaN for an invalid result (x86: 0xFFC00000) has the sign bit + set; hardware delivers the positive default NaN, so a program that stores + and compares the result sees the same bits. + `fpu_fix_invalid_s/d` (interpreter) and `emit_fix_invalid` (JIT) replace + a NaN result with 0x7FC00000 / 0x7FF80000_00000000 and add V when an + operand was a signalling NaN or no operand was a NaN (infinity times + zero, infinity minus infinity, the square root of a negative number). A + quiet NaN operand propagates unchanged. Both engines gate it on one + compare of the result's magnitude, with the classification on the cold + side. A finite result proves the operands raise nothing (a NaN operand + makes the host result a NaN, a zero divisor or an infinity makes it an + infinity, a negative square-rooted operand makes it a NaN), so for RECIP, + RSQRT and the MADD family the operand flags are not even evaluated on that + path (`fpu_fix_invalid_s/d` takes them as a closure, + `emit_commit_finite_first` as an emitter). CVT.S.D, which runs in `cvt_d_to_s_and_commit` for both engines, + does the same for a signalling NaN source. + + A quiet-NaN source to CVT.S.D has a related quirk: it is never equal to + its widened result, so it leaves the exact path and sets Cause and Flag I. Stated in `fpu_update_fcsr`'s doc comment as: *"not worth the IR to compute for ordinary arithmetic, and nothing in IRIX depends on them."* That trade is sound @@ -41,7 +82,7 @@ the computed *result* is correct in every case. | `fpu/vec_arith_single` | `o.flags` got `0x0`, want `0x1`/`0x5` — missing FI | | `fpu/vec_arith_double` | same | | `fpu/vec_sqrt` | `o.flags [sqrt.s]` got `0x0`, want `0x1` — missing FI | -| `fpu/cvt_s_d_rounds` | got `0x4`, want `0x5` — has FO, missing FI | +| `fpu/cvt_s_d_rounds` | got `0x4`, want `0x5` — has FO, missing FI (CVT.S.D, not covered by the ADD/SUB/MUL/DIV overflow rule) | | `fpu/cvt_out_of_range` | `cvt.w.s` of 2^31 does not set FV (though `+inf` does) | `fpu/cvt_out_of_range` is the odd one out: it is about Invalid on an diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index ee31f90cf..6db3fbc1d 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -3633,6 +3633,8 @@ fn core_offset_of_fpu_cvt_to_int_fn() -> i32 { std::mem::offset_of!(MipsCore, fp fn core_offset_of_fpu_cvt_int_to_float_fn() -> i32 { std::mem::offset_of!(MipsCore, fpu_cvt_int_to_float_fn) as i32 } #[cfg(feature = "jitv2")] fn core_offset_of_fpu_cvt_d_to_s_fn() -> i32 { std::mem::offset_of!(MipsCore, fpu_cvt_d_to_s_fn) as i32 } +#[cfg(feature = "jitv2")] +fn core_offset_of_fpu_binop_commit_fn() -> i32 { std::mem::offset_of!(MipsCore, fpu_binop_commit_fn) as i32 } fn core_offset_of_fpu_fir() -> i32 { std::mem::offset_of!(MipsCore, fpu_fir) as i32 } fn core_offset_of_fpu_fexr() -> i32 { std::mem::offset_of!(MipsCore, fpu_fexr) as i32 } fn core_offset_of_fpu_fenr() -> i32 { std::mem::offset_of!(MipsCore, fpu_fenr) as i32 } @@ -5599,6 +5601,119 @@ fn emit_fpu_update_fcsr_arith( ctx.builder.seal_block(continue_block); } +/// The NaN fix-up of `fpu_fix_invalid_s/d` (`mips_exec.rs`), for SQRT, RECIP, +/// RSQRT and the multiply-add family, on a result already known to be an +/// infinity or a NaN (`emit_commit_finite_first` sends only those here): a NaN +/// result is Invalid when `flags` already has V (a signalling NaN operand) or +/// when none of `ops` is a NaN, and then V is added and the default NaN +/// replaces the host's. Straight-line selects, no branches, since it runs in +/// a cold block. Returns the flags and result bits to use. +fn emit_fix_invalid(ctx: &mut EmitCtx, is_d: bool, flags: Value, result_bits: Value, ops: &[Value]) -> (Value, Value) { + let (abs_mask, inf): (i64, i64) = if is_d { + (0x7FFF_FFFF_FFFF_FFFFu64 as i64, 0x7FF0_0000_0000_0000u64 as i64) + } else { + (0x7FFF_FFFF, 0x7F80_0000) + }; + let is_nan = |ctx: &mut EmitCtx, v: Value| { + let abs = ctx.builder.ins().band_imm_s(v, abs_mask); + ctx.builder.ins().icmp_imm_s(IntCC::UnsignedGreaterThan, abs, inf) + }; + let result_is_nan = is_nan(ctx, result_bits); + let mut any_operand_nan = ctx.builder.ins().iconst(ir::types::I8, 0); + for &op in ops { + let n = is_nan(ctx, op); + any_operand_nan = ctx.builder.ins().bor(any_operand_nan, n); + } + let has_v = ctx.builder.ins().band_imm_s(flags, FCSR_FV_I64); + let has_v = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, has_v, 0); + let no_operand_nan = ctx.builder.ins().icmp_imm_s(IntCC::Equal, any_operand_nan, 0); + let invalid = ctx.builder.ins().bor(has_v, no_operand_nan); + let fix = ctx.builder.ins().band(result_is_nan, invalid); + let flags_v = ctx.builder.ins().bor_imm_s(flags, FCSR_FV_I64); + let new_flags = ctx.builder.ins().select(fix, flags_v, flags); + let default_nan = if is_d { + ctx.builder.ins().iconst(ir::types::I64, 0x7FF8_0000_0000_0000u64 as i64) + } else { + ctx.builder.ins().iconst(ir::types::I32, 0x7FC0_0000) + }; + let new_bits = ctx.builder.ins().select(fix, default_nan, result_bits); + (new_flags, new_bits) +} + +/// Commit the result of RECIP, RSQRT and the multiply-add family, mirroring `fpu_fix_invalid_s/d`: one +/// compare of the result's magnitude (the high word for double, so no 64-bit +/// immediate) tells a finite result from the rest, because a finite result +/// means no operand was a NaN, no divisor was zero and no square-rooted +/// operand was negative: nothing to flag, and the operand classification is +/// not emitted on that path at all. A finite result with Cause already clear +/// (the steady state, see `emit_fpu_update_fcsr_no_flags`) is a plain write +/// behind one conditional branch, the one these operations already had for +/// Cause: the two tests are combined without branching, so the finite check +/// adds no branch to the ordinary path. Cold, an infinity or NaN evaluates +/// `flags_fn`, applies `emit_fix_invalid` and does the full FCSR update, and +/// a finite result with Cause set only clears it. +fn emit_commit_finite_first( + ctx: &mut EmitCtx, + is_d: bool, + result_bits: Value, + ops: &[Value], + flags_fn: impl FnOnce(&mut EmitCtx) -> Value, + write_result: impl Fn(&mut EmitCtx, Value), +) { + let mem = MemFlagsData::trusted(); + let i32t = ir::types::I32; + const FCSR_CM: i64 = 0x0001_f000; + let non_finite = if is_d { + let hi = ctx.builder.ins().ushr_imm_s(result_bits, 32); + let hi = ctx.builder.ins().ireduce(i32t, hi); + let hi = ctx.builder.ins().band_imm_s(hi, 0x7FFF_FFFF); + ctx.builder.ins().icmp_imm_s(IntCC::UnsignedGreaterThanOrEqual, hi, 0x7FF0_0000) + } else { + let abs = ctx.builder.ins().band_imm_s(result_bits, 0x7FFF_FFFF); + ctx.builder.ins().icmp_imm_s(IntCC::UnsignedGreaterThanOrEqual, abs, 0x7F80_0000) + }; + let fcsr_off = ir::immediates::Offset32::new(core_offset_of_fpu_fcsr()); + let fcsr = ctx.builder.ins().load(i32t, mem, ctx.core_ptr, fcsr_off); + let cause = ctx.builder.ins().band_imm_s(fcsr, FCSR_CM); + let cause_set = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, cause, 0); + let off_path = ctx.builder.ins().bor(non_finite, cause_set); + let fast_block = ctx.builder.create_block(); + let cold_block = ctx.builder.create_block(); + let full_block = ctx.builder.create_block(); + let clear_block = ctx.builder.create_block(); + let done_block = ctx.builder.create_block(); + ctx.builder.ins().brif(off_path, cold_block, &[], fast_block, &[]); + + ctx.builder.switch_to_block(cold_block); + ctx.builder.set_cold_block(cold_block); + ctx.builder.seal_block(cold_block); + ctx.builder.ins().brif(non_finite, full_block, &[], clear_block, &[]); + ctx.builder.seal_block(full_block); + ctx.builder.seal_block(clear_block); + + ctx.builder.switch_to_block(full_block); + ctx.builder.set_cold_block(full_block); + let flags = flags_fn(ctx); + let (flags, bits) = emit_fix_invalid(ctx, is_d, flags, result_bits, ops); + emit_fpu_update_fcsr(ctx, flags, |ctx| write_result(ctx, bits)); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(clear_block); + ctx.builder.set_cold_block(clear_block); + let cleared = ctx.builder.ins().band_imm_s(fcsr, !FCSR_CM); + ctx.builder.ins().store(mem, cleared, ctx.core_ptr, fcsr_off); + write_result(ctx, result_bits); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(fast_block); + ctx.builder.seal_block(fast_block); + write_result(ctx, result_bits); + ctx.builder.ins().jump(done_block, &[]); + + ctx.builder.switch_to_block(done_block); + ctx.builder.seal_block(done_block); +} + /// IR counterparts of `mips_exec.rs`'s `fpu_arith_flags_*` free functions — /// same bit-pattern checks, no host FPU status read, feeding `flags` into /// `emit_fpu_update_fcsr`/`emit_fpu_update_fcsr_arith`. See @@ -5675,14 +5790,23 @@ fn emit_fpu_arith_flags_snan_only3_d(ctx: &mut EmitCtx, fr_bits: Value, ft_bits: /// DIV (S or D), and RECIP/RSQRT, which pass a constant `1.0` as the /// dividend — mirrors `fpu_arith_flags_div_s/d`. Invalid takes priority over -/// divide-by-zero, else Z when the divisor is zero. +/// divide-by-zero, else Z when the divisor is zero and the dividend is a +/// finite nonzero number. fn emit_fpu_arith_flags_div_s(ctx: &mut EmitCtx, fs_bits: Value, ft_bits: Value) -> Value { let snan = emit_fpu_arith_flags_snan_only_s(ctx, fs_bits, ft_bits); let ft_zero = { let masked = ctx.builder.ins().band_imm_s(ft_bits, 0x7FFF_FFFF); ctx.builder.ins().icmp_imm_s(IntCC::Equal, masked, 0) }; - let z_flag = ctx.builder.ins().uextend(ir::types::I32, ft_zero); + // Z needs a finite nonzero dividend: 0/0 is Invalid (found from its + // NaN result) and infinity/0 is an exact infinity. + let z = { + let dividend = ctx.builder.ins().band_imm_s(fs_bits, 0x7FFF_FFFF); + let minus_one = ctx.builder.ins().iadd_imm_s(dividend, -1); + let finite_nonzero = ctx.builder.ins().icmp_imm_s(IntCC::UnsignedLessThan, minus_one, 0x7F80_0000 - 1); + ctx.builder.ins().band(ft_zero, finite_nonzero) + }; + let z_flag = ctx.builder.ins().uextend(ir::types::I32, z); let z_flag = ctx.builder.ins().ishl_imm_s(z_flag, FCSR_FZ_I64.trailing_zeros() as i64); let snan_is_zero = ctx.builder.ins().icmp_imm_s(IntCC::Equal, snan, 0); ctx.builder.ins().select(snan_is_zero, z_flag, snan) @@ -5693,7 +5817,15 @@ fn emit_fpu_arith_flags_div_d(ctx: &mut EmitCtx, fs_bits: Value, ft_bits: Value) let masked = ctx.builder.ins().band_imm_s(ft_bits, 0x7FFF_FFFF_FFFF_FFFFu64 as i64); ctx.builder.ins().icmp_imm_s(IntCC::Equal, masked, 0) }; - let z_flag = ctx.builder.ins().uextend(ir::types::I32, ft_zero); + // Z needs a finite nonzero dividend: 0/0 is Invalid (found from its + // NaN result) and infinity/0 is an exact infinity. + let z = { + let dividend = ctx.builder.ins().band_imm_s(fs_bits, 0x7FFF_FFFF_FFFF_FFFFu64 as i64); + let minus_one = ctx.builder.ins().iadd_imm_s(dividend, -1); + let finite_nonzero = ctx.builder.ins().icmp_imm_s(IntCC::UnsignedLessThan, minus_one, (0x7FF0_0000_0000_0000u64 as i64) - 1); + ctx.builder.ins().band(ft_zero, finite_nonzero) + }; + let z_flag = ctx.builder.ins().uextend(ir::types::I32, z); let z_flag = ctx.builder.ins().ishl_imm_s(z_flag, FCSR_FZ_I64.trailing_zeros() as i64); let snan_is_zero = ctx.builder.ins().icmp_imm_s(IntCC::Equal, snan, 0); ctx.builder.ins().select(snan_is_zero, z_flag, snan) @@ -7616,38 +7748,30 @@ type SemanticsEmitter = fn(&mut EmitCtx); // bindings for readability. type Cp1Emitter = fn(&mut EmitCtx, fr_mode: FrMode); -/// Result of a binary FP arithmetic op, in the current block: the raw result -/// bits plus the two properties `emit_fpu_update_fcsr_arith` needs (whether -/// the result is a denormalized nonzero, and whether its sign bit is set). -/// Shared by both arms of `emit_fbinop_s`/`_d`'s operand split, so the -/// arithmetic and its result checks cannot drift between them. +/// Result bits of a binary FP arithmetic op, in the current block. Shared by +/// the inline and the call-out arms of `emit_fbinop`, so the arithmetic +/// cannot drift between them. fn emit_fbinop_result_s( ctx: &mut EmitCtx, op: fn(&mut FunctionBuilder, Value, Value) -> Value, fs_bits: Value, ft_bits: Value, -) -> (Value, Value, Value) { +) -> Value { let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); let ft_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), ft_bits); let result = op(ctx.builder, fs_val, ft_val); - let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); - let is_denorm = emit_is_subnormal_s(ctx, result_bits); - let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); - (result_bits, is_denorm, is_negative) + ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result) } fn emit_fbinop_result_d( ctx: &mut EmitCtx, op: fn(&mut FunctionBuilder, Value, Value) -> Value, fs_bits: Value, ft_bits: Value, -) -> (Value, Value, Value) { +) -> Value { let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); let ft_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), ft_bits); let result = op(ctx.builder, fs_val, ft_val); - let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); - let is_denorm = emit_is_subnormal_d(ctx, result_bits); - let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); - (result_bits, is_denorm, is_negative) + ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result) } /// Write the signed zero the Table 9-1/7-1 underflow-flush path delivers: @@ -7668,137 +7792,170 @@ fn emit_write_signed_zero_d(ctx: &mut EmitCtx, fd: u32, negative: Value, fr_mode emit_write_fpr_l(ctx, fd, signed_zero, fr_mode) } -/// Binary FP arithmetic op shape shared by ADD/SUB/MUL/DIV, S or D format: -/// read fs/ft, apply `op`, write fd, then `emit_fpu_update_fcsr`. Mirrors -/// `exec_fadd_s`/`exec_fsub_d`/etc.'s common structure exactly — CU1 is -/// already handled by the region-wide entry guard, not repeated per -/// instruction. `flags_fn` computes the [6:2] flags `Value` from the raw -/// operand bits (`emit_fpu_arith_flags_snan_only_s/d` for ADD/SUB/MUL, -/// `emit_fpu_arith_flags_div_s/d` for DIV) — never read from the host FPU. +/// ADD/SUB/MUL/DIV, S or D (`is_d`): read fs/ft, apply `op`, write fd and +/// update FCSR. Mirrors `exec_fadd_s`/`exec_fsub_d`/etc. — CU1 is already +/// handled by the region-wide entry guard, not repeated per instruction. /// -/// The operands are classified once, before anything else: if both are -/// ordinary finite numbers (`emit_is_normal_s`/`_d`), no operand check can -/// trap, no signalling NaN can be present and the divisor cannot be zero, so -/// this instruction provably raises no flag and the fast arm only has to -/// clear Cause (and even that store is skipped when Cause is already clear — -/// see `emit_fpu_update_fcsr_no_flags`). Everything else — zero, denormal, -/// infinity, NaN — takes the cold arm, which is the full previous body -/// verbatim: per-operand Unimplemented checks, the by-value flags, the -/// result-denormal rule and the full FCSR update. The result-denormal check -/// stays on both arms because an underflowing result is reachable from -/// ordinary finite operands. -fn emit_fbinop_s( +/// Only the ordinary case is inline: operands that are ordinary finite +/// numbers or zeros (`emit_is_normal_s`/`_d`, so no operand check can trap +/// and no signalling NaN is present; a zero divisor gives an infinity or a +/// NaN, caught by the result test) and a result that is a normal number or +/// zero. Such an instruction provably raises no flag; it only clears Cause +/// (and even that store is skipped when Cause is already clear, see +/// `emit_fpu_update_fcsr_no_flags`) and writes fd. One range test on the +/// result's magnitude splits off zero, denormals, ±MAX, ±infinity and NaNs; +/// zero is a common result (cleared arrays, sparse data) and an operand, so +/// it is told apart on the cold side and rejoins the plain write there. +/// +/// A denormal, infinite or NaN operand, or a denormal or top-of-range +/// result, joins one call into `fpu_binop_commit_fn` with the host's +/// result, which does what the interpreter does from the operand checks on. +/// That keeps the rare side of each instruction to a call and a trap check. +fn emit_fbinop( ctx: &mut EmitCtx, fr_mode: FrMode, + is_d: bool, op: fn(&mut FunctionBuilder, Value, Value) -> Value, - flags_fn: fn(&mut EmitCtx, Value, Value) -> Value, ) { + let i32t = ir::types::I32; + let i64t = ir::types::I64; let raw = ctx.raw; let fs = field_rd(raw); let ft = field_rt(raw); let fd = field_sa(raw); - let fs_bits = emit_read_fpr_w(ctx, fs, fr_mode); - let ft_bits = emit_read_fpr_w(ctx, ft, fr_mode); - - let fs_normal = emit_is_normal_s(ctx, fs_bits); - let ft_normal = emit_is_normal_s(ctx, ft_bits); + let (fs_bits, ft_bits) = if is_d { + (emit_read_fpr_l(ctx, fs, fr_mode), emit_read_fpr_l(ctx, ft, fr_mode)) + } else { + (emit_read_fpr_w(ctx, fs, fr_mode), emit_read_fpr_w(ctx, ft, fr_mode)) + }; + let is_normal = |ctx: &mut EmitCtx, v: Value| if is_d { emit_is_normal_d(ctx, v) } else { emit_is_normal_s(ctx, v) }; + let fs_normal = is_normal(ctx, fs_bits); + let ft_normal = is_normal(ctx, ft_bits); let operands_normal = ctx.builder.ins().band(fs_normal, ft_normal); - let fast_block = ctx.builder.create_block(); + let compute_block = ctx.builder.create_block(); let slow_block = ctx.builder.create_block(); + let call_block = ctx.builder.create_block(); + let plain_block = ctx.builder.create_block(); let done_block = ctx.builder.create_block(); - ctx.builder.ins().brif(operands_normal, fast_block, &[], slow_block, &[]); + // The host's result, widened to 64 bits, whichever arm computed it. + let call_result = ctx.builder.append_block_param(call_block, i64t); + let plain_result = ctx.builder.append_block_param(plain_block, if is_d { i64t } else { i32t }); + ctx.builder.ins().brif(operands_normal, compute_block, &[], slow_block, &[]); - ctx.builder.switch_to_block(slow_block); - ctx.builder.set_cold_block(slow_block); - ctx.builder.seal_block(slow_block); - emit_check_denorm_operand(ctx, fs_bits, false); - emit_check_denorm_operand(ctx, ft_bits, false); - let flags = flags_fn(ctx, fs_bits, ft_bits); - let (result_bits, is_denorm, is_negative) = emit_fbinop_result_s(ctx, op, fs_bits, ft_bits); - emit_fpu_update_fcsr_arith( - ctx, Some(flags), is_denorm, is_negative, - |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode), - |ctx, neg| emit_write_signed_zero_s(ctx, fd, neg, fr_mode), - ); - ctx.builder.ins().jump(done_block, &[]); - - ctx.builder.switch_to_block(fast_block); - ctx.builder.seal_block(fast_block); - let (result_bits, is_denorm, is_negative) = emit_fbinop_result_s(ctx, op, fs_bits, ft_bits); - emit_fpu_update_fcsr_arith( - ctx, None, is_denorm, is_negative, - |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode), - |ctx, neg| emit_write_signed_zero_s(ctx, fd, neg, fr_mode), - ); - ctx.builder.ins().jump(done_block, &[]); - - ctx.builder.switch_to_block(done_block); - ctx.builder.seal_block(done_block); -} -fn emit_fbinop_d( - ctx: &mut EmitCtx, - fr_mode: FrMode, - op: fn(&mut FunctionBuilder, Value, Value) -> Value, - flags_fn: fn(&mut EmitCtx, Value, Value) -> Value, -) { - let raw = ctx.raw; - let fs = field_rd(raw); - let ft = field_rt(raw); - let fd = field_sa(raw); - - let fs_bits = emit_read_fpr_l(ctx, fs, fr_mode); - let ft_bits = emit_read_fpr_l(ctx, ft, fr_mode); - - let fs_normal = emit_is_normal_d(ctx, fs_bits); - let ft_normal = emit_is_normal_d(ctx, ft_bits); - let operands_normal = ctx.builder.ins().band(fs_normal, ft_normal); - let fast_block = ctx.builder.create_block(); - let slow_block = ctx.builder.create_block(); - let done_block = ctx.builder.create_block(); - ctx.builder.ins().brif(operands_normal, fast_block, &[], slow_block, &[]); + // The host result of the operation as the width's bit pattern. + let host_result = |ctx: &mut EmitCtx| { + if is_d { + emit_fbinop_result_d(ctx, op, fs_bits, ft_bits) + } else { + emit_fbinop_result_s(ctx, op, fs_bits, ft_bits) + } + }; + let widen = |ctx: &mut EmitCtx, v: Value| if is_d { v } else { ctx.builder.ins().uextend(i64t, v) }; + let is_zero = |ctx: &mut EmitCtx, v: Value| { + let abs = ctx.builder.ins().band_imm_s(v, if is_d { 0x7FFF_FFFF_FFFF_FFFFu64 as i64 } else { 0x7FFF_FFFF }); + ctx.builder.ins().icmp_imm_s(IntCC::Equal, abs, 0) + }; + // Cold: an operand is zero, denormal, infinite or a NaN. Zeros compute + // like ordinary numbers; the rest go to the call. ctx.builder.switch_to_block(slow_block); ctx.builder.set_cold_block(slow_block); ctx.builder.seal_block(slow_block); - emit_check_denorm_operand(ctx, fs_bits, true); - emit_check_denorm_operand(ctx, ft_bits, true); - let flags = flags_fn(ctx, fs_bits, ft_bits); - let (result_bits, is_denorm, is_negative) = emit_fbinop_result_d(ctx, op, fs_bits, ft_bits); - emit_fpu_update_fcsr_arith( - ctx, Some(flags), is_denorm, is_negative, - |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode), - |ctx, neg| emit_write_signed_zero_d(ctx, fd, neg, fr_mode), - ); + let fs_zero = is_zero(ctx, fs_bits); + let ft_zero = is_zero(ctx, ft_bits); + let fs_ok = ctx.builder.ins().bor(fs_normal, fs_zero); + let ft_ok = ctx.builder.ins().bor(ft_normal, ft_zero); + let operands_ok = ctx.builder.ins().band(fs_ok, ft_ok); + let unusual_block = ctx.builder.create_block(); + ctx.builder.ins().brif(operands_ok, compute_block, &[], unusual_block, &[]); + ctx.builder.seal_block(compute_block); + + ctx.builder.switch_to_block(unusual_block); + ctx.builder.set_cold_block(unusual_block); + ctx.builder.seal_block(unusual_block); + let result_bits = host_result(ctx); + let wide = widen(ctx, result_bits); + ctx.builder.ins().jump(call_block, &[ir::BlockArg::Value(wide)]); + + ctx.builder.switch_to_block(compute_block); + let result_bits = host_result(ctx); + // Double precision classifies on the high word, exactly as + // `fpu_result_at_top_d` does, so no 64-bit immediate sits on the hot + // path. + let (min_normal, top) = if is_d { (0x0010_0000, 0x7FEF_FFFF) } else { (0x0080_0000, 0x7F7F_FFFF) }; + let word = if is_d { + let hi = ctx.builder.ins().ushr_imm_s(result_bits, 32); + ctx.builder.ins().ireduce(i32t, hi) + } else { + result_bits + }; + let abs = ctx.builder.ins().band_imm_s(word, 0x7FFF_FFFF); + // abs < min_normal or abs >= top, in one unsigned compare. + let biased = ctx.builder.ins().iadd_imm_s(abs, -min_normal); + let special = ctx.builder.ins().icmp_imm_s(IntCC::UnsignedGreaterThanOrEqual, biased, top - min_normal); + let special_block = ctx.builder.create_block(); + ctx.builder.ins().brif(special, special_block, &[], plain_block, &[ir::BlockArg::Value(result_bits)]); + + // Cold: zero rejoins the plain write; a denormal, ±MAX, infinity or NaN + // goes to the call. + ctx.builder.switch_to_block(special_block); + ctx.builder.set_cold_block(special_block); + ctx.builder.seal_block(special_block); + let result_zero = is_zero(ctx, result_bits); + let wide = widen(ctx, result_bits); + ctx.builder.ins().brif(result_zero, plain_block, &[ir::BlockArg::Value(result_bits)], call_block, &[ir::BlockArg::Value(wide)]); + ctx.builder.seal_block(call_block); + ctx.builder.seal_block(plain_block); + + ctx.builder.switch_to_block(plain_block); + emit_fpu_update_fcsr_no_flags(ctx, |ctx| { + if is_d { emit_write_fpr_l(ctx, fd, plain_result, fr_mode) } else { emit_write_fpr_w(ctx, fd, plain_result, fr_mode) } + }); ctx.builder.ins().jump(done_block, &[]); - ctx.builder.switch_to_block(fast_block); - ctx.builder.seal_block(fast_block); - let (result_bits, is_denorm, is_negative) = emit_fbinop_result_d(ctx, op, fs_bits, ft_bits); - emit_fpu_update_fcsr_arith( - ctx, None, is_denorm, is_negative, - |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode), - |ctx, neg| emit_write_signed_zero_d(ctx, fd, neg, fr_mode), - ); + ctx.builder.switch_to_block(call_block); + ctx.builder.set_cold_block(call_block); + let packed = (ctx.raw & 0x3F) + | if is_d { 0x100 } else { 0 } + | if matches!(fr_mode, FrMode::Fr1) { 0x200 } else { 0 } + | (fd << 10) + | (fs << 15) + | (ft << 20); + let op_word = ctx.builder.ins().iconst(i32t, packed as i64); + let ptr_ty = ctx.module.target_config().pointer_type(); + let callee = emit_hook_callee(ctx, core_offset_of_fpu_binop_commit_fn()); + let mut sig = ctx.module.make_signature(); + sig.params.push(AbiParam::new(ptr_ty)); + sig.params.push(AbiParam::new(i32t)); + sig.params.push(AbiParam::new(i64t)); + sig.returns.push(AbiParam::new(i32t)); + let sig_ref = ctx.builder.import_signature(sig); + let core_arg = callout_core_arg(ctx); + let call = ctx.builder.ins().call_indirect(sig_ref, callee, &[core_arg, op_word, call_result]); + let trapped = ctx.builder.inst_results(call)[0]; + emit_trap_if_nonzero(ctx, trapped); ctx.builder.ins().jump(done_block, &[]); ctx.builder.switch_to_block(done_block); ctx.builder.seal_block(done_block); } +fn emit_fbinop_s(ctx: &mut EmitCtx, fr_mode: FrMode, op: fn(&mut FunctionBuilder, Value, Value) -> Value) { emit_fbinop(ctx, fr_mode, false, op) } +fn emit_fbinop_d(ctx: &mut EmitCtx, fr_mode: FrMode, op: fn(&mut FunctionBuilder, Value, Value) -> Value) { emit_fbinop(ctx, fr_mode, true, op) } fn fop_add(builder: &mut FunctionBuilder, a: Value, b: Value) -> Value { builder.ins().fadd(a, b) } fn fop_sub(builder: &mut FunctionBuilder, a: Value, b: Value) -> Value { builder.ins().fsub(a, b) } fn fop_mul(builder: &mut FunctionBuilder, a: Value, b: Value) -> Value { builder.ins().fmul(a, b) } fn fop_div(builder: &mut FunctionBuilder, a: Value, b: Value) -> Value { builder.ins().fdiv(a, b) } -fn emit_fadd_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_add, emit_fpu_arith_flags_snan_only_s); } -fn emit_fadd_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_add, emit_fpu_arith_flags_snan_only_d); } -fn emit_fsub_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_sub, emit_fpu_arith_flags_snan_only_s); } -fn emit_fsub_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_sub, emit_fpu_arith_flags_snan_only_d); } -fn emit_fmul_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_mul, emit_fpu_arith_flags_snan_only_s); } -fn emit_fmul_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_mul, emit_fpu_arith_flags_snan_only_d); } -fn emit_fdiv_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_div, emit_fpu_arith_flags_div_s); } -fn emit_fdiv_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_div, emit_fpu_arith_flags_div_d); } +fn emit_fadd_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_add); } +fn emit_fadd_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_add); } +fn emit_fsub_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_sub); } +fn emit_fsub_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_sub); } +fn emit_fmul_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_mul); } +fn emit_fmul_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_mul); } +fn emit_fdiv_s(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_s(ctx, fr_mode, fop_div); } +fn emit_fdiv_d(ctx: &mut EmitCtx, fr_mode: FrMode) { emit_fbinop_d(ctx, fr_mode, fop_div); } /// SQRT.S/D fd, fs: fd = sqrt(fs). Unlike the binary ops, only one operand /// (`fs`; `ft`/`d.rt` is unused/ignored, matching the interpreter). Still @@ -7829,6 +7986,7 @@ fn emit_fsqrt_s(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); let result = ctx.builder.ins().sqrt(fs_val); let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); + let (flags, result_bits) = emit_fix_invalid(ctx, false, flags, result_bits, &[fs_bits]); let is_denorm = emit_is_subnormal_s(ctx, result_bits); let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); emit_fpu_update_fcsr_arith( @@ -7876,6 +8034,7 @@ fn emit_fsqrt_d(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); let result = ctx.builder.ins().sqrt(fs_val); let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); + let (flags, result_bits) = emit_fix_invalid(ctx, true, flags, result_bits, &[fs_bits]); let is_denorm = emit_is_subnormal_d(ctx, result_bits); let is_negative = ctx.builder.ins().icmp_imm_s(IntCC::SignedLessThan, result_bits, 0); emit_fpu_update_fcsr_arith( @@ -10383,7 +10542,6 @@ fn emit_fternop_s(ctx: &mut EmitCtx, fr_mode: FrMode, negate_fr: bool, negate_re let fr_bits = emit_read_fpr_w(ctx, fr, fr_mode); let ft_bits = emit_read_fpr_w(ctx, ft, fr_mode); let fs_bits = emit_read_fpr_w(ctx, fs, fr_mode); - let flags = emit_fpu_arith_flags_snan_only3_s(ctx, fr_bits, ft_bits, fs_bits); let fr_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fr_bits); let ft_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), ft_bits); @@ -10397,8 +10555,11 @@ fn emit_fternop_s(ctx: &mut EmitCtx, fr_mode: FrMode, negate_fr: bool, negate_re }; let result = if negate_result { ctx.builder.ins().fneg(sum) } else { sum }; let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); - - emit_fpu_update_fcsr(ctx, flags, move |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode)); + emit_commit_finite_first( + ctx, false, result_bits, &[fr_bits, ft_bits, fs_bits], + |ctx| emit_fpu_arith_flags_snan_only3_s(ctx, fr_bits, ft_bits, fs_bits), + move |ctx, bits| emit_write_fpr_w(ctx, fd, bits, fr_mode), + ); } fn emit_fternop_d(ctx: &mut EmitCtx, fr_mode: FrMode, negate_fr: bool, negate_result: bool) { @@ -10411,7 +10572,6 @@ fn emit_fternop_d(ctx: &mut EmitCtx, fr_mode: FrMode, negate_fr: bool, negate_re let fr_bits = emit_read_fpr_l(ctx, fr, fr_mode); let ft_bits = emit_read_fpr_l(ctx, ft, fr_mode); let fs_bits = emit_read_fpr_l(ctx, fs, fr_mode); - let flags = emit_fpu_arith_flags_snan_only3_d(ctx, fr_bits, ft_bits, fs_bits); let fr_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fr_bits); let ft_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), ft_bits); @@ -10425,8 +10585,11 @@ fn emit_fternop_d(ctx: &mut EmitCtx, fr_mode: FrMode, negate_fr: bool, negate_re }; let result = if negate_result { ctx.builder.ins().fneg(sum) } else { sum }; let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); - - emit_fpu_update_fcsr(ctx, flags, move |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode)); + emit_commit_finite_first( + ctx, true, result_bits, &[fr_bits, ft_bits, fs_bits], + |ctx| emit_fpu_arith_flags_snan_only3_d(ctx, fr_bits, ft_bits, fs_bits), + move |ctx, bits| emit_write_fpr_l(ctx, fd, bits, fr_mode), + ); } // fd = fs*ft + fr @@ -10460,11 +10623,14 @@ fn emit_frecip_s(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs_bits = emit_read_fpr_w(ctx, fs, fr_mode); let one = ctx.builder.ins().f32const(1.0); let one_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), one); - let flags = emit_fpu_arith_flags_div_s(ctx, one_bits, fs_bits); let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); let result = ctx.builder.ins().fdiv(one, fs_val); let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); - emit_fpu_update_fcsr(ctx, flags, move |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode)); + emit_commit_finite_first( + ctx, false, result_bits, &[fs_bits], + |ctx| emit_fpu_arith_flags_div_s(ctx, one_bits, fs_bits), + move |ctx, bits| emit_write_fpr_w(ctx, fd, bits, fr_mode), + ); } fn emit_frecip_d(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs = field_rd(ctx.raw); @@ -10472,35 +10638,44 @@ fn emit_frecip_d(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs_bits = emit_read_fpr_l(ctx, fs, fr_mode); let one = ctx.builder.ins().f64const(1.0); let one_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), one); - let flags = emit_fpu_arith_flags_div_d(ctx, one_bits, fs_bits); let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); let result = ctx.builder.ins().fdiv(one, fs_val); let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); - emit_fpu_update_fcsr(ctx, flags, move |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode)); + emit_commit_finite_first( + ctx, true, result_bits, &[fs_bits], + |ctx| emit_fpu_arith_flags_div_d(ctx, one_bits, fs_bits), + move |ctx, bits| emit_write_fpr_l(ctx, fd, bits, fr_mode), + ); } fn emit_frsqrt_s(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs = field_rd(ctx.raw); let fd = field_sa(ctx.raw); let fs_bits = emit_read_fpr_w(ctx, fs, fr_mode); let one = ctx.builder.ins().f32const(1.0); - let flags = emit_fpu_arith_flags_rsqrt_s(ctx, fs_bits); let fs_val = ctx.builder.ins().bitcast(ir::types::F32, MemFlagsData::new(), fs_bits); let root = ctx.builder.ins().sqrt(fs_val); let result = ctx.builder.ins().fdiv(one, root); let result_bits = ctx.builder.ins().bitcast(ir::types::I32, MemFlagsData::new(), result); - emit_fpu_update_fcsr(ctx, flags, move |ctx| emit_write_fpr_w(ctx, fd, result_bits, fr_mode)); + emit_commit_finite_first( + ctx, false, result_bits, &[fs_bits], + |ctx| emit_fpu_arith_flags_rsqrt_s(ctx, fs_bits), + move |ctx, bits| emit_write_fpr_w(ctx, fd, bits, fr_mode), + ); } fn emit_frsqrt_d(ctx: &mut EmitCtx, fr_mode: FrMode) { let fs = field_rd(ctx.raw); let fd = field_sa(ctx.raw); let fs_bits = emit_read_fpr_l(ctx, fs, fr_mode); let one = ctx.builder.ins().f64const(1.0); - let flags = emit_fpu_arith_flags_rsqrt_d(ctx, fs_bits); let fs_val = ctx.builder.ins().bitcast(ir::types::F64, MemFlagsData::new(), fs_bits); let root = ctx.builder.ins().sqrt(fs_val); let result = ctx.builder.ins().fdiv(one, root); let result_bits = ctx.builder.ins().bitcast(ir::types::I64, MemFlagsData::new(), result); - emit_fpu_update_fcsr(ctx, flags, move |ctx| emit_write_fpr_l(ctx, fd, result_bits, fr_mode)); + emit_commit_finite_first( + ctx, true, result_bits, &[fs_bits], + |ctx| emit_fpu_arith_flags_rsqrt_d(ctx, fs_bits), + move |ctx, bits| emit_write_fpr_l(ctx, fd, bits, fr_mode), + ); } /// PREFX — indexed prefetch. A prefetch is architecturally a hint with no diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 315eaa286..09c0684d3 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -1274,10 +1274,27 @@ mod tests { fn run_interpreter_fpu_fcsr(instr: u32, gpr: [u64; 32], fpr: [u64; 32], pc: u64, fr1: bool, fcsr: Option) -> CoreSnapshot { let (mut exec, _mem) = fpu_seeded_executor(gpr, fpr, pc, fr1); if let Some(fcsr) = fcsr { exec.core.fpu_fcsr = fcsr; } + let _rounding = HostRounding::from_fcsr(fcsr); exec.exec(instr); CoreSnapshot::capture(&exec.core) } + /// Sets the host FPU's rounding mode to FCSR.RM for one run and puts + /// round-to-nearest back on drop. Building an executor resets the host + /// mode to nearest, so it has to be set after construction, once the + /// test's FCSR is in place. The interpreter and compiled code both + /// round on the host, so without this only round-to-nearest runs. + struct HostRounding; + impl HostRounding { + fn from_fcsr(fcsr: Option) -> Self { + crate::platform::set_fpu_mode((fcsr.unwrap_or(0) & 3) as u8); + HostRounding + } + } + impl Drop for HostRounding { + fn drop(&mut self) { crate::platform::set_fpu_mode(0); } + } + fn run_jit_fpu(instr: u32, gpr: [u64; 32], fpr: [u64; 32], pc: u64, word_offset: u16, fr1: bool) -> Option { run_jit_fpu_fcsr(instr, gpr, fpr, pc, word_offset, fr1, None) } @@ -1299,6 +1316,7 @@ mod tests { let mut exec = Box::new(exec); exec.install_jit_hooks(); if let Some(fcsr) = fcsr { exec.core.fpu_fcsr = fcsr; } + let _rounding = HostRounding::from_fcsr(fcsr); unsafe { jit_fn(&mut exec.core as *mut MipsCore) }; std::mem::forget(codegen); @@ -5496,6 +5514,323 @@ mod tests { } } + /// ADD/SUB/MUL/DIV on operands whose results reach the top of the + /// range (±MAX, ±infinity, NaN), in every rounding mode, trapped and + /// untrapped: the JIT's cold classifier call must agree with the + /// interpreter on result, Cause, Flags and the exception. Two absolute + /// pins tie the shared answer to the manuals: MAX+MAX in single + /// precision overflows with Cause and Flag O|I (VR5000 manual + /// U11761EJ6V0UM, 9.4.4), and a trapped 0/0 raises Invalid, not + /// divide-by-zero (9.4.2, 9.4.3). + #[test] + fn arithmetic_at_top_of_range_matches_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FADD, FUNCT_FDIV, FUNCT_FMUL, FUNCT_FSUB, OP_COP1, RS_D, RS_S}; + // One case per rule, since each compile costs tens of milliseconds: + // the invalid operations (untrapped and trapped), an overflow per + // operation in all four rounding modes and trapped, a largest-finite + // result that is exact, MAX+1 rounded toward zero, an exact + // infinity, and the zero and denormal results below. + // (zero, one, two, half, MAX, infinity) by format. + let s = [0u64, 0x3F80_0000, 0x4000_0000, 0x3F00_0000, 0x7F7F_FFFF, 0x7F80_0000]; + let d = [ + 0u64, 0x3FF0_0000_0000_0000, 0x4000_0000_0000_0000, 0x3FE0_0000_0000_0000, + 0x7FEF_FFFF_FFFF_FFFF, 0x7FF0_0000_0000_0000, + ]; + let mut cases: Vec<(u32, u32, u64, u64, u32)> = Vec::new(); + for (fmt, v, sign) in [(RS_S, s, 1u64 << 31), (RS_D, d, 1u64 << 63)] { + let [zero, one, two, half, max, inf] = v; + for enables in [0, 0x0F80] { + cases.push((fmt, FUNCT_FADD, inf, inf | sign, enables)); + cases.push((fmt, FUNCT_FSUB, inf, inf, enables)); + cases.push((fmt, FUNCT_FMUL, zero, inf, enables)); + cases.push((fmt, FUNCT_FDIV, zero, zero, enables)); + cases.push((fmt, FUNCT_FDIV, inf, inf, enables)); + } + for fcsr in [0, 1, 2, 3, 0x0200] { + cases.push((fmt, FUNCT_FADD, max, max, fcsr)); + cases.push((fmt, FUNCT_FSUB, max | sign, max, fcsr)); + cases.push((fmt, FUNCT_FMUL, max, two, fcsr)); + cases.push((fmt, FUNCT_FDIV, max | sign, half, fcsr)); + } + cases.push((fmt, FUNCT_FMUL, max, one, 1)); + cases.push((fmt, FUNCT_FMUL, max & !0xFFFF_FFFF, one, 0)); + cases.push((fmt, FUNCT_FADD, max, one, 1)); + cases.push((fmt, FUNCT_FADD, inf, one, 0)); + cases.push((fmt, FUNCT_FDIV, inf, zero, 0)); + // The same range test also sends zero and denormal results to + // the cold side: an exact cancellation, and the smallest normal + // halved with FS clear (trap). + let min_normal = if fmt == RS_S { 0x0080_0000 } else { 0x0010_0000_0000_0000 }; + cases.push((fmt, FUNCT_FSUB, one, one, 0)); + cases.push((fmt, FUNCT_FMUL, min_normal, half, 0)); + } + for (fmt, funct, a, b, fcsr) in cases { + let mut fpr = [0u64; 32]; + fpr[1] = a; + fpr[2] = b; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + let instr = make_r(OP_COP1, fmt, 2, 1, 3, funct); + assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, true, Some(fcsr)); + } + let mut fpr = [0u64; 32]; + fpr[1] = 0x7F7F_FFFF; + fpr[2] = 0x7F7F_FFFF; + let jit = assert_fpu_matches_interpreter_fcsr(make_r(OP_COP1, RS_S, 2, 1, 3, FUNCT_FADD), [0u64; 32], fpr, true, Some(0)); + assert_eq!((jit.fpr[3] as u32, jit.fpu_fcsr), (0x7F80_0000, 0x0000_5014)); + let fpr = [0u64; 32]; + let jit = assert_fpu_matches_interpreter_fcsr(make_r(OP_COP1, RS_S, 2, 1, 3, FUNCT_FDIV), [0u64; 32], fpr, true, Some(0x0F80)); + assert_eq!(jit.fpu_fcsr, 0x0001_0F80); + } + + /// An instruction that raises nothing still ends with Cause clear, and + /// the JIT does that on the cold side of its one combined test: a stale + /// Cause left by the previous instruction, ordinary and zero operands, + /// a zero result, in both formats, for ADD/SUB/MUL/DIV and for RECIP, + /// RSQRT and the multiply-add family, whose finite results take their own + /// single branch. + #[test] + fn stale_cause_is_cleared_by_an_ordinary_result() { + use crate::cpu::mips_isa::{ + FUNCT_FADD, FUNCT_FDIV, FUNCT_FMUL, FUNCT_FRECIP, FUNCT_FRSQRT, FUNCT_FSUB, + FUNCT_MADD_D, FUNCT_MADD_S, OP_COP1, OP_COP1X, RS_D, RS_S, + }; + let s = [0u64, 0x3F80_0000, 0x4000_0000]; + let d = [0u64, 0x3FF0_0000_0000_0000, 0x4000_0000_0000_0000]; + let mut all: Vec<(u32, u32, u64, u64, u32)> = Vec::new(); + for (fmt, v) in [(RS_S, s), (RS_D, d)] { + let [zero, one, two] = v; + for stale in [0u32, 0x0000_1000, 0x0001_0000, 0x0001_F000, 0x0001_F000 | 0x0F80] { + for funct in [FUNCT_FADD, FUNCT_FSUB, FUNCT_FMUL, FUNCT_FDIV] { + // Ordinary operands, then a zero operand. + all.push((fmt, funct, one, two, stale)); + all.push((fmt, funct, zero, two, stale)); + } + // An exact cancellation: a zero result from ordinary operands. + all.push((fmt, FUNCT_FSUB, one, one, stale)); + } + } + for (fmt, funct, a, b, fcsr) in all { + let mut fpr = [0u64; 32]; + fpr[1] = a; + fpr[2] = b; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + let instr = make_r(OP_COP1, fmt, 2, 1, 3, funct); + let jit = assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, true, Some(fcsr)); + assert_eq!(jit.fpu_fcsr & 0x0001_F000, 0, "fmt={fmt} funct={funct:#x} a={a:#x} b={b:#x} fcsr={fcsr:#x}"); + } + // f1 = 2.0, f4 = 1.0, f6 = 1.0: RECIP and RSQRT of f1 and f2*f4+f6. + for (fmt, madd, two, one) in [(RS_S, FUNCT_MADD_S, s[2], s[1]), (RS_D, FUNCT_MADD_D, d[2], d[1])] { + for stale in [0u32, 0x0000_1000, 0x0001_F000] { + let mut fpr = [0u64; 32]; + fpr[1] = two; + fpr[2] = two; + fpr[4] = one; + fpr[6] = one; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + for instr in [ + make_r(OP_COP1, fmt, 0, 1, 3, FUNCT_FRECIP), + make_r(OP_COP1, fmt, 0, 1, 3, FUNCT_FRSQRT), + make_r(OP_COP1X, 6, 4, 2, 3, madd), + ] { + let jit = assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, true, Some(stale)); + assert_eq!(jit.fpu_fcsr & 0x0001_F000, 0, "instr={instr:#010x} fcsr={stale:#x}"); + } + } + } + } + + /// The operand side of the same call-out: signed zeros, denormal, + /// quiet-NaN and signalling-NaN operands and negative values in every + /// pair for ADD/SUB/MUL/DIV, with the denormal-result rule under FS + /// clear, FS set, and FS set with Underflow or Inexact enabled, and + /// cancelling operands under every rounding mode. Then the signs of + /// zero results pinned absolutely, since agreement between the engines + /// would also hold if both lost a sign. + #[test] + fn arithmetic_with_unusual_operands_matches_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FADD, FUNCT_FDIV, FUNCT_FMUL, FUNCT_FSUB, OP_COP1, RS_D, RS_S}; + const FS: u32 = 0x0100_0000; + // zero, -zero, denormal, quiet NaN, signalling NaN, one, -one, + // smallest normal, half + let s = [0u64, 0x8000_0000, 1, 0x7FC0_0000, 0x7FA0_0000, 0x3F80_0000, 0xBF80_0000, 0x0080_0000, 0x3F00_0000]; + let d = [ + 0u64, 1 << 63, 1, 0x7FF8_0000_0000_0000, 0x7FF4_0000_0000_0000, 0x3FF0_0000_0000_0000, + 0xBFF0_0000_0000_0000, 0x0010_0000_0000_0000, 0x3FE0_0000_0000_0000, + ]; + let mut cases: Vec<(u32, u32, u64, u64, u32)> = Vec::new(); + for (fmt, v) in [(RS_S, s), (RS_D, d)] { + let [_, _, denorm, _, _, one, neg_one, min_normal, half] = v; + for funct in [FUNCT_FADD, FUNCT_FSUB, FUNCT_FMUL, FUNCT_FDIV] { + for &a in &v[..8] { + for &b in &v[..8] { + cases.push((fmt, funct, a, b, 0)); + cases.push((fmt, funct, a, b, FS)); + } + } + // A denormal result from ordinary operands, per FCSR. + for fcsr in [0, FS, FS | 0x100, FS | 0x80, FS | 0x0F80] { + cases.push((fmt, funct, min_normal, half, fcsr)); + cases.push((fmt, funct, denorm, one, fcsr)); + } + // Exact cancellation and signed zeros, per rounding mode. + for rm in 0..4 { + cases.push((fmt, funct, one, neg_one, rm)); + cases.push((fmt, funct, neg_one, neg_one, rm)); + cases.push((fmt, funct, neg_one, v[0], rm)); + cases.push((fmt, funct, v[1], neg_one, rm)); + } + } + } + for (fmt, funct, a, b, fcsr) in cases { + let mut fpr = [0u64; 32]; + fpr[1] = a; + fpr[2] = b; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + let instr = make_r(OP_COP1, fmt, 2, 1, 3, funct); + assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, true, Some(fcsr)); + } + + // Absolute pins, double: [fd] = [fs] op [ft]. + let run = |funct: u32, a: u64, b: u64, rm: u32| { + let mut fpr = [0u64; 32]; + fpr[1] = a; + fpr[2] = b; + let instr = make_r(OP_COP1, RS_D, 2, 1, 3, funct); + assert_fpu_matches_interpreter_fcsr(instr, [0u64; 32], fpr, true, Some(rm)).fpr[3] + }; + let (pz, nz, one, neg_one) = (0u64, 1u64 << 63, 0x3FF0_0000_0000_0000u64, 0xBFF0_0000_0000_0000u64); + // x + (-x) is +0, or -0 rounding toward minus infinity. + for rm in 0..4 { + let want = if rm == 3 { nz } else { pz }; + assert_eq!(run(FUNCT_FADD, one, neg_one, rm), want, "1 + -1, rm {rm}"); + assert_eq!(run(FUNCT_FSUB, one, one, rm), want, "1 - 1, rm {rm}"); + } + assert_eq!(run(FUNCT_FADD, nz, nz, 0), nz, "-0 + -0"); + assert_eq!(run(FUNCT_FMUL, neg_one, pz, 0), nz, "-1 * 0"); + assert_eq!(run(FUNCT_FDIV, pz, neg_one, 0), nz, "0 / -1"); + } + + /// Two FPU instructions back to back through the interpreter and through + /// one compiled region, FR=1, FCSR zero; returns the JIT snapshot after + /// asserting the two agree. + fn assert_fpu_pair_matches_interpreter(first: u32, second: u32, fpr: [u64; 32]) -> CoreSnapshot { + let pc = 0xFFFF_FFFF_8000_1000u64; + let word_offset = (pc as u16 / 4) & 0x3FF; + let (mut exec, _mem) = fpu_seeded_executor([0u64; 32], fpr, pc, true); + exec.exec(first); + exec.exec(second); + let interp = CoreSnapshot::capture(&exec.core); + + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[word_offset as usize] = first; + page[word_offset as usize + 1] = second; + let mut analyzer = Analyzer::new(); + let (walked, non_empty) = analyzer.walk_bounded(&page, word_offset, 0, 2); + assert!(non_empty); + let mut instrs_owned = *walked; + let mut codegen = Codegen::new(); + let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, word_offset, true, false) + .expect("FPU pair must be compilable for this test to be meaningful"); + let (exec, _mem) = fpu_seeded_executor([0u64; 32], fpr, pc, true); + let mut exec = Box::new(exec); + exec.install_jit_hooks(); + unsafe { jit_fn(&mut exec.core as *mut MipsCore) }; + std::mem::forget(codegen); + let jit = CoreSnapshot::capture(&exec.core); + assert_eq!(jit, interp, "JIT and interpreter diverged for {first:#010x}; {second:#010x}"); + jit + } + + /// RECIP and RSQRT of zero are a divide by zero: positive infinity with + /// Cause and Flag Z (VR5000 manual U11761EJ6V0UM, 9.4.3), nothing else. + /// Pinned absolutely, since the two engines compute the flags the same + /// way and an equivalence check alone would pass if both lost them. + #[test] + fn recip_and_rsqrt_of_zero_are_a_divide_by_zero() { + use crate::cpu::mips_isa::{FUNCT_FRECIP, FUNCT_FRSQRT, OP_COP1, RS_D, RS_S}; + for (fmt, inf) in [(RS_S, 0x7F80_0000u64), (RS_D, 0x7FF0_0000_0000_0000)] { + for funct in [FUNCT_FRECIP, FUNCT_FRSQRT] { + let mut fpr = [0u64; 32]; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + let jit = assert_fpu_matches_interpreter_fcsr(make_r(OP_COP1, fmt, 0, 7, 3, funct), [0u64; 32], fpr, true, Some(0)); + let got = if fmt == RS_S { jit.fpr[3] & 0xFFFF_FFFF } else { jit.fpr[3] }; + assert_eq!((got, jit.fpu_fcsr), (inf, 0x0000_8020), "fmt={fmt} funct={funct:#x}"); + } + } + } + + /// SQRT, RSQRT, the multiply-add family and CVT.S.D leave an Invalid + /// Operation's result as the default NaN, not the host's + /// (0xFFC00000 on x86, which has the sign set). The next instruction + /// must then see a quiet NaN and take Unimplemented (E) rather than + /// Invalid (V). Both engines, both formats. + #[test] + fn invalid_result_is_the_default_nan_and_reads_as_quiet() { + use crate::cpu::mips_isa::{ + FUNCT_FADD, FUNCT_FCVT_S, FUNCT_FRECIP, FUNCT_FRSQRT, FUNCT_FSQRT, FUNCT_MADD_D, + FUNCT_MADD_S, FUNCT_MSUB_D, FUNCT_MSUB_S, FUNCT_NMADD_D, FUNCT_NMADD_S, FUNCT_NMSUB_D, + FUNCT_NMSUB_S, OP_COP1, OP_COP1X, RS_D, RS_S, + }; + const NAN_S: u64 = 0x7FC0_0000; + const NAN_D: u64 = 0x7FF8_0000_0000_0000; + // f1 = -1, f2 = infinity, f4 = zero, f6 = 1.0, f7 = a signalling + // NaN (quiet bit clear), by format. + let single = |f: &mut [u64; 32]| { + f[1] = 0xBF80_0000; f[2] = 0x7F80_0000; f[4] = 0; f[6] = 0x3F80_0000; + f[7] = 0x7FA0_0000; + }; + let double = |f: &mut [u64; 32]| { + f[1] = 0xBFF0_0000_0000_0000; f[2] = 0x7FF0_0000_0000_0000; f[4] = 0; + f[6] = 0x3FF0_0000_0000_0000; f[7] = 0x7FF4_0000_0000_0000; + }; + // (producer, double result, FPRs); every producer writes f3. + let mut cases: Vec<(u32, bool, [u64; 32])> = Vec::new(); + for (fmt, is_d, init) in [(RS_S, false, &single as &dyn Fn(&mut [u64; 32])), (RS_D, true, &double)] { + let mut f = [0u64; 32]; + init(&mut f); + cases.push((make_r(OP_COP1, fmt, 0, 1, 3, FUNCT_FSQRT), is_d, f)); + cases.push((make_r(OP_COP1, fmt, 0, 1, 3, FUNCT_FRSQRT), is_d, f)); + // RECIP of a signalling NaN: the operand's V is already flagged, + // the delivered value must still be the default NaN. + cases.push((make_r(OP_COP1, fmt, 0, 7, 3, FUNCT_FRECIP), is_d, f)); + // f3 = f2 * f4 +/- f6, negated for NMADD and NMSUB: infinity + // times zero. + let functs = if is_d { + [FUNCT_MADD_D, FUNCT_MSUB_D, FUNCT_NMADD_D, FUNCT_NMSUB_D] + } else { + [FUNCT_MADD_S, FUNCT_MSUB_S, FUNCT_NMADD_S, FUNCT_NMSUB_S] + }; + for funct in functs { + cases.push((make_r(OP_COP1X, 6, 4, 2, 3, funct), is_d, f)); + } + } + let mut f = [0u64; 32]; + double(&mut f); + cases.push((make_r(OP_COP1, RS_D, 0, 7, 3, FUNCT_FCVT_S), false, f)); + + for (producer, is_d, fpr) in cases { + let what = format!("producer {producer:#010x}"); + let jit = assert_fpu_matches_interpreter_fcsr(producer, [0u64; 32], fpr, true, Some(0)); + let want = if is_d { NAN_D } else { NAN_S }; + let got = if is_d { jit.fpr[3] } else { jit.fpr[3] & 0xFFFF_FFFF }; + assert_eq!(got, want, "{what}: default NaN"); + assert_eq!(jit.fpu_fcsr, 0x0001_0040, "{what}: Cause and Flag V"); + + // f5 = f3 + f6: the NaN operand is quiet, so Unimplemented, and + // never Invalid. The cvt's result is single even from a double. + let fmt = if is_d { RS_D } else { RS_S }; + let add = make_r(OP_COP1, fmt, 6, 3, 5, FUNCT_FADD); + let jit = assert_fpu_pair_matches_interpreter(producer, add, fpr); + assert_eq!(jit.fpu_fcsr & 0x0003_F000, 0x0002_0000, "{what}: ADD takes E, not V"); + + // With Enable.V set the same instruction traps: Cause V, the + // Flags unchanged, and f3 (zero here) not written. + let jit = assert_fpu_matches_interpreter_fcsr(producer, [0u64; 32], fpr, true, Some(0x0800)); + assert_eq!(jit.fpu_fcsr, 0x0001_0800, "{what}: trapped V"); + assert_eq!(jit.fpr[3], 0, "{what}: trapped result not stored"); + } + } + fn funop_case(funct: u32, fmt: u32, fs_val: f64) { let mut fpr = [0u64; 32]; if fmt == crate::cpu::mips_isa::RS_S { diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 068f7dab0..31ffe8b33 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -722,6 +722,12 @@ pub struct MipsCore { /// `ctx`: `fs_reg, fd_reg, fr1` (bool as u32). Returns nonzero if trapped. #[cfg(feature = "jitv2")] pub fpu_cvt_d_to_s_fn: unsafe extern "C" fn(*mut core::ffi::c_void, u32, u32, u32) -> u32, + /// ADD/SUB/MUL/DIV whose result is at the top of the range — see + /// `fpu_binop_commit`. Args after `ctx`: the packed instruction + /// fields and the result bits (single precision zero-extended). + /// Classifies, updates FCSR and writes fd. Returns nonzero if trapped. + #[cfg(feature = "jitv2")] + pub fpu_binop_commit_fn: unsafe extern "C" fn(*mut core::ffi::c_void, u32, u64) -> u32, /// `jitv2_lockstep` load/store verification scratch (see /// `MipsExecutor::lockstep_check_load_store`): records the real /// interpreter dispatch's virtual address, translated physical address, @@ -1275,6 +1281,10 @@ unsafe extern "C" fn jit_hooks_not_installed_fpu_cvt_int_to_float(_ctx: *mut cor unsafe extern "C" fn jit_hooks_not_installed_fpu_cvt_d_to_s(_ctx: *mut core::ffi::c_void, _fs: u32, _fd: u32, _fr1: u32) -> u32 { panic!("jitv2: fpu_cvt_d_to_s hook called before MipsExecutor::install_jit_hooks"); } +#[cfg(feature = "jitv2")] +unsafe extern "C" fn jit_hooks_not_installed_fpu_binop_commit(_ctx: *mut core::ffi::c_void, _op: u32, _r: u64) -> u32 { + panic!("jitv2: fpu_binop_commit hook called before MipsExecutor::install_jit_hooks"); +} // SAFETY: The raw pointer in status_changed_cb is only accessed from the CPU thread. unsafe impl Send for MipsCore {} @@ -1377,6 +1387,8 @@ impl MipsCore { fpu_cvt_int_to_float_fn: jit_hooks_not_installed_fpu_cvt_int_to_float, #[cfg(feature = "jitv2")] fpu_cvt_d_to_s_fn: jit_hooks_not_installed_fpu_cvt_d_to_s, + #[cfg(feature = "jitv2")] + fpu_binop_commit_fn: jit_hooks_not_installed_fpu_binop_commit, #[cfg(feature = "jitv2_lockstep")] lockstep_step_fn: jit_hooks_not_installed_lockstep_step, #[cfg(feature = "jitv2_lockstep")] diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 62aaee8de..32f78f4e3 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -193,6 +193,202 @@ fn is_qnan_d(bits: u64) -> bool { f64::from_bits(bits).is_nan() && !is_snan_d(bits) } +/// The quiet NaN delivered for an untrapped Invalid Operation: the IEEE +/// 754-2008 default NaN, quiet bit set and nothing else. +const FPU_DEFAULT_NAN_S: u32 = 0x7FC0_0000; +const FPU_DEFAULT_NAN_D: u64 = 0x7FF8_0000_0000_0000; + +/// SQRT, RECIP, RSQRT and the multiply-add family have no top-of-range +/// classifier like ADD/SUB/MUL/DIV, but the host NaN they produce for an +/// Invalid Operation (x86: 0xFFC00000) is a signalling NaN under the MIPS +/// encoding, so a guest that feeds it to the next instruction would see +/// Invalid where hardware takes Unimplemented. A NaN result is Invalid when +/// an operand was a signalling NaN (`flags` already has V) or when no +/// operand was a NaN at all (infinity times zero, infinity minus infinity, +/// the square root of a negative number). The latter raises V as well. A +/// quiet NaN operand is left to propagate as it did. Overflow and Inexact +/// are not computed for these operations. +/// +/// One compare of the result's magnitude settles the ordinary case: a +/// finite result means no operand was a NaN (a NaN operand makes the host +/// result a NaN), no divisor was zero and no operand was negative for a +/// square root, so there is nothing to flag and `flags`, which would +/// classify the operands, is not even evaluated. Infinities and NaNs take +/// the cold call. Returns the flags and the bits to deliver. +#[inline(always)] +fn fpu_fix_invalid_s(flags: impl FnOnce() -> u32, bits: u32, ops: [u32; N]) -> (u32, u32) { + if bits & 0x7FFF_FFFF < 0x7F80_0000 { + return (0, bits); + } + fpu_invalid_nan_s(flags(), bits, &ops) +} +#[cold] +#[inline(never)] +fn fpu_invalid_nan_s(flags: u32, bits: u32, ops: &[u32]) -> (u32, u32) { + if bits & 0x7FFF_FFFF <= 0x7F80_0000 { + return (flags, bits); + } + if flags & FCSR_FV != 0 || !ops.iter().any(|&o| o & 0x7FFF_FFFF > 0x7F80_0000) { + (flags | FCSR_FV, FPU_DEFAULT_NAN_S) + } else { + (flags, bits) + } +} +#[inline(always)] +fn fpu_fix_invalid_d(flags: impl FnOnce() -> u32, bits: u64, ops: [u64; N]) -> (u32, u64) { + if bits & 0x7FFF_FFFF_FFFF_FFFF < 0x7FF0_0000_0000_0000 { + return (0, bits); + } + fpu_invalid_nan_d(flags(), bits, &ops) +} +#[cold] +#[inline(never)] +fn fpu_invalid_nan_d(flags: u32, bits: u64, ops: &[u64]) -> (u32, u64) { + if bits & 0x7FFF_FFFF_FFFF_FFFF <= 0x7FF0_0000_0000_0000 { + return (flags, bits); + } + if flags & FCSR_FV != 0 || !ops.iter().any(|&o| o & 0x7FFF_FFFF_FFFF_FFFF > 0x7FF0_0000_0000_0000) { + (flags | FCSR_FV, FPU_DEFAULT_NAN_D) + } else { + (flags, bits) + } +} + +/// The result of ADD/SUB/MUL/DIV is ±MAX, ±infinity or a NaN: the only +/// results that can carry an Invalid Operation or an Overflow. The +/// classification lives in `fpu_binop_top_flags_s`/`_d`. +#[inline(always)] +fn fpu_result_at_top_s(bits: u32) -> bool { + (bits & 0x7FFF_FFFF) >= 0x7F7F_FFFF +} +/// Double precision tests the high word only, which keeps 64-bit +/// immediates off the hot path. It also takes the finite values just below +/// MAX that share its high word; `fpu_binop_top_flags_d` finds no overflow +/// for those. +#[inline(always)] +fn fpu_result_at_top_d(bits: u64) -> bool { + ((bits >> 32) as u32 & 0x7FFF_FFFF) >= 0x7FEF_FFFF +} + +/// Flags [6:2] for an ADD/SUB/MUL/DIV.S (`funct`) whose host result `r` is +/// at the top of the range (`fpu_result_at_top_s`). Nothing is passed in +/// from the operand checks: of the flags they could find, a signalling NaN +/// makes the result a NaN, and a divide by zero is found here. +/// +/// - A NaN result from operands that reach here (quiet NaNs and denormals +/// have already trapped) is an Invalid Operation: a signalling-NaN operand, +/// infinity minus infinity, zero times infinity, 0/0 or infinity/infinity +/// (VR5000 manual, 9.4.2). The caller delivers `FPU_DEFAULT_NAN_S`. +/// - An infinity or ±MAX from finite operands is an Overflow when the +/// rounded result with an unbounded exponent exceeds the largest finite +/// number, and Overflow also raises Inexact (VR5000 manual, 9.4.4 and +/// Table 9-2). The host already rounded to the right value for the +/// current mode (Table 9-1: infinity, or the largest finite number when +/// the mode rounds toward zero for that sign); only the flags are +/// missing. An infinity is always an overflow. ±MAX is one only when the +/// exact result reaches 2^128, which the operation repeated in double +/// precision shows: exact for ADD/SUB/MUL, monotonic for DIV, and 2^128 +/// is representable, so rounding in the same mode cannot cross it. +#[cold] +#[inline(never)] +fn fpu_binop_top_flags_s(funct: u32, a: u32, b: u32, r: u32) -> u32 { + let (fa, fb, fr) = (f32::from_bits(a), f32::from_bits(b), f32::from_bits(r)); + if fr.is_nan() { + return FCSR_FV; + } + // An infinite operand gives an exact infinity, which is no overflow, and + // a finite nonzero number over zero a correctly signed one that raises + // Divide by Zero (0/0 is a NaN and has returned). + if !fa.is_finite() || !fb.is_finite() { + return 0; + } + if funct == FUNCT_FDIV && fb == 0.0 { + return FCSR_FZ; + } + let overflow = fr.is_infinite() || { + let (a, b) = (fa as f64, fb as f64); + let x = match funct { + FUNCT_FADD => a + b, + FUNCT_FSUB => a - b, + FUNCT_FMUL => a * b, + _ => a / b, + }; + x.abs() >= f64::from_bits(0x47F0_0000_0000_0000) // 2^128 + }; + if overflow { FCSR_FO | FCSR_FI } else { 0 } +} + +/// `fpu_binop_top_flags_s` for double precision, also called for the finite +/// results just below MAX that `fpu_result_at_top_d` lets through; the +/// halved check below finds no overflow for those. There is no wider host +/// format, so a ±MAX result is checked by halving instead: the exact result +/// reaches 2^1024 exactly when the halved operation reaches 2^1023. Halving +/// a normal operand is exact (denormal operands have already trapped); for +/// MUL only the larger operand is halved, so the smaller one cannot lose a +/// bit, and for ADD/SUB a bit lost from a tiny operand cannot move a sum +/// near 2^1023 across it. +#[cold] +#[inline(never)] +fn fpu_binop_top_flags_d(funct: u32, a: u64, b: u64, r: u64) -> u32 { + let (fa, fb, fr) = (f64::from_bits(a), f64::from_bits(b), f64::from_bits(r)); + if fr.is_nan() { + return FCSR_FV; + } + if !fa.is_finite() || !fb.is_finite() { + return 0; + } + if funct == FUNCT_FDIV && fb == 0.0 { + return FCSR_FZ; + } + let overflow = fr.is_infinite() || { + let x = match funct { + FUNCT_FADD => fa * 0.5 + fb * 0.5, + FUNCT_FSUB => fa * 0.5 - fb * 0.5, + FUNCT_FMUL if fa.abs() >= fb.abs() => (fa * 0.5) * fb, + FUNCT_FMUL => fa * (fb * 0.5), + _ => (fa * 0.5) / fb, + }; + x.abs() >= f64::from_bits(0x7FE0_0000_0000_0000) // 2^1023 + }; + if overflow { FCSR_FO | FCSR_FI } else { 0 } +} + +/// Classify a computed ADD/SUB/MUL/DIV.S result in one range test: an +/// ordinary normal result passes straight through. Zero and denormals get +/// the `result_is_denorm` that `fpu_update_fcsr_full` takes, and the top of +/// the range goes to the cold `fpu_binop_top_flags_s`. Returns the flags, +/// the bits to deliver (the default NaN on Invalid) and that +/// `result_is_denorm`. The one test replaces the separate denormal-result +/// test, so ordinary arithmetic pays nothing for the rule. Only a `u32` +/// crosses the cold call, so the handlers need no stack frame for it. +#[inline(always)] +fn fpu_binop_finish_s(funct: u32, a: u32, b: u32, r: u32) -> (u32, u32, Option) { + let abs = r & 0x7FFF_FFFF; + if abs.wrapping_sub(0x0080_0000) < 0x7F7F_FFFF - 0x0080_0000 { + return (0, r, None); + } + if fpu_result_at_top_s(r) { + let flags = fpu_binop_top_flags_s(funct, a, b, r); + return (flags, if flags & FCSR_FV != 0 { FPU_DEFAULT_NAN_S } else { r }, None); + } + (0, r, (abs != 0).then_some(r & 0x8000_0000 != 0)) +} +#[inline(always)] +fn fpu_binop_finish_d(funct: u32, a: u64, b: u64, r: u64) -> (u32, u64, Option) { + // On the high word, as in `fpu_result_at_top_d`: below 0x00100000 is + // zero or denormal. + let hi = (r >> 32) as u32 & 0x7FFF_FFFF; + if hi.wrapping_sub(0x0010_0000) < 0x7FEF_FFFF - 0x0010_0000 { + return (0, r, None); + } + let abs = r & 0x7FFF_FFFF_FFFF_FFFF; + if fpu_result_at_top_d(r) { + let flags = fpu_binop_top_flags_d(funct, a, b, r); + return (flags, if flags & FCSR_FV != 0 { FPU_DEFAULT_NAN_D } else { r }, None); + } + (0, r, (abs != 0).then_some(r & 0x8000_0000_0000_0000 != 0)) +} + /// Every ADD/SUB/MUL/DIV/SQRT/RECIP/RSQRT/MADD-family handler's exception /// checks are computed from operand/result bit patterns alone — never from /// the host FPU's own exception flags (see @@ -1634,6 +1830,93 @@ unsafe extern "C" fn jit_cvt_d_to_s(ctx: *mut core::ffi::c_ cvt_d_to_s_and_commit(&mut exec.core, fs_reg, fd_reg, fr1 != 0) as u32 } +/// Everything about an ADD/SUB/MUL/DIV that compiled code does not handle +/// inline: operands that are not ordinary finite numbers, and results that +/// are zero, denormal or at the top of the range. Compiled code computes the +/// result `r` with the host FPU and calls this for the rest, so the cold +/// side of each instruction is one call and a trap check. Does what +/// `exec_fadd_d` and its siblings do after the arithmetic, with their own +/// helpers: a denormal or quiet-NaN operand is an Unimplemented Operation +/// (`fpu_check_denorm_operand_s/d`), `fpu_binop_finish_s/d` classifies the +/// result, and the commit is `fpu_update_fcsr_full`'s: Cause rewritten, a +/// denormal result flushed or Unimplemented, a trap leaving fd and the Flags +/// alone, otherwise fd gets the result (the default NaN on Invalid) and the +/// Flags accumulate. The operands are read back from the FPRs, which the +/// instruction has not written yet. Taking just the packed instruction and +/// the result keeps the call's argument registers, and so the compiled +/// region's register pressure, small. `op` packs funct (bits 0..5), double +/// (bit 8), FR=1 (bit 9), fd (bits 10..14), fs (15..19) and ft (20..24). +/// Returns true if the instruction traps. +#[cfg(feature = "jitv2")] +fn fpu_binop_commit(core: &mut MipsCore, op: u32, r: u64) -> bool { + let funct = op & 0x3F; + let is_d = op & 0x100 != 0; + let fr1 = op & 0x200 != 0; + let (fd, fs, ft) = ((op >> 10) & 0x1F, (op >> 15) & 0x1F, (op >> 20) & 0x1F); + let (a, b) = if is_d { + let rd = |c: &MipsCore, n| if fr1 { c.read_fpr_l(n) } else { read_fpr_l_fr0(c, n) }; + (rd(core, fs), rd(core, ft)) + } else { + let rd = |c: &MipsCore, n| u64::from(if fr1 { c.read_fpr_w(n) } else { read_fpr_w_fr0(c, n) }); + (rd(core, fs), rd(core, ft)) + }; + // Cause holds only this instruction's exceptions, so it is rewritten on + // every path out of here. + let operand_unimplemented = |bits: u64| { + if is_d { + let e = bits & 0x7FF0_0000_0000_0000; + (e == 0 || e == 0x7FF0_0000_0000_0000) && (f64::from_bits(bits).is_subnormal() || is_qnan_d(bits)) + } else { + let e = bits as u32 & 0x7F80_0000; + (e == 0 || e == 0x7F80_0000) && (f32::from_bits(bits as u32).is_subnormal() || is_qnan_s(bits as u32)) + } + }; + if operand_unimplemented(a) || operand_unimplemented(b) { + core.fpu_fcsr = (core.fpu_fcsr & !FCSR_CM) | FCSR_CE; + return true; + } + let (flags, bits, denorm) = if is_d { + fpu_binop_finish_d(funct, a, b, r) + } else { + let (f, bits, d) = fpu_binop_finish_s(funct, a as u32, b as u32, r as u32); + (f, u64::from(bits), d) + }; + core.fpu_fcsr &= !FCSR_CM; + let bits = if let Some(negative) = denorm { + // Table 9-1/7-1: with FS clear, or U or I enabled, a denormal result + // is an Unimplemented Operation and fd is untouched; otherwise it is + // flushed to a signed zero with Cause and Flag U and I set. + if core.fpu_fcsr & FCSR_FS == 0 || core.fpu_fcsr & (FCSR_UM | FCSR_IM) != 0 { + core.fpu_fcsr |= FCSR_CE; + return true; + } + core.fpu_fcsr |= FCSR_CU | FCSR_CI; + let sign = if is_d { 63 } else { 31 }; + u64::from(negative) << sign + } else { + bits + }; + let causes = (flags & FCSR_FM) << 10; + core.fpu_fcsr |= causes; + if ((causes >> 5) & (core.fpu_fcsr & FCSR_EM)) != 0 { + return true; + } + match (is_d, fr1) { + (true, true) => core.write_fpr_l(fd, bits), + (true, false) => write_fpr_l_fr0(core, fd, bits), + (false, true) => core.write_fpr_w(fd, bits as u32), + (false, false) => write_fpr_w_fr0(core, fd, bits as u32), + } + core.fpu_fcsr |= if denorm.is_some() { FCSR_FU | FCSR_FI } else { flags & FCSR_FM }; + false +} + +#[cfg(feature = "jitv2")] +unsafe extern "C" fn jit_fpu_binop_commit(ctx: *mut core::ffi::c_void, op: u32, r: u64) -> u32 { + let core = unsafe { &mut *core_from_arg(ctx) }; + fpu_binop_commit(core, op, r) as u32 +} + /// Inline lockstep step hook (`core.lockstep_step_fn`), called by compiled code /// at the START of every JIT instruction with its synthesized `pc`/`raw`/`bd`. /// Establishes the true per-instruction starting state, runs the instruction @@ -2034,6 +2317,19 @@ fn cvt_d_to_s_and_commit(core: &mut MipsCore, fs_reg: u32, fd_reg: u32, fr1: boo if fr1 { core.write_fpr_w(fd_reg, bits) } else { write_fpr_w_fr0(core, fd_reg, bits) } return false; } + // A NaN source never takes the path above (it is not equal to itself), + // so the signalling test costs an ordinary conversion nothing. A + // signalling NaN is an Invalid Operation: V, and the default NaN + // rather than the host's. A quiet NaN converts as before. + if is_snan_d(src.to_bits()) { + core.fpu_fcsr |= FCSR_CV; + if core.fpu_fcsr & 0x800 != 0 { // Enable.V = bit 11 + return true; + } + if fr1 { core.write_fpr_w(fd_reg, FPU_DEFAULT_NAN_S) } else { write_fpr_w_fr0(core, fd_reg, FPU_DEFAULT_NAN_S) } + core.fpu_fcsr |= FCSR_FV; + return false; + } let cause = if overflow { FCSR_CO } else { FCSR_CI }; core.fpu_fcsr |= cause; let enable_bit = if overflow { 0x200 } else { FCSR_IM }; // Enable.O = bit 9 @@ -2699,6 +2995,7 @@ impl MipsExecutor { self.core.fpu_cvt_to_int_fn = jit_cvt_to_int::; self.core.fpu_cvt_int_to_float_fn = jit_cvt_int_to_float::; self.core.fpu_cvt_d_to_s_fn = jit_cvt_d_to_s::; + self.core.fpu_binop_commit_fn = jit_fpu_binop_commit; #[cfg(feature = "jitv2_lockstep")] { self.core.lockstep_step_fn = jit_lockstep_step::; @@ -7542,11 +7839,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// /// `flags`: bits [6:2] (FV,FZ,FO,FU,FI) this instruction raised, computed /// by the caller from operand/result bit patterns — e.g. `FCSR_FV` for a - /// signalling-NaN operand, `FCSR_FZ` for divide-by-zero. Always 0 for - /// plain ADD/SUB/MUL/DIV/SQRT/RECIP/RSQRT/MADD-family unless one of those - /// specific conditions applies — Inexact and Overflow are deliberately - /// never set here (see the same rules note: not worth the IR to compute - /// for ordinary arithmetic, and nothing in IRIX depends on them). + /// signalling-NaN operand, `FCSR_FZ` for divide-by-zero, and for + /// ADD/SUB/MUL/DIV the Invalid and Overflow+Inexact that + /// `fpu_binop_top_flags_s/d` find in a result at the top of the range. + /// Inexact on its own is deliberately never set (see the same rules + /// note: not worth the IR to compute for ordinary arithmetic, and + /// nothing in IRIX depends on it). /// /// Always used as the terminal tail call of an FPU arithmetic exec_* /// handler (`self.fpu_update_fcsr(flags, |e| ...)`, nothing runs after), @@ -7656,9 +7954,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", (v.is_subnormal() || is_qnan_d(bits)).then(|| self.fpu_unimplemented()) } - /// Flags [6:2] for ADD/SUB/MUL (S or D): Invalid if either operand is a - /// signalling NaN (the case `fpu_check_denorm_operand_s/d`'s qNaN check - /// doesn't catch), else 0 — feeds `fpu_update_fcsr`/`_full`. See + /// Flags [6:2] for a two-operand op (S or D): Invalid if either operand + /// is a signalling NaN (the case `fpu_check_denorm_operand_s/d`'s qNaN + /// check doesn't catch), else 0. ADD/SUB/MUL/DIV take Invalid from their + /// NaN result instead (`fpu_binop_top_flags_s/d`), so only RECIP, through + /// the divide rule below, still asks. See /// `clanker/jitv2/fpu-flags-are-computed-not-read.md`. #[inline] fn fpu_arith_flags_snan_only_s(fs_bits: u32, ft_bits: u32) -> u32 { @@ -7680,27 +7980,27 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", if is_snan_d(a_bits) || is_snan_d(b_bits) || is_snan_d(c_bits) { FCSR_FV } else { 0 } } - /// Flags [6:2] for DIV/RECIP (S or D): Invalid takes priority over - /// divide-by-zero (a NaN dividend/divisor is Invalid, never Z), else Z - /// when the divisor is zero and the dividend is a finite non-NaN value — - /// matches the R4000 manual's Divide-by-zero definition ("the divisor is - /// zero and the dividend is a finite nonzero number"; zero/zero and - /// inf/zero are Invalid on real hardware, but qNaN operand already - /// traps via `fpu_check_denorm_operand_s/d` before this runs, and this - /// emulator doesn't distinguish 0/0 from finite/0 — both give Z, which - /// is the "gross problem" this exists to catch, not hardware-exact - /// classification of every zero/zero corner case). + /// Flags [6:2] for DIV/RECIP (S or D): Invalid for a signalling-NaN + /// operand, else Z when the divisor is zero and the dividend is a finite + /// nonzero number (VR5000 manual, 9.4.3). 0/0 is + /// an Invalid Operation, not a divide by zero (9.4.2); its NaN result is + /// classified by `fpu_binop_top_flags_s/d`. Infinity/0 is an exact + /// infinity and raises nothing. A quiet-NaN operand has already trapped. #[inline] fn fpu_arith_flags_div_s(fs_bits: u32, ft_bits: u32) -> u32 { let snan = Self::fpu_arith_flags_snan_only_s(fs_bits, ft_bits); if snan != 0 { return snan; } - if is_pos_zero_or_neg_zero_s(ft_bits) { FCSR_FZ } else { 0 } + if !is_pos_zero_or_neg_zero_s(ft_bits) { return 0; } + let dividend = fs_bits & 0x7FFF_FFFF; + if dividend != 0 && dividend < 0x7F80_0000 { FCSR_FZ } else { 0 } } #[inline] fn fpu_arith_flags_div_d(fs_bits: u64, ft_bits: u64) -> u32 { let snan = Self::fpu_arith_flags_snan_only_d(fs_bits, ft_bits); if snan != 0 { return snan; } - if is_pos_zero_or_neg_zero_d(ft_bits) { FCSR_FZ } else { 0 } + if !is_pos_zero_or_neg_zero_d(ft_bits) { return 0; } + let dividend = fs_bits & 0x7FFF_FFFF_FFFF_FFFF; + if dividend != 0 && dividend < 0x7FF0_0000_0000_0000 { FCSR_FZ } else { 0 } } /// Flags [6:2] for SQRT/RSQRT (S or D): Invalid for a signalling-NaN @@ -7828,11 +8128,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, fs_reg); let ft_bits = (self.fpr_read_w)(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_s(fs_bits).or_else(|| self.fpu_check_denorm_operand_s(ft_bits)) { return s; } - let flags = Self::fpu_arith_flags_snan_only_s(fs_bits, ft_bits); let result = f32::from_bits(fs_bits) + f32::from_bits(ft_bits); + let (flags, bits, denorm) = fpu_binop_finish_s(FUNCT_FADD, fs_bits, ft_bits, result.to_bits()); let write_w = self.fpr_write_w; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { (result).to_bits() })) + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { bits })) } fn exec_fsub_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -7840,11 +8139,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, fs_reg); let ft_bits = (self.fpr_read_w)(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_s(fs_bits).or_else(|| self.fpu_check_denorm_operand_s(ft_bits)) { return s; } - let flags = Self::fpu_arith_flags_snan_only_s(fs_bits, ft_bits); let result = f32::from_bits(fs_bits) - f32::from_bits(ft_bits); + let (flags, bits, denorm) = fpu_binop_finish_s(FUNCT_FSUB, fs_bits, ft_bits, result.to_bits()); let write_w = self.fpr_write_w; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { (result).to_bits() })) + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { bits })) } fn exec_fmul_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -7852,11 +8150,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, fs_reg); let ft_bits = (self.fpr_read_w)(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_s(fs_bits).or_else(|| self.fpu_check_denorm_operand_s(ft_bits)) { return s; } - let flags = Self::fpu_arith_flags_snan_only_s(fs_bits, ft_bits); let result = f32::from_bits(fs_bits) * f32::from_bits(ft_bits); + let (flags, bits, denorm) = fpu_binop_finish_s(FUNCT_FMUL, fs_bits, ft_bits, result.to_bits()); let write_w = self.fpr_write_w; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { (result).to_bits() })) + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { bits })) } fn exec_fdiv_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -7864,27 +8161,26 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, fs_reg); let ft_bits = (self.fpr_read_w)(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_s(fs_bits).or_else(|| self.fpu_check_denorm_operand_s(ft_bits)) { return s; } - let flags = Self::fpu_arith_flags_div_s(fs_bits, ft_bits); let fs_val = f32::from_bits(fs_bits); let ft_val = f32::from_bits(ft_bits); let result = fs_val / ft_val; if mips_log(MIPS_LOG_FPU) { dlog_dev!(LogModule::Mips, "FPU div.s PC={:016x} {} / {} = {}", self.core.pc, fs_val, ft_val, result); } + let (flags, bits, denorm) = fpu_binop_finish_s(FUNCT_FDIV, fs_bits, ft_bits, result.to_bits()); let write_w = self.fpr_write_w; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { (result).to_bits() })) + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { bits })) } fn exec_fsqrt_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } let fs_reg = d.rd as u32; let fd_reg = d.sa as u32; let fs_bits = (self.fpr_read_w)(&self.core, fs_reg); if let Some(s) = self.fpu_check_denorm_operand_s(fs_bits) { return s; } - let flags = Self::fpu_arith_flags_sqrt_s(fs_bits); let result = f32::from_bits(fs_bits).sqrt(); + let (flags, rbits) = fpu_fix_invalid_s(|| Self::fpu_arith_flags_sqrt_s(fs_bits), result.to_bits(), [fs_bits]); let write_w = self.fpr_write_w; let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { (result).to_bits() })) + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_w(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { 0x8000_0000 } else { 0 } } else { rbits })) } fn exec_fabs_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8042,14 +8338,14 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } let fs_reg = d.rd as u32; let fd_reg = d.sa as u32; let fs_bits = (self.fpr_read_w)(&self.core, fs_reg); - let flags = Self::fpu_arith_flags_div_s(1.0f32.to_bits(), fs_bits); let fs_val = f32::from_bits(fs_bits); let result = 1.0 / fs_val; if mips_log(MIPS_LOG_FPU) { dlog_dev!(LogModule::Mips, "FPU recip.s PC={:016x} 1 / {} = {}", self.core.pc, fs_val, result); } + let (flags, rbits) = fpu_fix_invalid_s(|| Self::fpu_arith_flags_div_s(1.0f32.to_bits(), fs_bits), result.to_bits(), [fs_bits]); let write_w = self.fpr_write_w; - self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, (result).to_bits())) + self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, rbits)) } fn exec_frsqrt_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8058,13 +8354,14 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // RSQRT = 1/sqrt(x): Invalid for a negative operand (real sqrt) or // signalling NaN, else Divide-by-zero when x is zero (sqrt(0)=0, so // 1/sqrt(0) divides by zero) — union of both component checks. - let flags = { + let flags = || { let sqrt_flags = Self::fpu_arith_flags_sqrt_s(fs_bits); if sqrt_flags != 0 { sqrt_flags } else if is_pos_zero_or_neg_zero_s(fs_bits) { FCSR_FZ } else { 0 } }; let result = 1.0 / f32::from_bits(fs_bits).sqrt(); + let (flags, rbits) = fpu_fix_invalid_s(flags, result.to_bits(), [fs_bits]); let write_w = self.fpr_write_w; - self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, (result).to_bits())) + self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, rbits)) } fn exec_fcvt_d_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8174,10 +8471,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let read_d = self.fpr_read_d; let write_d = self.fpr_write_d; let fs_val = read_d(&self.core, fs_reg); let ft_val = read_d(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_d(fs_val.to_bits()).or_else(|| self.fpu_check_denorm_operand_d(ft_val.to_bits())) { return s; } - let flags = Self::fpu_arith_flags_snan_only_d(fs_val.to_bits(), ft_val.to_bits()); let result = fs_val + ft_val; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { result })) + let (flags, bits, denorm) = fpu_binop_finish_d(FUNCT_FADD, fs_val.to_bits(), ft_val.to_bits(), result.to_bits()); + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { f64::from_bits(bits) })) } fn exec_fsub_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8185,10 +8481,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let read_d = self.fpr_read_d; let write_d = self.fpr_write_d; let fs_val = read_d(&self.core, fs_reg); let ft_val = read_d(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_d(fs_val.to_bits()).or_else(|| self.fpu_check_denorm_operand_d(ft_val.to_bits())) { return s; } - let flags = Self::fpu_arith_flags_snan_only_d(fs_val.to_bits(), ft_val.to_bits()); let result = fs_val - ft_val; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { result })) + let (flags, bits, denorm) = fpu_binop_finish_d(FUNCT_FSUB, fs_val.to_bits(), ft_val.to_bits(), result.to_bits()); + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { f64::from_bits(bits) })) } fn exec_fmul_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8196,10 +8491,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let read_d = self.fpr_read_d; let write_d = self.fpr_write_d; let fs_val = read_d(&self.core, fs_reg); let ft_val = read_d(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_d(fs_val.to_bits()).or_else(|| self.fpu_check_denorm_operand_d(ft_val.to_bits())) { return s; } - let flags = Self::fpu_arith_flags_snan_only_d(fs_val.to_bits(), ft_val.to_bits()); let result = fs_val * ft_val; - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { result })) + let (flags, bits, denorm) = fpu_binop_finish_d(FUNCT_FMUL, fs_val.to_bits(), ft_val.to_bits(), result.to_bits()); + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { f64::from_bits(bits) })) } fn exec_fdiv_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8208,13 +8502,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_val = read_d(&self.core, fs_reg); let ft_val = read_d(&self.core, ft_reg); if let Some(s) = self.fpu_check_denorm_operand_d(fs_val.to_bits()).or_else(|| self.fpu_check_denorm_operand_d(ft_val.to_bits())) { return s; } - let flags = Self::fpu_arith_flags_div_d(fs_val.to_bits(), ft_val.to_bits()); let result = fs_val / ft_val; if mips_log(MIPS_LOG_FPU) { dlog_dev!(LogModule::Mips, "FPU div.d PC={:016x} {} / {} = {}", self.core.pc, fs_val, ft_val, result); } - let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { result })) + let (flags, bits, denorm) = fpu_binop_finish_d(FUNCT_FDIV, fs_val.to_bits(), ft_val.to_bits(), result.to_bits()); + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { f64::from_bits(bits) })) } fn exec_fsqrt_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8222,10 +8515,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let read_d = self.fpr_read_d; let write_d = self.fpr_write_d; let fs_val = read_d(&self.core, fs_reg); if let Some(s) = self.fpu_check_denorm_operand_d(fs_val.to_bits()) { return s; } - let flags = Self::fpu_arith_flags_sqrt_d(fs_val.to_bits()); let result = fs_val.sqrt(); + let (flags, rbits) = fpu_fix_invalid_d(|| Self::fpu_arith_flags_sqrt_d(fs_val.to_bits()), result.to_bits(), [fs_val.to_bits()]); let denorm = result.is_subnormal().then(|| result.is_sign_negative()); - self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { result })) + self.fpu_update_fcsr_full(flags, denorm, |exec, flush| write_d(&mut exec.core, fd_reg, if flush { if result.is_sign_negative() { -0.0 } else { 0.0 } } else { f64::from_bits(rbits) })) } fn exec_fabs_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8330,12 +8623,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_reg = d.rd as u32; let fd_reg = d.sa as u32; let read_d = self.fpr_read_d; let write_d = self.fpr_write_d; let fs_val = read_d(&self.core, fs_reg); - let flags = Self::fpu_arith_flags_div_d(1.0f64.to_bits(), fs_val.to_bits()); let result = 1.0 / fs_val; if mips_log(MIPS_LOG_FPU) { dlog_dev!(LogModule::Mips, "FPU recip.d PC={:016x} 1 / {} = {}", self.core.pc, fs_val, result); } - self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, result)) + let (flags, rbits) = fpu_fix_invalid_d(|| Self::fpu_arith_flags_div_d(1.0f64.to_bits(), fs_val.to_bits()), result.to_bits(), [fs_val.to_bits()]); + self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, f64::from_bits(rbits))) } fn exec_frsqrt_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8343,12 +8636,13 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let read_d = self.fpr_read_d; let write_d = self.fpr_write_d; let fs_val = read_d(&self.core, fs_reg); let fs_bits = fs_val.to_bits(); - let flags = { + let flags = || { let sqrt_flags = Self::fpu_arith_flags_sqrt_d(fs_bits); if sqrt_flags != 0 { sqrt_flags } else if is_pos_zero_or_neg_zero_d(fs_bits) { FCSR_FZ } else { 0 } }; let result = 1.0 / fs_val.sqrt(); - self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, result)) + let (flags, rbits) = fpu_fix_invalid_d(flags, result.to_bits(), [fs_bits]); + self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, f64::from_bits(rbits))) } fn exec_fcvt_s_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8491,9 +8785,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, d.rd as u32); let (fr_val, ft_val, fs_val) = (f32::from_bits(fr_bits), f32::from_bits(ft_bits), f32::from_bits(fs_bits)); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits); + let (flags, rbits) = fpu_fix_invalid_s(|| Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits), (fs_val * ft_val + fr_val).to_bits(), [fr_bits, ft_bits, fs_bits]); let write_w = self.fpr_write_w; - self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, (fs_val * ft_val + fr_val).to_bits())) + self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, rbits)) } fn exec_madd_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8502,8 +8796,8 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let ft_val = read_d(&self.core, d.rt as u32); let fs_val = read_d(&self.core, d.rd as u32); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()); - self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, fs_val * ft_val + fr_val)) + let (flags, rbits) = fpu_fix_invalid_d(|| Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()), (fs_val * ft_val + fr_val).to_bits(), [fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()]); + self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, f64::from_bits(rbits))) } fn exec_msub_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8512,9 +8806,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, d.rd as u32); let (fr_val, ft_val, fs_val) = (f32::from_bits(fr_bits), f32::from_bits(ft_bits), f32::from_bits(fs_bits)); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits); + let (flags, rbits) = fpu_fix_invalid_s(|| Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits), (fs_val * ft_val - fr_val).to_bits(), [fr_bits, ft_bits, fs_bits]); let write_w = self.fpr_write_w; - self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, (fs_val * ft_val - fr_val).to_bits())) + self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, rbits)) } fn exec_msub_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8523,8 +8817,8 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let ft_val = read_d(&self.core, d.rt as u32); let fs_val = read_d(&self.core, d.rd as u32); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()); - self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, fs_val * ft_val - fr_val)) + let (flags, rbits) = fpu_fix_invalid_d(|| Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()), (fs_val * ft_val - fr_val).to_bits(), [fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()]); + self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, f64::from_bits(rbits))) } fn exec_nmadd_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8533,9 +8827,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, d.rd as u32); let (fr_val, ft_val, fs_val) = (f32::from_bits(fr_bits), f32::from_bits(ft_bits), f32::from_bits(fs_bits)); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits); + let (flags, rbits) = fpu_fix_invalid_s(|| Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits), fneg_bits_s(fs_val * ft_val + fr_val), [fr_bits, ft_bits, fs_bits]); let write_w = self.fpr_write_w; - self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, fneg_bits_s(fs_val * ft_val + fr_val))) + self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, rbits)) } fn exec_nmadd_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8544,8 +8838,8 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let ft_val = read_d(&self.core, d.rt as u32); let fs_val = read_d(&self.core, d.rd as u32); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()); - self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, fneg_d_exact(fs_val * ft_val + fr_val))) + let (flags, rbits) = fpu_fix_invalid_d(|| Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()), fneg_d_exact(fs_val * ft_val + fr_val).to_bits(), [fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()]); + self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, f64::from_bits(rbits))) } fn exec_nmsub_s(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8554,9 +8848,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fs_bits = (self.fpr_read_w)(&self.core, d.rd as u32); let (fr_val, ft_val, fs_val) = (f32::from_bits(fr_bits), f32::from_bits(ft_bits), f32::from_bits(fs_bits)); let fd_reg = d.sa as u32; - let flags = Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits); + let (flags, rbits) = fpu_fix_invalid_s(|| Self::fpu_arith_flags_snan_only3_s(fr_bits, ft_bits, fs_bits), fneg_bits_s(fs_val * ft_val - fr_val), [fr_bits, ft_bits, fs_bits]); let write_w = self.fpr_write_w; - self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, fneg_bits_s(fs_val * ft_val - fr_val))) + self.fpu_update_fcsr(flags, |exec| write_w(&mut exec.core, fd_reg, rbits)) } fn exec_nmsub_d(&mut self, d: &DecodedInstr) -> ExecStatus { if (self.core.cp0_status & STATUS_CU1) == 0 { return self.cpu_unusable(1); } @@ -8564,9 +8858,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let fr_val = read_d(&self.core, d.rs as u32); let ft_val = read_d(&self.core, d.rt as u32); let fs_val = read_d(&self.core, d.rd as u32); - let flags = Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()); let fd_reg = d.sa as u32; - self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, fneg_d_exact(fs_val * ft_val - fr_val))) + let (flags, rbits) = fpu_fix_invalid_d(|| Self::fpu_arith_flags_snan_only3_d(fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()), fneg_d_exact(fs_val * ft_val - fr_val).to_bits(), [fr_val.to_bits(), ft_val.to_bits(), fs_val.to_bits()]); + self.fpu_update_fcsr(flags, |exec| write_d(&mut exec.core, fd_reg, f64::from_bits(rbits))) } // LWC1 - Load Word to FPU diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 4098813ec..dca0823b7 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -2960,6 +2960,299 @@ mod tests { assert_eq!(exec.core.read_fpr_d(5), 2.5); } + /// Runs one ADD/SUB/MUL/DIV with FCSR set to `fcsr` (through CTC1's own + /// path, so the host rounding mode follows RM) and a sentinel in the + /// destination. Returns the status, the destination bits and FCSR. + /// Puts the host back in round-to-nearest before returning, since test + /// threads are reused. + fn fpu_top_case(fmt: u32, funct: u32, a: u64, b: u64, fcsr: u32) -> (crate::cpu::mips_exec::ExecStatus, u64, u32) { + struct RestoreRounding; + impl Drop for RestoreRounding { + fn drop(&mut self) { crate::platform::set_fpu_mode(0); } + } + let _restore = RestoreRounding; + let (mut exec, _) = create_executor(); + exec.core.cp0_status |= STATUS_CU1 | STATUS_FR; + exec.update_fpr_mode(); + exec.core.write_fpu_control(31, fcsr); + if fmt == RS_S { + exec.core.write_fpr_w(1, a as u32); + exec.core.write_fpr_w(2, b as u32); + exec.core.write_fpr_w(0, 0x5a5a5a5a); + } else { + exec.core.write_fpr_l(1, a); + exec.core.write_fpr_l(2, b); + exec.core.write_fpr_l(0, 0x5a5a5a5a_5a5a5a5a); + } + let s = exec.exec(make_cop1_compute(fmt, 2, 1, 0, funct)); + let r = if fmt == RS_S { exec.core.read_fpr_w(0) as u64 } else { exec.core.read_fpr_l(0) }; + (s, r, exec.core.fpu_fcsr) + } + + const FCSR_CAUSE_V: u32 = 0x0001_0000; + const FCSR_CAUSE_Z: u32 = 0x0000_8000; + const FCSR_CAUSE_OI: u32 = 0x0000_5000; + const FCSR_FLAG_V: u32 = 0x0000_0040; + const FCSR_FLAG_OI: u32 = 0x0000_0014; + const FCSR_ENABLE_ALL: u32 = 0x0000_0F80; + const FCSR_ENABLE_O: u32 = 0x0000_0200; + const FCSR_ENABLE_V: u32 = 0x0000_0800; + + /// Invalid operations: infinity minus infinity, zero times infinity, + /// 0/0 and infinity/infinity (VR5000 manual U11761EJ6V0UM, 9.4.2, the + /// MIPS IV list). Untrapped, Cause.V and Flag.V are set and the result + /// is the default quiet NaN, 0x7FC00000 / 0x7FF80000_00000000 (MIPS64 + /// Volume I, Table 6.3, IEEE 754-2008 column). 0/0 is not a divide by + /// zero: Z needs a finite nonzero dividend (9.4.3). + #[test] + fn test_fpu_invalid_operation_untrapped() { + let si = f32::INFINITY.to_bits() as u64; + let s0 = 0u64; + let s1 = 1.0f32.to_bits() as u64; + let di = f64::INFINITY.to_bits(); + let d1 = 1.0f64.to_bits(); + let cases = [ + (RS_S, FUNCT_FADD, si, (-f32::INFINITY).to_bits() as u64), + (RS_S, FUNCT_FSUB, si, si), + (RS_S, FUNCT_FMUL, s0, si), + (RS_S, FUNCT_FDIV, s0, s0), + (RS_S, FUNCT_FDIV, si, si), + (RS_D, FUNCT_FADD, di, (-f64::INFINITY).to_bits()), + (RS_D, FUNCT_FSUB, di, di), + (RS_D, FUNCT_FMUL, di, 0), + (RS_D, FUNCT_FDIV, 0, 0), + (RS_D, FUNCT_FDIV, di, di), + ]; + for (fmt, funct, a, b) in cases { + let (s, r, fcsr) = fpu_top_case(fmt, funct, a, b, 0); + assert_eq!(s, EXEC_COMPLETE, "fmt={fmt} funct={funct}"); + let want = if fmt == RS_S { 0x7FC0_0000 } else { 0x7FF8_0000_0000_0000 }; + assert_eq!(r, want, "default NaN, fmt={fmt} funct={funct}"); + assert_eq!(fcsr, FCSR_CAUSE_V | FCSR_FLAG_V, "fmt={fmt} funct={funct}"); + } + // Not invalid: these stay as they were. + let (_, r, fcsr) = fpu_top_case(RS_S, FUNCT_FDIV, s1, s0, 0); + assert_eq!((r, fcsr), (si, FCSR_CAUSE_Z | 0x20), "1/0 is a divide by zero"); + let (_, r, fcsr) = fpu_top_case(RS_S, FUNCT_FDIV, si, s0, 0); + assert_eq!((r, fcsr), (si, 0), "infinity/0 is an exact infinity"); + let (_, r, fcsr) = fpu_top_case(RS_D, FUNCT_FADD, di, d1, 0); + assert_eq!((r, fcsr), (di, 0), "infinity plus a finite number is exact"); + } + + /// The default NaN an Invalid Operation delivers, fed back in as an + /// operand, is a quiet NaN and not a second Invalid Operation. A + /// signalling NaN operand delivers the default NaN too. Both formats. + #[test] + fn test_fpu_default_nan_result_reused_as_operand() { + for (fmt, snan, one) in [ + (RS_S, 0x7FA0_0000u64, 1.0f32.to_bits() as u64), + (RS_D, 0x7FF4_0000_0000_0000, 1.0f64.to_bits()), + ] { + let (s, default_nan, fcsr) = fpu_top_case(fmt, FUNCT_FADD, snan, one, 0); + assert_eq!(s, EXEC_COMPLETE, "fmt={fmt}"); + let want = if fmt == RS_S { 0x7FC0_0000 } else { 0x7FF8_0000_0000_0000 }; + assert_eq!(default_nan, want, "sNaN operand delivers the default NaN, fmt={fmt}"); + assert_eq!(fcsr, FCSR_CAUSE_V | FCSR_FLAG_V, "fmt={fmt}"); + + let (s, r, fcsr) = fpu_top_case(fmt, FUNCT_FADD, default_nan, one, 0); + assert!(s & EXEC_IS_EXCEPTION != 0, "quiet NaN operand traps, fmt={fmt}"); + assert_eq!((s >> 2) & 0x1F, crate::cpu::mips_exec::EXC_FPE); + assert_eq!(fcsr, 0x0002_0000, "Cause.E alone, no Invalid, fmt={fmt}"); + let sentinel = if fmt == RS_S { 0x5a5a5a5a } else { 0x5a5a5a5a_5a5a5a5a }; + assert_eq!(r, sentinel, "destination untouched, fmt={fmt}"); + } + } + + /// The same invalid 0/0 with every exception enabled, as a diagnostic + /// does: Cause.V alone (not Z), a Floating-Point exception, and the + /// result and Flag bits untouched. R10000 User's Manual v2.0, 15.4: + /// "When a Cause bit is set and its corresponding Enable bit is also + /// set, an exception is taken on the instruction. The result of the + /// instruction is not stored, and the Flag bits are not changed." + #[test] + fn test_fpu_invalid_operation_trapped() { + for fmt in [RS_S, RS_D] { + let (s, r, fcsr) = fpu_top_case(fmt, FUNCT_FDIV, 0, 0, FCSR_ENABLE_ALL); + assert!(s & EXEC_IS_EXCEPTION != 0, "fmt={fmt}"); + assert_eq!((s >> 2) & 0x1F, crate::cpu::mips_exec::EXC_FPE); + assert_eq!(fcsr, FCSR_ENABLE_ALL | FCSR_CAUSE_V, "fmt={fmt}"); + let sentinel = if fmt == RS_S { 0x5a5a5a5a } else { 0x5a5a5a5a_5a5a5a5a }; + assert_eq!(r, sentinel, "destination untouched, fmt={fmt}"); + } + } + + /// Overflow from finite operands, all four rounding modes. VR5000 manual + /// 9.4.4: Overflow "also sets the Inexact exception and Flag bits", and + /// the untrapped result follows Table 9-1: RN gives a signed infinity; + /// RZ the largest finite number with the result's sign; RP +infinity or + /// the most negative finite number; RM the largest finite number or + /// -infinity. + #[test] + fn test_fpu_overflow_result_per_rounding_mode() { + let smax = f32::MAX.to_bits() as u64; + let sinf = f32::INFINITY.to_bits() as u64; + let sneg = |b: u64| b | 0x8000_0000; + let dmax = f64::MAX.to_bits(); + let dinf = f64::INFINITY.to_bits(); + let dneg = |b: u64| b | 0x8000_0000_0000_0000; + // (rm, positive result, negative result) + let s_want = [(0, sinf, sneg(sinf)), (1, smax, sneg(smax)), (2, sinf, sneg(smax)), (3, smax, sneg(sinf))]; + let d_want = [(0, dinf, dneg(dinf)), (1, dmax, dneg(dmax)), (2, dinf, dneg(dmax)), (3, dmax, dneg(dinf))]; + for (rm, pos, neg) in s_want { + for (funct, a, b, want) in [ + (FUNCT_FADD, smax, smax, pos), + (FUNCT_FSUB, sneg(smax), smax, neg), + (FUNCT_FMUL, smax, 2.0f32.to_bits() as u64, pos), + (FUNCT_FDIV, sneg(smax), 0.5f32.to_bits() as u64, neg), + ] { + let (s, r, fcsr) = fpu_top_case(RS_S, funct, a, b, rm); + assert_eq!(s, EXEC_COMPLETE, "rm={rm} funct={funct}"); + assert_eq!(r, want, "single rm={rm} funct={funct}"); + assert_eq!(fcsr, rm | FCSR_CAUSE_OI | FCSR_FLAG_OI, "single rm={rm} funct={funct}"); + } + } + for (rm, pos, neg) in d_want { + for (funct, a, b, want) in [ + (FUNCT_FADD, dmax, dmax, pos), + (FUNCT_FSUB, dneg(dmax), dmax, neg), + (FUNCT_FMUL, 2.0f64.to_bits(), dmax, pos), + (FUNCT_FDIV, dneg(dmax), 0.5f64.to_bits(), neg), + ] { + let (s, r, fcsr) = fpu_top_case(RS_D, funct, a, b, rm); + assert_eq!(s, EXEC_COMPLETE, "rm={rm} funct={funct}"); + assert_eq!(r, want, "double rm={rm} funct={funct}"); + assert_eq!(fcsr, rm | FCSR_CAUSE_OI | FCSR_FLAG_OI, "double rm={rm} funct={funct}"); + } + } + } + + /// A largest-finite result that is not an overflow raises nothing: MAX + /// times 1 is exact, and MAX plus 1 rounded toward zero is MAX because + /// the exact sum is below 2^128 (2^1024 for double). Nor does a double + /// just below MAX. + #[test] + fn test_fpu_largest_finite_result_is_not_overflow() { + let smax = f32::MAX.to_bits() as u64; + let dmax = f64::MAX.to_bits(); + for rm in 0..4 { + let (_, r, fcsr) = fpu_top_case(RS_S, FUNCT_FMUL, smax, 1.0f32.to_bits() as u64, rm); + assert_eq!((r, fcsr), (smax, rm), "single MAX*1 rm={rm}"); + let (_, r, fcsr) = fpu_top_case(RS_D, FUNCT_FMUL, dmax, 1.0f64.to_bits(), rm); + assert_eq!((r, fcsr), (dmax, rm), "double MAX*1 rm={rm}"); + // Shares MAX's high word, which is all the double-precision + // range test looks at. + let near = 0x7FEF_FFFF_0000_0000u64; + let (_, r, fcsr) = fpu_top_case(RS_D, FUNCT_FMUL, near, 1.0f64.to_bits(), rm); + assert_eq!((r, fcsr), (near, rm), "double just below MAX rm={rm}"); + } + let (_, r, fcsr) = fpu_top_case(RS_S, FUNCT_FADD, smax, 1.0f32.to_bits() as u64, 1); + assert_eq!((r, fcsr), (smax, 1), "single MAX+1 toward zero"); + let (_, r, fcsr) = fpu_top_case(RS_D, FUNCT_FADD, dmax, 1.0f64.to_bits(), 1); + assert_eq!((r, fcsr), (dmax, 1), "double MAX+1 toward zero"); + } + + /// Overflow with its Enable set traps with Cause.O and Cause.I (VR5000 + /// Table 9-2, "Exponent overflow O,I"), destination and Flags untouched. + #[test] + fn test_fpu_overflow_trapped() { + let (s, r, fcsr) = fpu_top_case(RS_S, FUNCT_FADD, f32::MAX.to_bits() as u64, f32::MAX.to_bits() as u64, FCSR_ENABLE_O); + assert!(s & EXEC_IS_EXCEPTION != 0); + assert_eq!((s >> 2) & 0x1F, crate::cpu::mips_exec::EXC_FPE); + assert_eq!(fcsr, FCSR_ENABLE_O | FCSR_CAUSE_OI); + assert_eq!(r, 0x5a5a5a5a); + let (s, r, fcsr) = fpu_top_case(RS_D, FUNCT_FMUL, f64::MAX.to_bits(), 4.0f64.to_bits(), FCSR_ENABLE_O); + assert!(s & EXEC_IS_EXCEPTION != 0); + assert_eq!(fcsr, FCSR_ENABLE_O | FCSR_CAUSE_OI); + assert_eq!(r, 0x5a5a5a5a_5a5a5a5a); + } + + /// Runs one instruction with f1 = `a`, f2 = `b`, f4 = `c` and a sentinel + /// in f0, the destination. Returns the status, the destination bits + /// (read as a word when `double_result` is false) and FCSR. + fn fpu_instr_case(instr: u32, double: bool, a: u64, b: u64, c: u64, fcsr: u32, double_result: bool) -> (crate::cpu::mips_exec::ExecStatus, u64, u32) { + struct RestoreRounding; + impl Drop for RestoreRounding { + fn drop(&mut self) { crate::platform::set_fpu_mode(0); } + } + let _restore = RestoreRounding; + let (mut exec, _) = create_executor_m4(); + exec.core.cp0_status |= STATUS_CU1 | STATUS_FR; + exec.update_fpr_mode(); + exec.core.write_fpu_control(31, fcsr); + if double { + exec.core.write_fpr_l(1, a); + exec.core.write_fpr_l(2, b); + exec.core.write_fpr_l(4, c); + } else { + exec.core.write_fpr_w(1, a as u32); + exec.core.write_fpr_w(2, b as u32); + exec.core.write_fpr_w(4, c as u32); + } + exec.core.write_fpr_l(0, 0x5a5a5a5a_5a5a5a5a); + let s = exec.exec(instr); + let r = if double_result { exec.core.read_fpr_l(0) } else { exec.core.read_fpr_w(0) as u64 }; + (s, r, exec.core.fpu_fcsr) + } + + /// Instructions whose invalid result does not come through the plain + /// arithmetic path: SQRT and RSQRT of -1, every multiply-add on + /// infinity times zero, and CVT.S.D of a signalling NaN (quiet bit + /// clear). Each entry is (instruction, source is double, + /// f1, f2, f4, result is double); f2 is zero, f4 is 1.0. + fn fpu_invalid_producers() -> Vec<(u32, bool, u64, u64, u64, bool)> { + let mut v = Vec::new(); + for (fmt, d) in [(RS_S, false), (RS_D, true)] { + let (m1, inf, zero, one) = if d { + ((-1.0f64).to_bits(), f64::INFINITY.to_bits(), 0u64, 1.0f64.to_bits()) + } else { + ((-1.0f32).to_bits() as u64, f32::INFINITY.to_bits() as u64, 0u64, 1.0f32.to_bits() as u64) + }; + v.push((make_cop1_compute(fmt, 0, 1, 0, FUNCT_FSQRT), d, m1, zero, one, d)); + v.push((make_cop1_compute(fmt, 0, 1, 0, FUNCT_FRSQRT), d, m1, zero, one, d)); + // fd = fs * ft op fr with fs = f1 (infinity), ft = f2 (zero), fr = f4. + let functs = if d { + [FUNCT_MADD_D, FUNCT_MSUB_D, FUNCT_NMADD_D, FUNCT_NMSUB_D] + } else { + [FUNCT_MADD_S, FUNCT_MSUB_S, FUNCT_NMADD_S, FUNCT_NMSUB_S] + }; + for funct in functs { + v.push((make_cop1x(4, 2, 1, 0, funct), d, inf, zero, one, d)); + } + } + v.push((make_cop1_compute(RS_D, 0, 1, 0, FUNCT_FCVT_S), true, 0x7FF4_0000_0000_0000, 0, 0, false)); + v + } + + /// SQRT, RSQRT, the multiply-add family and CVT.S.D give the default + /// quiet NaN with Cause.V and Flag.V for an invalid operation (VR5000 + /// manual U11761EJ6V0UM, 9.4.2; MIPS64 Volume I, Table 6.3), not the + /// host's NaN (0xFFC00000 on x86, a quiet NaN with the sign set). + #[test] + fn test_fpu_invalid_unary_and_madd_untrapped() { + for (instr, dbl, a, b, c, dbl_result) in fpu_invalid_producers() { + let (s, r, fcsr) = fpu_instr_case(instr, dbl, a, b, c, 0, dbl_result); + assert_eq!(s, EXEC_COMPLETE, "instr={instr:#010x}"); + let want = if dbl_result { 0x7FF8_0000_0000_0000u64 } else { 0x7FC0_0000 }; + assert_eq!(r, want, "default NaN, instr={instr:#010x}"); + assert_eq!(fcsr, FCSR_CAUSE_V | FCSR_FLAG_V, "instr={instr:#010x}"); + } + } + + /// The same operations with Enable.V set raise a Floating-Point + /// exception with Cause.V, store nothing and leave the Flags alone + /// (R10000 User's Manual v2.0, 15.4). + #[test] + fn test_fpu_invalid_unary_and_madd_trapped() { + for (instr, dbl, a, b, c, dbl_result) in fpu_invalid_producers() { + let (s, r, fcsr) = fpu_instr_case(instr, dbl, a, b, c, FCSR_ENABLE_V, dbl_result); + assert!(s & EXEC_IS_EXCEPTION != 0, "instr={instr:#010x}"); + assert_eq!((s >> 2) & 0x1F, crate::cpu::mips_exec::EXC_FPE, "instr={instr:#010x}"); + assert_eq!(fcsr, FCSR_ENABLE_V | FCSR_CAUSE_V, "instr={instr:#010x}"); + let sentinel = if dbl_result { 0x5a5a5a5a_5a5a5a5a } else { 0x5a5a5a5a }; + assert_eq!(r, sentinel, "destination untouched, instr={instr:#010x}"); + } + } + #[test] fn test_fpu_sqrt_abs_neg_double() { let (mut exec, _) = create_executor(); From 2622c73229eff2cf409267ee2c8de56dcc91f482 Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 1 Nov 2026 20:12:31 -0700 Subject: [PATCH 2/3] jitv2: flush a denormal FP result when FCSR.FS is set With FS set and neither Underflow nor Inexact enabled, an R10000 replaces a denormal ADD/SUB/MUL/DIV/SQRT result with a signed zero and sets Cause and Flag U|I. The interpreter does that. Compiled code trapped with Unimplemented Operation instead, for every denormal result, whatever FS said. The trap decision negated the "FS is set" compare with bnot. A Cranelift compare yields 0 or 1, and bnot(1) is 0xFE, which still reads as true, so "FS clear or U|I enabled" was always true. The same mistake in the round-to-integer path was fixed earlier. This one uses a compare against zero too. The new equivalence test halves the smallest normal number in both formats with FS set, FS clear and FS set with Underflow enabled, and checks the compiled result against the interpreter and the manual. The trap and flush blocks are marked cold too. They hang off the cold denormal block but were not cold themselves, so Cranelift laid them out among the hot instructions of every FP op. On the benchmark suite that cost fpu/linpack its fast mode in more runs (median 565 against 619 MIPS on main) and left fpu/transcend about 1% behind main; with the blocks cold linpack matches main and transcend is 8% ahead. --- CHANGELOG.md | 6 ++++++ src/cpu/jitv2/codegen.rs | 6 +++++- src/cpu/jitv2/equiv_test.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 366855d33..427a2a201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,12 @@ hashes are given where a change is easiest to understand by reading the commit. replaces the denormal test, and the multiply-add family, RECIP and RSQRT now classify their operands only when the result is an infinity or a NaN. +### jitv2: FCSR.FS flushes denormal results + +- Compiled code trapped every denormal ADD/SUB/MUL/DIV/SQRT result as + Unimplemented, even with FS set. It now flushes to a signed zero with Cause + and Flag U|I, like the interpreter. + ### Disk write cache - A disk now has a `write_cache` setting in its `[scsi.N]` table (and a diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index 6db3fbc1d..d0d02f25c 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -5567,7 +5567,9 @@ fn emit_fpu_update_fcsr_arith( let fs_set = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, fs_set, 0); let ui_enabled = ctx.builder.ins().band_imm_s(fcsr, FCSR_UM_IM); let ui_enabled = ctx.builder.ins().icmp_imm_s(IntCC::NotEqual, ui_enabled, 0); - let fs_clear = ctx.builder.ins().bnot(fs_set); + // Not `bnot(fs_set)`: an icmp result is 0 or 1, and bnot(1) = 0xFE is + // still true, which made every denormal result trap even with FS set. + let fs_clear = ctx.builder.ins().icmp_imm_s(IntCC::Equal, fs_set, 0); let must_trap = ctx.builder.ins().bor(fs_clear, ui_enabled); let trap_block = ctx.builder.create_block(); @@ -5575,10 +5577,12 @@ fn emit_fpu_update_fcsr_arith( ctx.builder.ins().brif(must_trap, trap_block, &[], flush_block, &[]); ctx.builder.switch_to_block(trap_block); + ctx.builder.set_cold_block(trap_block); ctx.builder.seal_block(trap_block); emit_fpu_unimplemented(ctx); ctx.builder.switch_to_block(flush_block); + ctx.builder.set_cold_block(flush_block); ctx.builder.seal_block(flush_block); let fcsr_cleared = ctx.builder.ins().band_imm_s(fcsr, !FCSR_CM); let fcsr_with_cause = ctx.builder.ins().bor_imm_s(fcsr_cleared, FCSR_CU_CI); diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 09c0684d3..13b51aefc 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -5759,6 +5759,38 @@ mod tests { } } + /// A denormal result with FCSR.FS set and neither Underflow nor Inexact + /// enabled is replaced by a signed zero with Cause and Flag U|I, no trap + /// (R10000 User's Manual v2.0, 15.4, Flush (FS) bit: "A denormalized + /// result is replaced with zero"; VR5000 manual U11761EJ6V0UM, 9.4.5). + /// With FS clear, or with Underflow enabled, it is an Unimplemented + /// Operation. The smallest normal halved gives an exact denormal. + #[test] + fn denormal_result_with_fs_set_flushes_like_the_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FMUL, OP_COP1, RS_D, RS_S}; + const FS: u32 = 0x0100_0000; + for (fmt, min_normal, half, sign) in [ + (RS_S, 0x0080_0000u64, 0x3F00_0000u64, 1u64 << 31), + (RS_D, 0x0010_0000_0000_0000, 0x3FE0_0000_0000_0000, 1u64 << 63), + ] { + for fcsr in [FS, 0, FS | 0x0100] { + let mut fpr = [0u64; 32]; + fpr[1] = min_normal | sign; + fpr[2] = half; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + let jit = assert_fpu_matches_interpreter_fcsr( + make_r(OP_COP1, fmt, 2, 1, 3, FUNCT_FMUL), [0u64; 32], fpr, true, Some(fcsr)); + if fcsr == FS { + assert_eq!(jit.fpu_fcsr, FS | 0x0000_300C, "fmt={fmt}: Cause and Flag U|I, no trap"); + let r = if fmt == RS_S { jit.fpr[3] & 0xFFFF_FFFF } else { jit.fpr[3] }; + assert_eq!(r, sign, "fmt={fmt}: negative zero delivered"); + } else { + assert_eq!(jit.fpu_fcsr & 0x0003_F000, 0x0002_0000, "fmt={fmt} fcsr={fcsr:#x}: Cause E alone"); + } + } + } + } + /// SQRT, RSQRT, the multiply-add family and CVT.S.D leave an Invalid /// Operation's result as the default NaN, not the host's /// (0xFFC00000 on x86, which has the sign set). The next instruction From bc31a234ea486c0ac8a138496c4f4857b224cf6a Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 1 Nov 2026 20:20:00 -0700 Subject: [PATCH 3/3] cpu: classify FP NaNs with the MIPS encoding The R4400, R5000 and R10000 use the original MIPS NaN encoding: a NaN with the fraction MSB set is signalling, one with it clear is quiet. The default NaN is 0x7FBFFFFF for that reason. iris used the IEEE 754-2008 convention, the reverse, in the interpreter and in compiled code. The evidence for the original encoding is the NEC VR10000 instruction manual, U12754E, first edition, August 2000, printed page 226, table 2-2, and the results of iris's own R4400 in cpu-tests: fpu/qnan_operand feeds 0x7FC00000 to ADD and gets Invalid and the default NaN, not Unimplemented, and the signalling-predicate compares agree. Upstream issue 361 asked about it. Swap the sense of the fraction-MSB test in is_snan and is_qnan and in their compiled twins, including the compare path, and make the default NaN 0x7FBFFFFF and 0x7FF7FFFF_FFFFFFFF. With the bit set the fraction is nonzero by construction, so the interpreter's signalling test shrinks to one mask and compare, and an ordinary operand skips the subnormal and quiet-NaN checks after one exponent test. Compiled code swaps a comparison and costs the same. The tests that fix a NaN's bit pattern move with this commit. The new ones cover the default NaN as an operand (Unimplemented, no Invalid), a signalling NaN operand (Invalid, trapped and untrapped), and a compiled-versus-interpreter check over ADD, MUL, DIV, SQRT and the quiet and signalling compare predicates. Without this commit the rest of the series still builds and passes its tests; the Octane PROM's ide test then rejects the default NaN's bit pattern. --- CHANGELOG.md | 16 ++- .../jitv2/fpu-flags-are-computed-not-read.md | 21 ++-- src/cpu/jitv2/codegen.rs | 48 ++++---- src/cpu/jitv2/equiv_test.rs | 70 ++++++++++-- src/cpu/mips_exec.rs | 35 ++++-- src/cpu/mips_exec_test.rs | 108 +++++++++++++++--- 6 files changed, 230 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 427a2a201..305ab2879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,8 +83,8 @@ hashes are given where a change is easiest to understand by reading the commit. - **Invalid operations raise Invalid.** Infinity minus infinity, zero times infinity, 0/0 and infinity/infinity set Cause and Flag V and, untrapped, - deliver the default quiet NaN (0x7FC00000, 0x7FF80000_00000000) instead of - the host's NaN. 0/0 raises Invalid rather than divide-by-zero, and + deliver the MIPS default quiet NaN (0x7FBFFFFF, 0x7FF7FFFF_FFFFFFFF) instead + of the host's NaN. 0/0 raises Invalid rather than divide-by-zero, and infinity/0 raises nothing. - **Overflow raises Overflow and Inexact** in ADD, SUB, MUL and DIV. The result was already rounded for the FCSR rounding mode (an infinity, or the largest @@ -98,6 +98,18 @@ hashes are given where a change is easiest to understand by reading the commit. instructions than before in either engine: one range compare on the result replaces the denormal test, and the multiply-add family, RECIP and RSQRT now classify their operands only when the result is an infinity or a NaN. +- The Octane PROM diagnostics' FPU invalid-result and overflow-result tests + pass (14 of 14 `ide fpu` subtests). + +### cpu: signalling and quiet NaNs use the MIPS encoding + +- The FPU told the two NaN kinds apart with the IEEE 754-2008 quiet bit. The + R4400, R5000 and R10000 use the original MIPS encoding, where a NaN with the + fraction MSB set is signalling and one with it clear is quiet, so the default + NaN is 0x7FBFFFFF and 0x7FC00000 is signalling. A signalling operand now + raises Invalid and a quiet one takes the Unimplemented path, in the + interpreter and in compiled code, matching the R4400 results in cpu-tests + `fpu/qnan_operand`. ### jitv2: FCSR.FS flushes denormal results diff --git a/clanker/jitv2/fpu-flags-are-computed-not-read.md b/clanker/jitv2/fpu-flags-are-computed-not-read.md index d23c73bcc..968c49fe9 100644 --- a/clanker/jitv2/fpu-flags-are-computed-not-read.md +++ b/clanker/jitv2/fpu-flags-are-computed-not-read.md @@ -30,8 +30,8 @@ particular: the denormal-result test on the same path, sends zero, denormals, ±MAX, ±infinity and NaN to a cold side. There `fpu_binop_top_flags_s/d` (`mips_exec.rs`) classifies the top of the range: a NaN from non-NaN - operands is Invalid (inf-inf, 0*inf, 0/0, inf/inf) and delivers the - default NaN 0x7FC00000 / 0x7FF80000_00000000; an infinity, or a ±MAX whose + operands is Invalid (inf-inf, 0*inf, 0/0, inf/inf) and delivers the MIPS + default NaN 0x7FBFFFFF / 0x7FF7FFFF_FFFFFFFF; an infinity, or a ±MAX whose exact value reaches 2^(emax+1), from finite operands is Overflow+Inexact. The JIT reaches the same function through `fpu_binop_commit_fn`, one call that also does the operand checks and the denormal-result rule for @@ -45,12 +45,12 @@ particular: an overflowing result from any of them is delivered as the host rounds it, with FCSR untouched. CVT.S.D has its own overflow check but no Inexact-with-Overflow. -- **Invalid results of the operations above deliver the default NaN.** - The host's NaN for an invalid result (x86: 0xFFC00000) has the sign bit - set; hardware delivers the positive default NaN, so a program that stores - and compares the result sees the same bits. +- **Invalid results of the operations above deliver the MIPS default NaN.** + The host's NaN for an invalid result (x86: 0xFFC00000) has the quiet bit + set, which is a *signalling* NaN under the MIPS encoding, so an + instruction that read it would raise V where hardware takes E. `fpu_fix_invalid_s/d` (interpreter) and `emit_fix_invalid` (JIT) replace - a NaN result with 0x7FC00000 / 0x7FF80000_00000000 and add V when an + a NaN result with 0x7FBFFFFF / 0x7FF7FFFF_FFFFFFFF and add V when an operand was a signalling NaN or no operand was a NaN (infinity times zero, infinity minus infinity, the square root of a negative number). A quiet NaN operand propagates unchanged. Both engines gate it on one @@ -63,6 +63,13 @@ particular: `emit_commit_finite_first` as an emitter). CVT.S.D, which runs in `cvt_d_to_s_and_commit` for both engines, does the same for a signalling NaN source. + Known limit: RECIP, RSQRT and the MADD family have no operand check, so a + quiet-NaN operand (quiet bit clear, the MIPS encoding) propagates with the + host's quieting, which sets the fraction MSB. The result, for example + 0x7FE00000, then reads as a *signalling* NaN to the next operation, which + raises V where hardware probably takes E. SQRT traps a quiet-NaN operand + with E before this point. What an R10000 does with a quiet-NaN operand to + these operations is not stated in the manuals we have, so it is left. A quiet-NaN source to CVT.S.D has a related quirk: it is never equal to its widened result, so it leaves the exact path and sets Cause and Flag I. diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index d0d02f25c..802f37ac6 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -5352,7 +5352,7 @@ fn emit_check_snan_operand(ctx: &mut EmitCtx, bits: Value, is_d: bool, write_res } /// `bits` is a signalling NaN: exponent all-1s, mantissa nonzero, mantissa -/// MSB (the "is quiet" bit) clear. Mirrors `mips_exec.rs`'s `is_snan_s`/`_d`. +/// MSB set (the original MIPS NaN encoding). Mirrors `mips_exec.rs`'s `is_snan_s`/`_d`. fn emit_is_snan_s(ctx: &mut EmitCtx, bits: Value) -> Value { let b = &mut ctx.builder; let exp_is_max = b.ins().band_imm_s(bits, 0x7F80_0000); @@ -5360,9 +5360,9 @@ fn emit_is_snan_s(ctx: &mut EmitCtx, bits: Value) -> Value { let mantissa_nonzero = b.ins().band_imm_s(bits, 0x007F_FFFF); let mantissa_nonzero = b.ins().icmp_imm_s(IntCC::NotEqual, mantissa_nonzero, 0); let is_nan = b.ins().band(exp_is_max, mantissa_nonzero); - let quiet_bit = b.ins().band_imm_s(bits, 0x0040_0000); - let quiet_bit_clear = b.ins().icmp_imm_s(IntCC::Equal, quiet_bit, 0); - b.ins().band(is_nan, quiet_bit_clear) + let snan_bit = b.ins().band_imm_s(bits, 0x0040_0000); + let snan_bit_set = b.ins().icmp_imm_s(IntCC::NotEqual, snan_bit, 0); + b.ins().band(is_nan, snan_bit_set) } fn emit_is_snan_d(ctx: &mut EmitCtx, bits: Value) -> Value { let b = &mut ctx.builder; @@ -5371,9 +5371,9 @@ fn emit_is_snan_d(ctx: &mut EmitCtx, bits: Value) -> Value { let mantissa_nonzero = b.ins().band_imm_s(bits, 0x000F_FFFF_FFFF_FFFFu64 as i64); let mantissa_nonzero = b.ins().icmp_imm_s(IntCC::NotEqual, mantissa_nonzero, 0); let is_nan = b.ins().band(exp_is_max, mantissa_nonzero); - let quiet_bit = b.ins().band_imm_s(bits, 0x0008_0000_0000_0000u64 as i64); - let quiet_bit_clear = b.ins().icmp_imm_s(IntCC::Equal, quiet_bit, 0); - b.ins().band(is_nan, quiet_bit_clear) + let snan_bit = b.ins().band_imm_s(bits, 0x0008_0000_0000_0000u64 as i64); + let snan_bit_set = b.ins().icmp_imm_s(IntCC::NotEqual, snan_bit, 0); + b.ins().band(is_nan, snan_bit_set) } /// `bits` is a quiet NaN (any NaN that is not a signalling NaN) — mirrors @@ -5456,9 +5456,9 @@ fn emit_is_subnormal_or_qnan_s(ctx: &mut EmitCtx, bits: Value) -> Value { let exp_is_max = b.ins().icmp_imm_s(IntCC::Equal, exp, 0x7F80_0000); let is_nan = b.ins().band(exp_is_max, mantissa_nonzero); - let quiet_bit = b.ins().band_imm_s(bits, 0x0040_0000); - let quiet_bit_set = b.ins().icmp_imm_s(IntCC::NotEqual, quiet_bit, 0); - let is_qnan = b.ins().band(is_nan, quiet_bit_set); + let snan_bit = b.ins().band_imm_s(bits, 0x0040_0000); + let snan_bit_clear = b.ins().icmp_imm_s(IntCC::Equal, snan_bit, 0); + let is_qnan = b.ins().band(is_nan, snan_bit_clear); b.ins().bor(is_subnormal, is_qnan) } @@ -5472,9 +5472,9 @@ fn emit_is_subnormal_or_qnan_d(ctx: &mut EmitCtx, bits: Value) -> Value { let exp_is_max = b.ins().icmp_imm_s(IntCC::Equal, exp, 0x7FF0_0000_0000_0000u64 as i64); let is_nan = b.ins().band(exp_is_max, mantissa_nonzero); - let quiet_bit = b.ins().band_imm_s(bits, 0x0008_0000_0000_0000u64 as i64); - let quiet_bit_set = b.ins().icmp_imm_s(IntCC::NotEqual, quiet_bit, 0); - let is_qnan = b.ins().band(is_nan, quiet_bit_set); + let snan_bit = b.ins().band_imm_s(bits, 0x0008_0000_0000_0000u64 as i64); + let snan_bit_clear = b.ins().icmp_imm_s(IntCC::Equal, snan_bit, 0); + let is_qnan = b.ins().band(is_nan, snan_bit_clear); b.ins().bor(is_subnormal, is_qnan) } @@ -5609,7 +5609,7 @@ fn emit_fpu_update_fcsr_arith( /// RSQRT and the multiply-add family, on a result already known to be an /// infinity or a NaN (`emit_commit_finite_first` sends only those here): a NaN /// result is Invalid when `flags` already has V (a signalling NaN operand) or -/// when none of `ops` is a NaN, and then V is added and the default NaN +/// when none of `ops` is a NaN, and then V is added and the MIPS default NaN /// replaces the host's. Straight-line selects, no branches, since it runs in /// a cold block. Returns the flags and result bits to use. fn emit_fix_invalid(ctx: &mut EmitCtx, is_d: bool, flags: Value, result_bits: Value, ops: &[Value]) -> (Value, Value) { @@ -5636,9 +5636,9 @@ fn emit_fix_invalid(ctx: &mut EmitCtx, is_d: bool, flags: Value, result_bits: Va let flags_v = ctx.builder.ins().bor_imm_s(flags, FCSR_FV_I64); let new_flags = ctx.builder.ins().select(fix, flags_v, flags); let default_nan = if is_d { - ctx.builder.ins().iconst(ir::types::I64, 0x7FF8_0000_0000_0000u64 as i64) + ctx.builder.ins().iconst(ir::types::I64, 0x7FF7_FFFF_FFFF_FFFFu64 as i64) } else { - ctx.builder.ins().iconst(ir::types::I32, 0x7FC0_0000) + ctx.builder.ins().iconst(ir::types::I32, 0x7FBF_FFFF) }; let new_bits = ctx.builder.ins().select(fix, default_nan, result_bits); (new_flags, new_bits) @@ -9022,7 +9022,7 @@ fn emit_fcc(ctx: &mut EmitCtx, fr_mode: FrMode, is_d: bool) { // signalling predicates (funct bit 3) whenever either operand is any // NaN — mirrors `exec_fcc_s`/`exec_fcc_d`'s `invalid` computation // exactly. `is_snan`: exponent all-1s, mantissa nonzero, mantissa MSB - // (the "is quiet" bit) clear. + // set (the original MIPS NaN encoding). let is_snan = |b: &mut cranelift_frontend::FunctionBuilder, bits: ir::Value, is_d: bool| -> ir::Value { if is_d { let exp_mantissa_nonzero = { @@ -9033,20 +9033,20 @@ fn emit_fcc(ctx: &mut EmitCtx, fr_mode: FrMode, is_d: bool) { let mantissa_nonzero = b.ins().icmp_imm_s(IntCC::NotEqual, mantissa_nonzero, 0); b.ins().band(exp_is_max, mantissa_nonzero) }; - let quiet_bit_clear = { - let quiet_bit = b.ins().band_imm_s(bits, 0x0008_0000_0000_0000u64 as i64); - b.ins().icmp_imm_s(IntCC::Equal, quiet_bit, 0) + let snan_bit_set = { + let snan_bit = b.ins().band_imm_s(bits, 0x0008_0000_0000_0000u64 as i64); + b.ins().icmp_imm_s(IntCC::NotEqual, snan_bit, 0) }; - b.ins().band(exp_mantissa_nonzero, quiet_bit_clear) + b.ins().band(exp_mantissa_nonzero, snan_bit_set) } else { let exp_is_max = b.ins().band_imm_s(bits, 0x7F80_0000); let exp_is_max = b.ins().icmp_imm_s(IntCC::Equal, exp_is_max, 0x7F80_0000); let mantissa_nonzero = b.ins().band_imm_s(bits, 0x007F_FFFF); let mantissa_nonzero = b.ins().icmp_imm_s(IntCC::NotEqual, mantissa_nonzero, 0); let exp_mantissa_nonzero = b.ins().band(exp_is_max, mantissa_nonzero); - let quiet_bit = b.ins().band_imm_s(bits, 0x0040_0000); - let quiet_bit_clear = b.ins().icmp_imm_s(IntCC::Equal, quiet_bit, 0); - b.ins().band(exp_mantissa_nonzero, quiet_bit_clear) + let snan_bit = b.ins().band_imm_s(bits, 0x0040_0000); + let snan_bit_set = b.ins().icmp_imm_s(IntCC::NotEqual, snan_bit, 0); + b.ins().band(exp_mantissa_nonzero, snan_bit_set) } }; let a_snan = is_snan(ctx.builder, a_bits, is_d); diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 13b51aefc..2abc11b0d 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -5514,6 +5514,53 @@ mod tests { } } + /// Quiet and signalling NaNs use the original MIPS encoding: the fraction + /// MSB set means signalling. The default NaN the FPU delivers (0x7FBFFFFF, + /// 0x7FF7FFFF_FFFFFFFF) is therefore quiet and takes the Unimplemented + /// path as an operand, while 0x7FC00000 / 0x7FF80000_00000000 is an + /// Invalid Operation. Compiled code must classify like the interpreter on + /// ADD, MUL, DIV, SQRT and the compare predicates. + #[test] + fn nan_operand_encoding_matches_interpreter() { + use crate::cpu::mips_isa::{FUNCT_FADD, FUNCT_FC_EQ, FUNCT_FC_LT, FUNCT_FDIV, FUNCT_FMUL, FUNCT_FSQRT, OP_COP1, RS_D, RS_S}; + for (fmt, quiet, signalling) in [ + (RS_S, 0x7FBF_FFFFu64, 0x7FC0_0000u64), + (RS_D, 0x7FF7_FFFF_FFFF_FFFF, 0x7FF8_0000_0000_0000), + ] { + let one = if fmt == RS_S { 0x3F80_0000u64 } else { 0x3FF0_0000_0000_0000 }; + for funct in [FUNCT_FADD, FUNCT_FMUL, FUNCT_FDIV, FUNCT_FSQRT, FUNCT_FC_EQ, FUNCT_FC_LT] { + for fcsr in [0u32, 0x0800] { + for (nan, other) in [(quiet, one), (one, quiet), (signalling, one), (one, signalling)] { + // SQRT reads fs only. + if funct == FUNCT_FSQRT && nan == one { + continue; + } + let mut fpr = [0u64; 32]; + fpr[1] = nan; + fpr[2] = other; + fpr[3] = 0xDEAD_BEEF_DEAD_BEEF; + let jit = assert_fpu_matches_interpreter_fcsr( + make_r(OP_COP1, fmt, 2, 1, 3, funct), [0u64; 32], fpr, true, Some(fcsr)); + let sig = nan == signalling || other == signalling; + let cause = jit.fpu_fcsr & 0x0003_F000; + let what = format!("fmt={fmt} funct={funct:#x} fcsr={fcsr:#x} nan={nan:#x} other={other:#x}"); + if funct == FUNCT_FC_EQ || funct == FUNCT_FC_LT { + // Compare never takes Unimplemented. A signalling + // NaN is Invalid on every predicate; a quiet one + // only on the signalling predicates, such as LT. + let invalid = sig || funct == FUNCT_FC_LT; + assert_eq!(cause, if invalid { 0x0001_0000 } else { 0 }, "{what}"); + } else if sig { + assert_eq!(cause, 0x0001_0000, "{what}: Invalid alone"); + } else { + assert_eq!(cause, 0x0002_0000, "{what}: Unimplemented alone"); + } + } + } + } + } + } + /// ADD/SUB/MUL/DIV on operands whose results reach the top of the /// range (±MAX, ±infinity, NaN), in every rounding mode, trapped and /// untrapped: the JIT's cold classifier call must agree with the @@ -5652,9 +5699,9 @@ mod tests { const FS: u32 = 0x0100_0000; // zero, -zero, denormal, quiet NaN, signalling NaN, one, -one, // smallest normal, half - let s = [0u64, 0x8000_0000, 1, 0x7FC0_0000, 0x7FA0_0000, 0x3F80_0000, 0xBF80_0000, 0x0080_0000, 0x3F00_0000]; + let s = [0u64, 0x8000_0000, 1, 0x7FA0_0000, 0x7FC0_0000, 0x3F80_0000, 0xBF80_0000, 0x0080_0000, 0x3F00_0000]; let d = [ - 0u64, 1 << 63, 1, 0x7FF8_0000_0000_0000, 0x7FF4_0000_0000_0000, 0x3FF0_0000_0000_0000, + 0u64, 1 << 63, 1, 0x7FF4_0000_0000_0000, 0x7FF8_0000_0000_0000, 0x3FF0_0000_0000_0000, 0xBFF0_0000_0000_0000, 0x0010_0000_0000_0000, 0x3FE0_0000_0000_0000, ]; let mut cases: Vec<(u32, u32, u64, u64, u32)> = Vec::new(); @@ -5792,10 +5839,11 @@ mod tests { } /// SQRT, RSQRT, the multiply-add family and CVT.S.D leave an Invalid - /// Operation's result as the default NaN, not the host's - /// (0xFFC00000 on x86, which has the sign set). The next instruction - /// must then see a quiet NaN and take Unimplemented (E) rather than - /// Invalid (V). Both engines, both formats. + /// Operation's result as the MIPS default NaN, not the host's + /// (0xFFC00000 on x86, which is a signalling NaN under the MIPS + /// encoding). The next instruction must then see a quiet NaN and take + /// Unimplemented (E) rather than Invalid (V). Both engines, both + /// formats. #[test] fn invalid_result_is_the_default_nan_and_reads_as_quiet() { use crate::cpu::mips_isa::{ @@ -5803,17 +5851,17 @@ mod tests { FUNCT_MADD_S, FUNCT_MSUB_D, FUNCT_MSUB_S, FUNCT_NMADD_D, FUNCT_NMADD_S, FUNCT_NMSUB_D, FUNCT_NMSUB_S, OP_COP1, OP_COP1X, RS_D, RS_S, }; - const NAN_S: u64 = 0x7FC0_0000; - const NAN_D: u64 = 0x7FF8_0000_0000_0000; + const NAN_S: u64 = 0x7FBF_FFFF; + const NAN_D: u64 = 0x7FF7_FFFF_FFFF_FFFF; // f1 = -1, f2 = infinity, f4 = zero, f6 = 1.0, f7 = a signalling - // NaN (quiet bit clear), by format. + // NaN (quiet bit set, the MIPS encoding), by format. let single = |f: &mut [u64; 32]| { f[1] = 0xBF80_0000; f[2] = 0x7F80_0000; f[4] = 0; f[6] = 0x3F80_0000; - f[7] = 0x7FA0_0000; + f[7] = 0x7FC0_0000; }; let double = |f: &mut [u64; 32]| { f[1] = 0xBFF0_0000_0000_0000; f[2] = 0x7FF0_0000_0000_0000; f[4] = 0; - f[6] = 0x3FF0_0000_0000_0000; f[7] = 0x7FF4_0000_0000_0000; + f[6] = 0x3FF0_0000_0000_0000; f[7] = 0x7FF8_0000_0000_0000; }; // (producer, double result, FPRs); every producer writes f3. let mut cases: Vec<(u32, bool, [u64; 32])> = Vec::new(); diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 32f78f4e3..c439085ea 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -132,11 +132,16 @@ const FCSR_IM: u32 = 0x00000080; // Enable: inexact const FCSR_FS: u32 = 0x01000000; // Flush denormalized results to zero /// True if `bits` (an IEEE-754 single-precision pattern) is a signalling NaN: -/// exponent all-1s, mantissa nonzero, and the mantissa's MSB (the "is quiet" -/// bit) clear. A quiet NaN has that bit set. +/// exponent all-1s, mantissa nonzero, and the mantissa's MSB set. This is the +/// original MIPS NaN encoding that the R4400, R5000 and R10000 use (MIPS64 +/// Volume I, Table 6.2), the reverse of IEEE 754-2008: a quiet NaN has that bit +/// clear, which is why the default NaN is 0x7FBFFFFF and not 0x7FC00000. The +/// R4400's behaviour is pinned by cpu-tests `fpu/qnan_operand`. #[inline] fn is_snan_s(bits: u32) -> bool { - (bits & 0x7F80_0000) == 0x7F80_0000 && (bits & 0x007F_FFFF) != 0 && (bits & 0x0040_0000) == 0 + // Exponent all ones and the fraction MSB set; the MSB makes the fraction + // nonzero, so that is the whole test. + (bits & 0x7FC0_0000) == 0x7FC0_0000 } /// Negate by flipping the sign bit, which is what IEEE-754 negation *is*. @@ -176,9 +181,7 @@ fn fneg_d_exact(v: f64) -> f64 { /// Same as `is_snan_s`, for IEEE-754 double-precision bit patterns. #[inline] fn is_snan_d(bits: u64) -> bool { - (bits & 0x7FF0_0000_0000_0000) == 0x7FF0_0000_0000_0000 - && (bits & 0x000F_FFFF_FFFF_FFFF) != 0 - && (bits & 0x0008_0000_0000_0000) == 0 + (bits & 0x7FF8_0000_0000_0000) == 0x7FF8_0000_0000_0000 } /// True if `bits` is a quiet NaN (any NaN that is not a signalling NaN). @@ -193,10 +196,11 @@ fn is_qnan_d(bits: u64) -> bool { f64::from_bits(bits).is_nan() && !is_snan_d(bits) } -/// The quiet NaN delivered for an untrapped Invalid Operation: the IEEE -/// 754-2008 default NaN, quiet bit set and nothing else. -const FPU_DEFAULT_NAN_S: u32 = 0x7FC0_0000; -const FPU_DEFAULT_NAN_D: u64 = 0x7FF8_0000_0000_0000; +/// The quiet NaN an R10000 delivers for an untrapped Invalid Operation: +/// MIPS64 Volume I, Table 6.3 ("Value Supplied When a New Quiet NaN Is +/// Created"), the column for the original MIPS NaN encoding the R10000 uses. +const FPU_DEFAULT_NAN_S: u32 = 0x7FBF_FFFF; +const FPU_DEFAULT_NAN_D: u64 = 0x7FF7_FFFF_FFFF_FFFF; /// SQRT, RECIP, RSQRT and the multiply-add family have no top-of-range /// classifier like ADD/SUB/MUL/DIV, but the host NaN they produce for an @@ -2319,7 +2323,7 @@ fn cvt_d_to_s_and_commit(core: &mut MipsCore, fs_reg: u32, fd_reg: u32, fr1: boo } // A NaN source never takes the path above (it is not equal to itself), // so the signalling test costs an ordinary conversion nothing. A - // signalling NaN is an Invalid Operation: V, and the default NaN + // signalling NaN is an Invalid Operation: V, and the MIPS default NaN // rather than the host's. A quiet NaN converts as before. if is_snan_d(src.to_bits()) { core.fpu_fcsr |= FCSR_CV; @@ -7945,11 +7949,20 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// itself must end the instruction, `None` to proceed to computation. #[inline] fn fpu_check_denorm_operand_s(&mut self, bits: u32) -> Option { + // One exponent test settles an ordinary number, the common case. + let e = bits & 0x7F80_0000; + if e != 0 && e != 0x7F80_0000 { + return None; + } let v = f32::from_bits(bits); (v.is_subnormal() || is_qnan_s(bits)).then(|| self.fpu_unimplemented()) } #[inline] fn fpu_check_denorm_operand_d(&mut self, bits: u64) -> Option { + let e = bits & 0x7FF0_0000_0000_0000; + if e != 0 && e != 0x7FF0_0000_0000_0000 { + return None; + } let v = f64::from_bits(bits); (v.is_subnormal() || is_qnan_d(bits)).then(|| self.fpu_unimplemented()) } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index dca0823b7..21f625f76 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -2744,14 +2744,15 @@ mod tests { } /// Same rule for a quiet NaN operand — the surprising half of the table: - /// an R4400/R5000 does not propagate NaN through ADD in hardware. + /// an R4400/R5000 does not propagate NaN through ADD in hardware. The + /// MIPS encoding has a quiet NaN with the fraction MSB clear. #[test] fn test_fpu_qnan_operand_is_unimplemented() { let (mut exec, _) = create_executor(); exec.core.cp0_status |= STATUS_CU1 | STATUS_FR; exec.update_fpr_mode(); - exec.core.write_fpr_w(1, 0x7FC0_0000); // quiet NaN + exec.core.write_fpr_w(1, 0x7FA0_0000); // quiet NaN exec.core.write_fpr_s(2, 1.0); exec.core.write_fpr_w(0, 0x5a5a5a5a); @@ -2762,6 +2763,87 @@ mod tests { assert_eq!(exec.core.read_fpr_w(0), 0x5a5a5a5a, "destination must be untouched on trap"); } + /// The default NaN an untrapped Invalid Operation delivers (0x7FBFFFFF, + /// 0x7FF7FFFF_FFFFFFFF) is a quiet NaN in the MIPS encoding, so used as an + /// operand it is not an Invalid Operation. It takes the quiet-NaN path + /// (Unimplemented Operation), the same as any other quiet NaN. Under + /// IEEE 754-2008 the fraction MSB set means quiet, which would make this + /// pattern signalling. + #[test] + fn test_fpu_default_nan_operand_is_not_invalid() { + for (is_d, nan) in [(false, 0x7FBF_FFFFu64), (true, 0x7FF7_FFFF_FFFF_FFFFu64)] { + for funct in [FUNCT_FADD, FUNCT_FMUL, FUNCT_FDIV] { + let (mut exec, _) = create_executor(); + exec.core.cp0_status |= STATUS_CU1 | STATUS_FR; + exec.update_fpr_mode(); + + let fmt = if is_d { RS_D } else { RS_S }; + if is_d { + exec.core.write_fpr_d(1, f64::from_bits(nan)); + exec.core.write_fpr_d(2, 1.0); + exec.core.write_fpr_d(0, f64::from_bits(0x5a5a_5a5a_5a5a_5a5a)); + } else { + exec.core.write_fpr_w(1, nan as u32); + exec.core.write_fpr_s(2, 1.0); + exec.core.write_fpr_w(0, 0x5a5a_5a5a); + } + let s = exec.exec(make_cop1_compute(fmt, 2, 1, 0, funct)); + let fcsr = exec.core.read_fpu_control(31); + let what = format!("is_d={is_d} funct={funct:#x}"); + assert!(s & EXEC_IS_EXCEPTION != 0, "{what}: quiet NaN operand traps"); + assert_eq!((s >> 2) & 0x1F, crate::cpu::mips_exec::EXC_FPE, "{what}"); + assert_eq!(fcsr & 0x0002_0000, 0x0002_0000, "{what}: Cause.E"); + assert_eq!(fcsr & 0x0001_0000, 0, "{what}: no Cause.V"); + assert_eq!(fcsr & 0x40, 0, "{what}: no Flag.V"); + } + } + } + + /// A signalling NaN (fraction MSB set, MIPS encoding) operand is an + /// Invalid Operation, not Unimplemented: untrapped it sets Cause.V and + /// Flag.V and completes. cpu-tests `fpu/qnan_operand` measures this on an + /// R4400 with 0x7FC00000. + #[test] + fn test_fpu_snan_operand_is_invalid() { + for (is_d, nan) in [(false, 0x7FC0_0000u64), (true, 0x7FF8_0000_0000_0000u64), + (false, 0x7FFF_FFFFu64), (true, 0xFFF8_0000_0000_0001u64)] { + for funct in [FUNCT_FADD, FUNCT_FMUL, FUNCT_FDIV] { + let (mut exec, _) = create_executor(); + exec.core.cp0_status |= STATUS_CU1 | STATUS_FR; + exec.update_fpr_mode(); + + let fmt = if is_d { RS_D } else { RS_S }; + if is_d { + exec.core.write_fpr_d(1, f64::from_bits(nan)); + exec.core.write_fpr_d(2, 1.0); + } else { + exec.core.write_fpr_w(1, nan as u32); + exec.core.write_fpr_s(2, 1.0); + } + let s = exec.exec(make_cop1_compute(fmt, 2, 1, 0, funct)); + let fcsr = exec.core.read_fpu_control(31); + let what = format!("is_d={is_d} nan={nan:#x} funct={funct:#x}"); + assert!(s & EXEC_IS_EXCEPTION == 0, "{what}: untrapped Invalid completes"); + assert_eq!(fcsr & 0x0001_f000, 0x0001_0000, "{what}: Cause is V alone"); + assert_eq!(fcsr & 0x7c, 0x40, "{what}: Flag is V alone"); + } + } + + // With the Invalid enable set it traps, Cause.V, destination untouched. + let (mut exec, _) = create_executor(); + exec.core.cp0_status |= STATUS_CU1 | STATUS_FR; + exec.update_fpr_mode(); + exec.core.fpu_fcsr = 0x0000_0800; + exec.core.write_fpr_w(1, 0x7FC0_0000); + exec.core.write_fpr_s(2, 1.0); + exec.core.write_fpr_w(0, 0x5a5a_5a5a); + let s = exec.exec(make_cop1_compute(RS_S, 2, 1, 0, FUNCT_FADD)); + assert!(s & EXEC_IS_EXCEPTION != 0, "enabled Invalid traps"); + assert_eq!((s >> 2) & 0x1F, crate::cpu::mips_exec::EXC_FPE); + assert_eq!(exec.core.read_fpu_control(31) & 0x0001_f000, 0x0001_0000, "Cause is V alone"); + assert_eq!(exec.core.read_fpr_w(0), 0x5a5a_5a5a, "destination untouched on trap"); + } + /// Denormalized result, FS clear: Unimplemented Operation, destination /// untouched. 2^-126 * 0.5 = 2^-127, denormal in single precision. #[test] @@ -3001,8 +3083,8 @@ mod tests { /// Invalid operations: infinity minus infinity, zero times infinity, /// 0/0 and infinity/infinity (VR5000 manual U11761EJ6V0UM, 9.4.2, the /// MIPS IV list). Untrapped, Cause.V and Flag.V are set and the result - /// is the default quiet NaN, 0x7FC00000 / 0x7FF80000_00000000 (MIPS64 - /// Volume I, Table 6.3, IEEE 754-2008 column). 0/0 is not a divide by + /// is the default quiet NaN, 0x7FBFFFFF / 0x7FF7FFFF_FFFFFFFF (MIPS64 + /// Volume I, Table 6.3, original NaN encoding). 0/0 is not a divide by /// zero: Z needs a finite nonzero dividend (9.4.3). #[test] fn test_fpu_invalid_operation_untrapped() { @@ -3026,7 +3108,7 @@ mod tests { for (fmt, funct, a, b) in cases { let (s, r, fcsr) = fpu_top_case(fmt, funct, a, b, 0); assert_eq!(s, EXEC_COMPLETE, "fmt={fmt} funct={funct}"); - let want = if fmt == RS_S { 0x7FC0_0000 } else { 0x7FF8_0000_0000_0000 }; + let want = if fmt == RS_S { 0x7FBF_FFFF } else { 0x7FF7_FFFF_FFFF_FFFF }; assert_eq!(r, want, "default NaN, fmt={fmt} funct={funct}"); assert_eq!(fcsr, FCSR_CAUSE_V | FCSR_FLAG_V, "fmt={fmt} funct={funct}"); } @@ -3045,12 +3127,12 @@ mod tests { #[test] fn test_fpu_default_nan_result_reused_as_operand() { for (fmt, snan, one) in [ - (RS_S, 0x7FA0_0000u64, 1.0f32.to_bits() as u64), - (RS_D, 0x7FF4_0000_0000_0000, 1.0f64.to_bits()), + (RS_S, 0x7FC0_0000u64, 1.0f32.to_bits() as u64), + (RS_D, 0x7FF8_0000_0000_0000, 1.0f64.to_bits()), ] { let (s, default_nan, fcsr) = fpu_top_case(fmt, FUNCT_FADD, snan, one, 0); assert_eq!(s, EXEC_COMPLETE, "fmt={fmt}"); - let want = if fmt == RS_S { 0x7FC0_0000 } else { 0x7FF8_0000_0000_0000 }; + let want = if fmt == RS_S { 0x7FBF_FFFF } else { 0x7FF7_FFFF_FFFF_FFFF }; assert_eq!(default_nan, want, "sNaN operand delivers the default NaN, fmt={fmt}"); assert_eq!(fcsr, FCSR_CAUSE_V | FCSR_FLAG_V, "fmt={fmt}"); @@ -3196,8 +3278,8 @@ mod tests { /// Instructions whose invalid result does not come through the plain /// arithmetic path: SQRT and RSQRT of -1, every multiply-add on - /// infinity times zero, and CVT.S.D of a signalling NaN (quiet bit - /// clear). Each entry is (instruction, source is double, + /// infinity times zero, and CVT.S.D of a signalling NaN (quiet bit set, + /// the MIPS encoding). Each entry is (instruction, source is double, /// f1, f2, f4, result is double); f2 is zero, f4 is 1.0. fn fpu_invalid_producers() -> Vec<(u32, bool, u64, u64, u64, bool)> { let mut v = Vec::new(); @@ -3219,20 +3301,20 @@ mod tests { v.push((make_cop1x(4, 2, 1, 0, funct), d, inf, zero, one, d)); } } - v.push((make_cop1_compute(RS_D, 0, 1, 0, FUNCT_FCVT_S), true, 0x7FF4_0000_0000_0000, 0, 0, false)); + v.push((make_cop1_compute(RS_D, 0, 1, 0, FUNCT_FCVT_S), true, 0x7FF8_0000_0000_0000, 0, 0, false)); v } /// SQRT, RSQRT, the multiply-add family and CVT.S.D give the default /// quiet NaN with Cause.V and Flag.V for an invalid operation (VR5000 /// manual U11761EJ6V0UM, 9.4.2; MIPS64 Volume I, Table 6.3), not the - /// host's NaN (0xFFC00000 on x86, a quiet NaN with the sign set). + /// host's NaN, which is a signalling NaN under the MIPS encoding. #[test] fn test_fpu_invalid_unary_and_madd_untrapped() { for (instr, dbl, a, b, c, dbl_result) in fpu_invalid_producers() { let (s, r, fcsr) = fpu_instr_case(instr, dbl, a, b, c, 0, dbl_result); assert_eq!(s, EXEC_COMPLETE, "instr={instr:#010x}"); - let want = if dbl_result { 0x7FF8_0000_0000_0000u64 } else { 0x7FC0_0000 }; + let want = if dbl_result { 0x7FF7_FFFF_FFFF_FFFFu64 } else { 0x7FBF_FFFF }; assert_eq!(r, want, "default NaN, instr={instr:#010x}"); assert_eq!(fcsr, FCSR_CAUSE_V | FCSR_FLAG_V, "instr={instr:#010x}"); }