cpu: fix R10000 TLB widths, refused stores, the O2 alias and partial LWR

CHANGELOG.md

@@ -114,6 +114,47 @@ hashes are given where a change is easiest to understand by reading the commit.
   the cost follows what the guest wrote. Reflinking is unchanged. On Windows the
   copy is marked sparse first.
 
+### R10000 TLB widths
+
+- **Full 44-bit user addresses.** On the R10000 (Octane, IP28, and an O2 set to
+  `r10000`) EntryHi, the TLB match and the user-segment check keep VA bits 43:40;
+  they stopped at bit 39 before, so an address above 1 TiB took an Address Error.
+  The R4400 and R5000 are unchanged.
+- **FrameMask (CP0 register 21)** on the R10000 clears the EntryLo bits it
+  names, 33:18, on every TLB write, so a kernel that keeps software flags there
+  gets the physical address it meant. The register read as zero and ignored
+  writes before. The R4400 and R5000 have none.
+- **EntryLo** on the R10000 keeps its 28-bit PFN (bits 33:6, a 40-bit physical
+  address) and the uncached attribute in 63:62; it kept 30 bits before. The
+  attribute is stored and read back, not sent to the bus.
+- **TLB writes on the R10000** store zeros in the PFN bits below the page size,
+  so TLBR reads them back as zero, as the part does; on 16 KB pages that is
+  EntryLo bits 7:6. Translation is unchanged. The PROM's `ide` `tlb` test
+  failed 320 compares on this. The manual documents it for the R10000; the
+  R12000, R14000 and R16000 do the same as a model choice. The R4400 and
+  R5000 still keep the whole PFN.
+- **TLB refill vector on the R10000** follows the address space that missed:
+  a user address takes the XTLB vector only with UX set, xsseg with SX, and
+  kernel addresses with KX. A 64-bit kernel touching a 32-bit user address now
+  refills through 0x000, as the part does, not 0x080. The R4400 and R5000 still
+  choose by the current mode.
+
+### Refused uncached stores on the R10000 family
+
+- On the R10000 family (Octane, IP28, O2), an uncached store that nothing answers now
+  completes instead of raising a Data Bus Error, as the processor does. iris prints
+  "Unsupported uncached write ... (posted, completes)" for the first four such
+  stores and then at each power of two. Loads, instruction fetches, the R4400 and
+  the R5000 are unchanged.
+
+### Partial LWR on the R10000
+
+- On the R10000 (Octane, IP28, and an O2 with the R10000), an `LWR` that loads fewer
+  than four bytes now keeps bits 63:32 of the destination instead of sign-extending,
+  as the reference results for the part show. R4400 and R5000 machines are unchanged.
+  The JIT matches the interpreter, and the code-cache key changes only for the
+  machines that have the behaviour.
+
 ### `IRIS_GUI_GL` works in iris-gui
 
 - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never
@@ -513,6 +554,9 @@ hashes are given where a change is easiest to understand by reading the commit.
 
 ### SGI O2 (IP32)
 
+- **The O2's memory alias holds for pages with high PTE bits set.** A mapped page
+  whose address had a bit set above bit 31 missed the alias of RAM at
+  0x8000_0000 and reached the PCI window instead.
 - **The O2 runs on the shadow cache from reset.** The full cache model, and
   the switch to and from it during POST and reboots, are gone
   (`src/ip32_handover.rs`), and with them a branch on every cached load, store

src/cpu/jitv2/codegen.rs

@@ -868,6 +868,10 @@ impl Codegen {
         for v in isa.isa_flags() {
             let _ = write!(s, " {v}");
         }
+        // Only when set, so machines without the behaviour keep their warm cache.
+        if self.jit_consts.lwr_keeps_upper {
+            s.push_str(" lwr_upper=1");
+        }
         s.push_str(" ops=");
         s.extend(crate::cpu::jitv2::opcode_support::enabled_snapshot().iter().map(|&b| if b { '1' } else { '0' }));
         Some(blake3::hash(s.as_bytes()).as_bytes()[..16].try_into().unwrap())
@@ -4379,6 +4383,11 @@ pub struct JitConsts {
     /// to loading from memory, exactly as before, so an un-stamped `Codegen`
     /// stays correct (just no faster).
     pub core: Option<core::num::NonZeroUsize>,
+    /// A partial LWR keeps bits 63:32 of the destination, as the R10000
+    /// does (`CpuModel::PARTIAL_LWR_KEEPS_UPPER`). A model fact rather than a
+    /// core one, published beside the core address because both reach a worker's
+    /// `Codegen` the same way. The code-cache fingerprint carries it when set.
+    pub lwr_keeps_upper: bool,
 }
 
 impl JitConsts {
@@ -10013,6 +10022,37 @@ fn emit_lwl_ldl(ctx: &mut EmitCtx, size: MemSize) {
 /// `MipsExecutor::exec_lwr`/`exec_ldr` exactly: shift amount is
 /// `(width-1-byte_offset)*8`, the opposite sense from LWL/LDL's plain
 /// `byte_offset*8`.
+/// LWR on a model whose partial LWR keeps rt[63:32] (`lwr_keeps_upper`),
+/// mirroring `MipsExecutor::exec_lwr`. The merge runs on the whole register,
+/// so rt's upper word survives it (`!mask` has ones there). Offsets 0 to 2
+/// keep it while 64-bit operations are on; offset 3 and any word with them
+/// off sign-extends the merged low half. The executor keeps
+/// `core.lwr_upper_below` for that test, 3 while 64-bit operations are on
+/// and 0 otherwise, so the choice is one unsigned compare of the offset with
+/// a byte (a Status change inside a region, a fallback MTC0 or ERET,
+/// resyncs it before the next instruction runs). `loaded` is the aligned
+/// word, zero-extended.
+fn emit_lwr_keeping_upper(ctx: &mut EmitCtx, loaded: Value, byte_offset: Value) {
+    let i64t = ir::types::I64;
+    // Bytes of the word the register keeps: 8 * (3 - offset).
+    let inverted = ctx.builder.ins().bxor_imm_s(byte_offset, 3);
+    let shift = ctx.builder.ins().ishl_imm_s(inverted, 3);
+    let ones = ctx.builder.ins().iconst(i64t, 0xFFFF_FFFF);
+    let mask = ctx.builder.ins().ushr(ones, shift);
+    let shifted_mem = ctx.builder.ins().ushr(loaded, shift);
+    let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw));
+    let not_mask = ctx.builder.ins().bnot(mask);
+    let preserved = ctx.builder.ins().band(rt_val, not_mask);
+    let merged = ctx.builder.ins().bor(shifted_mem, preserved);
+    let low = ctx.builder.ins().ireduce(ir::types::I32, merged);
+    let sign_extended = ctx.builder.ins().sextend(i64t, low);
+    let below_off = ir::immediates::Offset32::new(std::mem::offset_of!(MipsCore, lwr_upper_below) as i32);
+    let below = ctx.builder.ins().uload8(i64t, MemFlagsData::trusted(), ctx.core_ptr, below_off);
+    let keep = ctx.builder.ins().icmp(IntCC::UnsignedLessThan, byte_offset, below);
+    let result = ctx.builder.ins().select(keep, merged, sign_extended);
+    emit_write_gpr(ctx, field_rt(ctx.raw), result);
+}
+
 fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) {
     let base = emit_read_gpr(ctx, field_rs(ctx.raw));
     let imm = field_imm16_sext(ctx.builder, ctx.raw);
@@ -10024,6 +10064,11 @@ fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) {
 
     let loaded = emit_mem_read(ctx, aligned_addr, size);
 
+    if size == MemSize::B4 && ctx.jit_consts.lwr_keeps_upper {
+        emit_lwr_keeping_upper(ctx, loaded, byte_offset);
+        return;
+    }
+
     let ity = size.ir_type();
     let mem_val = if size == MemSize::B8 { loaded } else { ctx.builder.ins().ireduce(ity, loaded) };
     let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw));
@@ -10811,6 +10856,18 @@ mod tests {
         (op << 26) | (rs << 21) | (rt << 16) | (rd << 11) | (sa << 6) | funct
     }
 
+    #[test]
+    fn cache_fingerprint_includes_the_lwr_choice() {
+        let base = Codegen::new().cache_fingerprint(false).expect("default codegen is cacheable");
+        let mut codegen = Codegen::new();
+        codegen.jit_consts.lwr_keeps_upper = true;
+        assert_ne!(
+            codegen.cache_fingerprint(false).expect("lwr codegen is cacheable"),
+            base,
+            "the LWR choice must select a different persistent cache namespace",
+        );
+    }
+
     /// `is_inlinable` must reject on *both* of its conditions, not just the
     /// bounds one. The bounds half is what panicked
     /// (`clanker/jitv2/nested-delay-slot-across-page-boundary.md`); the

src/cpu/jitv2/equiv_test.rs

@@ -4355,6 +4355,109 @@ mod tests {
         }
     }
 
+    /// Run `words` from `pc` once through the interpreter and once as a
+    /// compiled region, both on CPU model `C` over a fresh `bus()` and seeded
+    /// with `gpr` and CP0 Status `status`. The JIT learns the model's choices from `install_jit_hooks`,
+    /// as in production.
+    fn interpreter_and_jit<C>(bus: impl Fn() -> Arc<dyn BusDevice>, pc: u64, gpr: [u64; 32], status: u32, words: &[u32])
+        -> (CoreSnapshot, CoreSnapshot)
+    where
+        C: crate::cpu::mips_cache_v2::CpuModel + From<Arc<dyn BusDevice>>,
+    {
+        let build = || {
+            let cfg = MipsCpuConfig::for_model::<C>(Arc::new(crate::ip22_bus::memory_map(false)));
+            let mut exec = Box::new(MipsExecutor::<PassthroughTlb, C>::new(bus(), PassthroughTlb::default(), &cfg));
+            exec.core.gpr = gpr;
+            exec.core.pc = pc;
+            exec.set_cp0_status(status);
+            exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed);
+            #[cfg(feature = "fetchverify")]
+            { exec.core.fetch_verify_on = false; }
+            exec
+        };
+        let mut interp = build();
+        interp.jitv2_dispatch_enabled = false;
+        for (i, &word) in words.iter().enumerate() {
+            if interp.core.pc != pc + 4 * i as u64 {
+                break; // an exception redirected the PC
+            }
+            interp.exec(word);
+        }
+
+        let mut page = [0u32; ENTRIES_PER_PAGE];
+        page[..words.len()].copy_from_slice(words);
+        let page_base = (pc & !(PAGE_SIZE as u64 - 1)) as u32;
+        let mut analyzer = Analyzer::new();
+        let (walked, non_empty) = analyzer.walk_bounded(&page, 0, page_base, words.len());
+        assert!(non_empty);
+        let mut instrs_owned = *walked;
+        let mut codegen = Codegen::new();
+        codegen.jit_consts.lwr_keeps_upper = C::PARTIAL_LWR_KEEPS_UPPER;
+        let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false)
+            .expect("region must be compilable for this test to be meaningful");
+        let mut jit = build();
+        jit.install_jit_hooks();
+        // The compile pool's workers read the model's choice from here, so a
+        // missing publication would not show if `codegen` were stamped by hand.
+        assert_eq!(jit.jitv2.lock().jit_consts.lock().lwr_keeps_upper, C::PARTIAL_LWR_KEEPS_UPPER);
+        unsafe { jit_fn(&mut jit.core as *mut MipsCore) };
+        std::mem::forget(codegen);
+        (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core))
+    }
+
+    /// The R10000 keeps bits 63:32 on a partial LWR while 64-bit operations are
+    /// enabled for the current mode and the R5000 sign-extends; with them off
+    /// (a 32-bit user process, UX=0) the R10000 sign-extends too. The JIT and
+    /// the interpreter must agree at every offset in every mode, and the
+    /// interpreter's result is the one `partial_lwr_keeps_the_upper_word_only_on_the_r10000`
+    /// pins.
+    #[test]
+    fn jit_lwr_matches_the_interpreter_on_the_r10000_and_r5000() {
+        use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R5000ShadowCache};
+        fn check<C: crate::cpu::mips_cache_v2::CpuModel + From<Arc<dyn BusDevice>>>(model_keeps_upper: bool) {
+            use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_KX, STATUS_SX, STATUS_UX};
+            const RESERVED: u32 = 3 << 3;
+            const USER: u32 = 2 << 3;
+            const SUPER: u32 = 1 << 3;
+            // (Status, whether 64-bit operations are on): each mode's own
+            // bit decides, whatever the other two hold.
+            for (status, ops64, mode) in [
+                (STATUS_KX, true, "kernel KX=1"),
+                (0, false, "kernel KX=0"),
+                (STATUS_UX | STATUS_SX, false, "kernel KX=0, UX/SX=1"),
+                (USER | STATUS_UX, true, "user UX=1"),
+                (USER | STATUS_KX | STATUS_SX, false, "user UX=0, KX/SX=1"),
+                (SUPER | STATUS_SX, true, "supervisor SX=1"),
+                (SUPER | STATUS_KX | STATUS_UX, false, "supervisor SX=0"),
+                (USER | STATUS_EXL | STATUS_KX, true, "EXL forces kernel, KX=1"),
+                (USER | STATUS_EXL | STATUS_UX, false, "EXL forces kernel, UX=1 KX=0"),
+                (SUPER | STATUS_ERL | STATUS_KX, true, "ERL forces kernel, KX=1"),
+                (SUPER | STATUS_ERL | STATUS_SX, false, "ERL forces kernel, SX=1 KX=0"),
+                (RESERVED | STATUS_KX, true, "reserved KSU counts as kernel, KX=1"),
+                (RESERVED | STATUS_UX | STATUS_SX, false, "reserved KSU counts as kernel, KX=0"),
+            ] {
+                for offset in 0..4u64 {
+                    let mut gpr = [0u64; 32];
+                    gpr[1] = 0x2000 + offset;
+                    gpr[2] = 0x1122_3344_5566_7788;
+                    let bus = || -> Arc<dyn BusDevice> {
+                        let mem = Arc::new(MockMemory::new_not_compilable());
+                        mem.set_word(0x2000, 0xDEAD_BEEF);
+                        mem
+                    };
+                    let lwr = make_i(crate::cpu::mips_isa::OP_LWR, 1, 2, 0);
+                    let (interp, jit) = interpreter_and_jit::<C>(bus, 0x1000, gpr, status, &[lwr]);
+                    assert_eq!(jit, interp, "{} JIT/interpreter LWR offset {offset}, {mode}", C::NAME);
+                    let upper_kept = interp.gpr[2] >> 32 == 0x1122_3344;
+                    assert_eq!(upper_kept, model_keeps_upper && ops64 && offset != 3,
+                        "{} LWR offset {offset}, {mode}", C::NAME);
+                }
+            }
+        }
+        check::<R10000ShadowCache>(true);
+        check::<R5000ShadowCache>(false);
+    }
+
     #[test]
     fn lwl_lwr_combine_to_a_full_unaligned_word_load_matches_interpreter() {
         // The canonical compiler-generated idiom for an unaligned word
@@ -7879,4 +7982,74 @@ mod tests {
             println!("{:<12} {:8.3} ns/instruction", name, ns);
         }
     }
+
+    /// R10000: a refused uncached store completes in compiled code exactly as
+    /// in the interpreter, through both store sites (SW, and SWL by the
+    /// masked-store helper), and the instructions after it run. The same access
+    /// as a load, and a store on the R5000, still fault in both engines.
+    #[test]
+    fn refused_uncached_store_matches_the_interpreter_on_the_r10000_and_r5000() {
+        use crate::cpu::mips_cache_v2::{R10000Cache, R5000Cache, CpuModel};
+        use crate::cpu::mips_core::STATUS_EXL;
+        use crate::cpu::mips_isa::{OP_ADDIU, OP_LW, OP_SW, OP_SWL};
+
+        /// Nothing answers: every read and write is refused (the trait's defaults).
+        struct RefusingBus;
+        impl BusDevice for RefusingBus {}
+
+        fn run<C: CpuModel + From<Arc<dyn BusDevice>>>(op: u32) -> (CoreSnapshot, CoreSnapshot) {
+            let pc = 0xFFFF_FFFF_8000_0000u64;
+            let mut gpr = [0u64; 32];
+            gpr[16] = 0xFFFF_FFFF_A100_0000; // kseg1: uncached, and nothing answers
+            gpr[6] = 0x1234_5678;
+            let words = [make_i(op, 16, 6, 0), make_i(OP_ADDIU, 0, 7, 2)];
+            let build = || {
+                let bus: Arc<dyn BusDevice> = Arc::new(RefusingBus);
+                let cfg = MipsCpuConfig::for_model::<C>(Arc::new(crate::ip22_bus::memory_map(false)));
+                let mut exec = Box::new(MipsExecutor::<PassthroughTlb, C>::new(bus, PassthroughTlb::default(), &cfg));
+                exec.core.gpr = gpr;
+                exec.core.pc = pc;
+                exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed);
+                #[cfg(feature = "fetchverify")]
+                { exec.core.fetch_verify_on = false; }
+                exec
+            };
+            let mut interp = build();
+            interp.jitv2_dispatch_enabled = false;
+            for (i, &word) in words.iter().enumerate() {
+                if interp.core.pc != pc + 4 * i as u64 {
+                    break; // an exception redirected the PC
+                }
+                interp.exec(word);
+            }
+
+            let mut page = [0u32; ENTRIES_PER_PAGE];
+            page[..words.len()].copy_from_slice(&words);
+            let page_base = (pc & !(PAGE_SIZE as u64 - 1)) as u32;
+            let mut analyzer = Analyzer::new();
+            let (walked, non_empty) = analyzer.walk_bounded(&page, 0, page_base, words.len());
+            assert!(non_empty);
+            let mut instrs_owned = *walked;
+            let mut codegen = Codegen::new();
+            let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false)
+                .expect("region must be compilable for this test to be meaningful");
+            let mut jit = build();
+            jit.install_jit_hooks();
+            unsafe { jit_fn(&mut jit.core as *mut MipsCore) };
+            std::mem::forget(codegen);
+            (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core))
+        }
+
+        for op in [OP_SW, OP_SWL] {
+            let (interp, jit) = run::<R10000Cache>(op);
+            assert_eq!(jit, interp, "op {op:#x}: JIT and interpreter diverged");
+            assert_eq!(jit.cp0_status & STATUS_EXL, 0, "op {op:#x}: no exception");
+            assert_eq!(jit.gpr[7], 2, "op {op:#x}: the instruction after the store ran");
+        }
+        for (name, (interp, jit)) in [("R10000 load", run::<R10000Cache>(OP_LW)), ("R5000 store", run::<R5000Cache>(OP_SW))] {
+            assert_eq!(jit, interp, "{name}: JIT and interpreter diverged");
+            assert_ne!(jit.cp0_status & STATUS_EXL, 0, "{name}: must take its Data Bus Error");
+            assert_eq!(jit.gpr[7], 0, "{name}: nothing after the fault ran");
+        }
+    }
 }

src/cpu/jitv2/mod.rs

@@ -92,6 +92,7 @@ mod zz_corpus {
             // (baked immediate vs load) is identical to production.
             crate::cpu::jitv2::codegen::JitConsts {
                 core: core::num::NonZeroUsize::new(core as *mut MipsCore as usize),
+                ..Default::default()
             }
         };
         // Synthetic corpus-measurement geometry: 16 KiB direct-mapped L1-D

src/cpu/mips_cache_shadow.rs

@@ -347,7 +347,10 @@ impl<
         _ => "shadow",
     };
     const CACHE_KIND: &'static str = "shadow";
+    const R10K_FAMILY: bool = Self::R10K_CACHE_OPS;
+    const PARTIAL_LWR_KEEPS_UPPER: bool = MODEL == model::R10000;
     const R10K_CACHE_OPS: bool = MODEL == model::R10000;
+    const POSTS_UNCACHED_STORES: bool = Self::R10K_CACHE_OPS;
 }
 
 impl<