diff --git a/CHANGELOG.md b/CHANGELOG.md index e73a77b8d..32f8de788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,55 @@ hashes are given where a change is easiest to understand by reading the commit. the cost follows what the guest wrote. Reflinking is unchanged. On Windows the copy is marked sparse first. +### R10000 TLB widths + +- **Full 44-bit user addresses.** On the R10000 (Octane, IP28, and an O2 set to + `r10000`) EntryHi, the TLB match and the user-segment check keep VA bits 43:40; + they stopped at bit 39 before, so an address above 1 TiB took an Address Error. + The R4400 and R5000 are unchanged. +- **FrameMask (CP0 register 21)** on the R10000 clears the EntryLo bits it + names, 33:18, on every TLB write, so a kernel that keeps software flags there + gets the physical address it meant. The register read as zero and ignored + writes before. The R4400 and R5000 have none. +- **EntryLo** on the R10000 keeps its 28-bit PFN (bits 33:6, a 40-bit physical + address) and the uncached attribute in 63:62; it kept 30 bits before. The + attribute is stored and read back, not sent to the bus. +- **TLB writes on the R10000** store zeros in the PFN bits below the page size, + so TLBR reads them back as zero, as the part does; on 16 KB pages that is + EntryLo bits 7:6. Translation is unchanged. The PROM's `ide` `tlb` test + failed 320 compares on this. The manual documents it for the R10000; the + R12000, R14000 and R16000 do the same as a model choice. The R4400 and + R5000 still keep the whole PFN. +- **TLB refill vector on the R10000** follows the address space that missed: + a user address takes the XTLB vector only with UX set, xsseg with SX, and + kernel addresses with KX. A 64-bit kernel touching a 32-bit user address now + refills through 0x000, as the part does, not 0x080. The R4400 and R5000 still + choose by the current mode. + +### Refused uncached stores on the R10000 family + +- On the R10000 family (Octane, IP28, O2), an uncached store that nothing answers now + completes instead of raising a Data Bus Error, as the processor does. iris prints + "Unsupported uncached write ... (posted, completes)" for the first four such + stores and then at each power of two. Loads, instruction fetches, the R4400 and + the R5000 are unchanged. + +### Partial LWR on the R10000 + +- On the R10000 (Octane, IP28, and an O2 with the R10000), an `LWR` that loads fewer + than four bytes now keeps bits 63:32 of the destination instead of sign-extending, + as the reference results for the part show. R4400 and R5000 machines are unchanged. + The JIT matches the interpreter, and the code-cache key changes only for the + machines that have the behaviour. + +### R10000 cache operations + +- **Index Load Data and Index Store Data** on the R10000's primary caches keep their + data (they read back zero before). The secondary cache's data slot is the address + shifted right by 3 instead of 4: one slot per doubleword. +- **Secondary-cache tags** on the R10000 family read the reserved bits (TagLo 9, 12, 13 and + TagHi 30:4) as zero; the MRU bit and the other tag fields are kept. + ### `IRIS_GUI_GL` works in iris-gui - The GUI's GL capture path (`IRIS_GUI_GL=1`, or `[debug] gui_gl_capture`) never @@ -513,6 +562,9 @@ hashes are given where a change is easiest to understand by reading the commit. ### SGI O2 (IP32) +- **The O2's memory alias holds for pages with high PTE bits set.** A mapped page + whose address had a bit set above bit 31 missed the alias of RAM at + 0x8000_0000 and reached the PCI window instead. - **The O2 runs on the shadow cache from reset.** The full cache model, and the switch to and from it during POST and reboots, are gone (`src/ip32_handover.rs`), and with them a branch on every cached load, store diff --git a/src/cpu/jitv2/codegen.rs b/src/cpu/jitv2/codegen.rs index ee31f90cf..ce266af9e 100644 --- a/src/cpu/jitv2/codegen.rs +++ b/src/cpu/jitv2/codegen.rs @@ -868,6 +868,10 @@ impl Codegen { for v in isa.isa_flags() { let _ = write!(s, " {v}"); } + // Only when set, so machines without the behaviour keep their warm cache. + if self.jit_consts.lwr_keeps_upper { + s.push_str(" lwr_upper=1"); + } s.push_str(" ops="); s.extend(crate::cpu::jitv2::opcode_support::enabled_snapshot().iter().map(|&b| if b { '1' } else { '0' })); Some(blake3::hash(s.as_bytes()).as_bytes()[..16].try_into().unwrap()) @@ -4379,6 +4383,11 @@ pub struct JitConsts { /// to loading from memory, exactly as before, so an un-stamped `Codegen` /// stays correct (just no faster). pub core: Option, + /// A partial LWR keeps bits 63:32 of the destination, as the R10000 + /// does (`CpuModel::PARTIAL_LWR_KEEPS_UPPER`). A model fact rather than a + /// core one, published beside the core address because both reach a worker's + /// `Codegen` the same way. The code-cache fingerprint carries it when set. + pub lwr_keeps_upper: bool, } impl JitConsts { @@ -10013,6 +10022,37 @@ fn emit_lwl_ldl(ctx: &mut EmitCtx, size: MemSize) { /// `MipsExecutor::exec_lwr`/`exec_ldr` exactly: shift amount is /// `(width-1-byte_offset)*8`, the opposite sense from LWL/LDL's plain /// `byte_offset*8`. +/// LWR on a model whose partial LWR keeps rt[63:32] (`lwr_keeps_upper`), +/// mirroring `MipsExecutor::exec_lwr`. The merge runs on the whole register, +/// so rt's upper word survives it (`!mask` has ones there). Offsets 0 to 2 +/// keep it while 64-bit operations are on; offset 3 and any word with them +/// off sign-extends the merged low half. The executor keeps +/// `core.lwr_upper_below` for that test, 3 while 64-bit operations are on +/// and 0 otherwise, so the choice is one unsigned compare of the offset with +/// a byte (a Status change inside a region, a fallback MTC0 or ERET, +/// resyncs it before the next instruction runs). `loaded` is the aligned +/// word, zero-extended. +fn emit_lwr_keeping_upper(ctx: &mut EmitCtx, loaded: Value, byte_offset: Value) { + let i64t = ir::types::I64; + // Bytes of the word the register keeps: 8 * (3 - offset). + let inverted = ctx.builder.ins().bxor_imm_s(byte_offset, 3); + let shift = ctx.builder.ins().ishl_imm_s(inverted, 3); + let ones = ctx.builder.ins().iconst(i64t, 0xFFFF_FFFF); + let mask = ctx.builder.ins().ushr(ones, shift); + let shifted_mem = ctx.builder.ins().ushr(loaded, shift); + let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw)); + let not_mask = ctx.builder.ins().bnot(mask); + let preserved = ctx.builder.ins().band(rt_val, not_mask); + let merged = ctx.builder.ins().bor(shifted_mem, preserved); + let low = ctx.builder.ins().ireduce(ir::types::I32, merged); + let sign_extended = ctx.builder.ins().sextend(i64t, low); + let below_off = ir::immediates::Offset32::new(std::mem::offset_of!(MipsCore, lwr_upper_below) as i32); + let below = ctx.builder.ins().uload8(i64t, MemFlagsData::trusted(), ctx.core_ptr, below_off); + let keep = ctx.builder.ins().icmp(IntCC::UnsignedLessThan, byte_offset, below); + let result = ctx.builder.ins().select(keep, merged, sign_extended); + emit_write_gpr(ctx, field_rt(ctx.raw), result); +} + fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) { let base = emit_read_gpr(ctx, field_rs(ctx.raw)); let imm = field_imm16_sext(ctx.builder, ctx.raw); @@ -10024,6 +10064,11 @@ fn emit_lwr_ldr(ctx: &mut EmitCtx, size: MemSize) { let loaded = emit_mem_read(ctx, aligned_addr, size); + if size == MemSize::B4 && ctx.jit_consts.lwr_keeps_upper { + emit_lwr_keeping_upper(ctx, loaded, byte_offset); + return; + } + let ity = size.ir_type(); let mem_val = if size == MemSize::B8 { loaded } else { ctx.builder.ins().ireduce(ity, loaded) }; let rt_val = emit_read_gpr(ctx, field_rt(ctx.raw)); @@ -10811,6 +10856,18 @@ mod tests { (op << 26) | (rs << 21) | (rt << 16) | (rd << 11) | (sa << 6) | funct } + #[test] + fn cache_fingerprint_includes_the_lwr_choice() { + let base = Codegen::new().cache_fingerprint(false).expect("default codegen is cacheable"); + let mut codegen = Codegen::new(); + codegen.jit_consts.lwr_keeps_upper = true; + assert_ne!( + codegen.cache_fingerprint(false).expect("lwr codegen is cacheable"), + base, + "the LWR choice must select a different persistent cache namespace", + ); + } + /// `is_inlinable` must reject on *both* of its conditions, not just the /// bounds one. The bounds half is what panicked /// (`clanker/jitv2/nested-delay-slot-across-page-boundary.md`); the diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 315eaa286..24a631f3f 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -4355,6 +4355,109 @@ mod tests { } } + /// Run `words` from `pc` once through the interpreter and once as a + /// compiled region, both on CPU model `C` over a fresh `bus()` and seeded + /// with `gpr` and CP0 Status `status`. The JIT learns the model's choices from `install_jit_hooks`, + /// as in production. + fn interpreter_and_jit(bus: impl Fn() -> Arc, pc: u64, gpr: [u64; 32], status: u32, words: &[u32]) + -> (CoreSnapshot, CoreSnapshot) + where + C: crate::cpu::mips_cache_v2::CpuModel + From>, + { + let build = || { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(bus(), PassthroughTlb::default(), &cfg)); + exec.core.gpr = gpr; + exec.core.pc = pc; + exec.set_cp0_status(status); + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + exec + }; + let mut interp = build(); + interp.jitv2_dispatch_enabled = false; + for (i, &word) in words.iter().enumerate() { + if interp.core.pc != pc + 4 * i as u64 { + break; // an exception redirected the PC + } + interp.exec(word); + } + + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[..words.len()].copy_from_slice(words); + let page_base = (pc & !(PAGE_SIZE as u64 - 1)) as u32; + let mut analyzer = Analyzer::new(); + let (walked, non_empty) = analyzer.walk_bounded(&page, 0, page_base, words.len()); + assert!(non_empty); + let mut instrs_owned = *walked; + let mut codegen = Codegen::new(); + codegen.jit_consts.lwr_keeps_upper = C::PARTIAL_LWR_KEEPS_UPPER; + let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false) + .expect("region must be compilable for this test to be meaningful"); + let mut jit = build(); + jit.install_jit_hooks(); + // The compile pool's workers read the model's choice from here, so a + // missing publication would not show if `codegen` were stamped by hand. + assert_eq!(jit.jitv2.lock().jit_consts.lock().lwr_keeps_upper, C::PARTIAL_LWR_KEEPS_UPPER); + unsafe { jit_fn(&mut jit.core as *mut MipsCore) }; + std::mem::forget(codegen); + (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core)) + } + + /// The R10000 keeps bits 63:32 on a partial LWR while 64-bit operations are + /// enabled for the current mode and the R5000 sign-extends; with them off + /// (a 32-bit user process, UX=0) the R10000 sign-extends too. The JIT and + /// the interpreter must agree at every offset in every mode, and the + /// interpreter's result is the one `partial_lwr_keeps_the_upper_word_only_on_the_r10000` + /// pins. + #[test] + fn jit_lwr_matches_the_interpreter_on_the_r10000_and_r5000() { + use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R5000ShadowCache}; + fn check>>(model_keeps_upper: bool) { + use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_KX, STATUS_SX, STATUS_UX}; + const RESERVED: u32 = 3 << 3; + const USER: u32 = 2 << 3; + const SUPER: u32 = 1 << 3; + // (Status, whether 64-bit operations are on): each mode's own + // bit decides, whatever the other two hold. + for (status, ops64, mode) in [ + (STATUS_KX, true, "kernel KX=1"), + (0, false, "kernel KX=0"), + (STATUS_UX | STATUS_SX, false, "kernel KX=0, UX/SX=1"), + (USER | STATUS_UX, true, "user UX=1"), + (USER | STATUS_KX | STATUS_SX, false, "user UX=0, KX/SX=1"), + (SUPER | STATUS_SX, true, "supervisor SX=1"), + (SUPER | STATUS_KX | STATUS_UX, false, "supervisor SX=0"), + (USER | STATUS_EXL | STATUS_KX, true, "EXL forces kernel, KX=1"), + (USER | STATUS_EXL | STATUS_UX, false, "EXL forces kernel, UX=1 KX=0"), + (SUPER | STATUS_ERL | STATUS_KX, true, "ERL forces kernel, KX=1"), + (SUPER | STATUS_ERL | STATUS_SX, false, "ERL forces kernel, SX=1 KX=0"), + (RESERVED | STATUS_KX, true, "reserved KSU counts as kernel, KX=1"), + (RESERVED | STATUS_UX | STATUS_SX, false, "reserved KSU counts as kernel, KX=0"), + ] { + for offset in 0..4u64 { + let mut gpr = [0u64; 32]; + gpr[1] = 0x2000 + offset; + gpr[2] = 0x1122_3344_5566_7788; + let bus = || -> Arc { + let mem = Arc::new(MockMemory::new_not_compilable()); + mem.set_word(0x2000, 0xDEAD_BEEF); + mem + }; + let lwr = make_i(crate::cpu::mips_isa::OP_LWR, 1, 2, 0); + let (interp, jit) = interpreter_and_jit::(bus, 0x1000, gpr, status, &[lwr]); + assert_eq!(jit, interp, "{} JIT/interpreter LWR offset {offset}, {mode}", C::NAME); + let upper_kept = interp.gpr[2] >> 32 == 0x1122_3344; + assert_eq!(upper_kept, model_keeps_upper && ops64 && offset != 3, + "{} LWR offset {offset}, {mode}", C::NAME); + } + } + } + check::(true); + check::(false); + } + #[test] fn lwl_lwr_combine_to_a_full_unaligned_word_load_matches_interpreter() { // The canonical compiler-generated idiom for an unaligned word @@ -7879,4 +7982,74 @@ mod tests { println!("{:<12} {:8.3} ns/instruction", name, ns); } } + + /// R10000: a refused uncached store completes in compiled code exactly as + /// in the interpreter, through both store sites (SW, and SWL by the + /// masked-store helper), and the instructions after it run. The same access + /// as a load, and a store on the R5000, still fault in both engines. + #[test] + fn refused_uncached_store_matches_the_interpreter_on_the_r10000_and_r5000() { + use crate::cpu::mips_cache_v2::{R10000Cache, R5000Cache, CpuModel}; + use crate::cpu::mips_core::STATUS_EXL; + use crate::cpu::mips_isa::{OP_ADDIU, OP_LW, OP_SW, OP_SWL}; + + /// Nothing answers: every read and write is refused (the trait's defaults). + struct RefusingBus; + impl BusDevice for RefusingBus {} + + fn run>>(op: u32) -> (CoreSnapshot, CoreSnapshot) { + let pc = 0xFFFF_FFFF_8000_0000u64; + let mut gpr = [0u64; 32]; + gpr[16] = 0xFFFF_FFFF_A100_0000; // kseg1: uncached, and nothing answers + gpr[6] = 0x1234_5678; + let words = [make_i(op, 16, 6, 0), make_i(OP_ADDIU, 0, 7, 2)]; + let build = || { + let bus: Arc = Arc::new(RefusingBus); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(bus, PassthroughTlb::default(), &cfg)); + exec.core.gpr = gpr; + exec.core.pc = pc; + exec.core.hot.interrupts.store(0, std::sync::atomic::Ordering::Relaxed); + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + exec + }; + let mut interp = build(); + interp.jitv2_dispatch_enabled = false; + for (i, &word) in words.iter().enumerate() { + if interp.core.pc != pc + 4 * i as u64 { + break; // an exception redirected the PC + } + interp.exec(word); + } + + let mut page = [0u32; ENTRIES_PER_PAGE]; + page[..words.len()].copy_from_slice(&words); + let page_base = (pc & !(PAGE_SIZE as u64 - 1)) as u32; + let mut analyzer = Analyzer::new(); + let (walked, non_empty) = analyzer.walk_bounded(&page, 0, page_base, words.len()); + assert!(non_empty); + let mut instrs_owned = *walked; + let mut codegen = Codegen::new(); + let jit_fn: JitFn = codegen.compile_region(&mut instrs_owned, 0, true, false) + .expect("region must be compilable for this test to be meaningful"); + let mut jit = build(); + jit.install_jit_hooks(); + unsafe { jit_fn(&mut jit.core as *mut MipsCore) }; + std::mem::forget(codegen); + (CoreSnapshot::capture(&interp.core), CoreSnapshot::capture(&jit.core)) + } + + for op in [OP_SW, OP_SWL] { + let (interp, jit) = run::(op); + assert_eq!(jit, interp, "op {op:#x}: JIT and interpreter diverged"); + assert_eq!(jit.cp0_status & STATUS_EXL, 0, "op {op:#x}: no exception"); + assert_eq!(jit.gpr[7], 2, "op {op:#x}: the instruction after the store ran"); + } + for (name, (interp, jit)) in [("R10000 load", run::(OP_LW)), ("R5000 store", run::(OP_SW))] { + assert_eq!(jit, interp, "{name}: JIT and interpreter diverged"); + assert_ne!(jit.cp0_status & STATUS_EXL, 0, "{name}: must take its Data Bus Error"); + assert_eq!(jit.gpr[7], 0, "{name}: nothing after the fault ran"); + } + } } diff --git a/src/cpu/jitv2/mod.rs b/src/cpu/jitv2/mod.rs index 60e01f749..6aeeb5b6e 100644 --- a/src/cpu/jitv2/mod.rs +++ b/src/cpu/jitv2/mod.rs @@ -92,6 +92,7 @@ mod zz_corpus { // (baked immediate vs load) is identical to production. crate::cpu::jitv2::codegen::JitConsts { core: core::num::NonZeroUsize::new(core as *mut MipsCore as usize), + ..Default::default() } }; // Synthetic corpus-measurement geometry: 16 KiB direct-mapped L1-D diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 53b6673eb..e58d37a29 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -23,7 +23,9 @@ //! to it and returns them unchanged, so a walking-1s test over tag or data //! SRAM passes without anyone having to know the hardware's field layout. //! Where the emulator *must* know the layout — to decide whether a line is -//! valid, say — that is a separate question from storing the bits. +//! valid, say — that is a separate question from storing the bits. The one +//! exception is the R10000's secondary tag, whose reserved bits read as zero +//! (`R10K_L2_TAG_MASK`). //! //! The shadow is deliberately not consulted by `read`/`write`/`fetch`. If it //! ever needs to be, this type is the wrong shape and should say so loudly @@ -90,15 +92,11 @@ const MRU_BIT: u64 = 1 << 63; /// R4400 and R5000 index differently — see `tag_slot`. const WAYS: usize = 2; -/// Bits a secondary-cache tag retains. -/// -/// All of them, now. A 36-bit mask was inferred here from the PROM's -/// walking-0s phase, which writes an all-ones TagLo and expects back -/// 0x0000000f_ffffcdfe — but that truncation comes from the tag being -/// assembled as `(TagHi << 32) | TagLo[31:0]`, not from the array dropping -/// bits. Once the executor carried TagHi properly the mask did nothing except -/// discard the MRU bit, which the PROM writes as TagHi[31] and reads back. -const L2_TAG_MASK: u64 = u64::MAX; +/// Bits an R10000 secondary-cache tag retains: MRU (bit 63), the physical tag +/// (TagHi 3:0 and TagLo 31:14), state, virtual index and ECC. TagLo bits 9, 12 +/// and 13 and TagHi 30:4 are reserved and read as zero (R10000 UM p. 270, +/// figure 11-31). Other parts keep every bit; see `CpuModel::SECONDARY_TAG_MASK`. +const R10K_L2_TAG_MASK: u64 = 0x8000_000f_ffff_cdff; /// Is the shadow cache's operation trace on, at this level of detail? /// @@ -204,11 +202,11 @@ impl< lladdr: UnsafeCell::new(0), ecc_in: UnsafeCell::new(0), ecc_out: UnsafeCell::new(0), - ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, 0)), - dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, 0)), - // Only the R10000 secondary keeps a data shadow: it is the one - // the IP28 PROM walks with Index_Store_Data, and a 1 MB array is - // cheap once. The R4000 family has no index data ops. + // The R10000 keeps data arrays for all three caches, for the PROM's + // Index_Load_Data / Index_Store_Data walks. The R4000 family has no + // index data ops. + ic: UnsafeCell::new(Shadow::new(Self::IC_LINES, if Self::R10K { IC_SIZE / 4 } else { 0 })), + dc: UnsafeCell::new(Shadow::new(Self::DC_LINES, if Self::R10K { DC_SIZE / 4 } else { 0 })), l2: UnsafeCell::new(Shadow::new(Self::L2_LINES, if Self::R10K { L2_SIZE / 8 } else { 0 })), tc_base: UnsafeCell::new(std::ptr::null_mut()), tc_bitmap: UnsafeCell::new(0), @@ -296,17 +294,54 @@ impl< /// Data slot for an R10000 `Index_Load_Data` / `Index_Store_Data`. /// - /// Same shape: way in bit 0, the rest addressing the array. The PROM walks - /// it at `…00`, `…01`, `…10`, `…11`, `…20` — one doubleword per operation - /// with the way bit shifted in beneath it. - fn data_slot(&self, len: usize, virt_addr: u64) -> usize { + /// Same shape: way in bit 0, the rest addressing the array, with four-byte + /// primary and eight-byte secondary slots. This is a model choice, enough + /// for a data walk to read back what it wrote. + fn data_slot(&self, sel: u32, len: usize, virt_addr: u64) -> usize { if len == 0 { return 0; } let way = (virt_addr as usize) & (WAYS - 1); - let word = (virt_addr as usize) >> 4; + let shift = if matches!(sel, CACH_PI | CACH_PD) { 2 } else { 3 }; + let word = (virt_addr as usize) >> shift; (word * WAYS + way) % len } + + /// R10000 `Index_Store_Data`, kept out of line: it runs only in a PROM's + /// diagnostic walk, and inlining it grew `cache_op` and moved the code + /// laid out after it. + #[cold] + #[inline(never)] + fn index_store_data(&self, sel: u32, virt_addr: u64, phys_addr: u64) { + let s = self.shadow(sel); + if s.data.is_empty() { + return; + } + let slot = self.data_slot(sel, s.data.len(), virt_addr); + // Diagnostic field widths: R10000 UM pp. 262, 271-272. + let (data_mask, ecc_mask) = if sel == CACH_PI { + (0xf_ffff_ffff, 1) + } else if sel == CACH_PD { + (0xffff_ffff, 0xf) + } else { + (u64::MAX, 0x3ff) + }; + let ecc = unsafe { *self.ecc_in.get() } & ecc_mask; + s.data[slot] = phys_addr & data_mask; + s.ecc[slot] = ecc; + if matches!(sel, CACH_SI | CACH_SD) { + // Index Store Data (S) writes a quadword: the selected + // doubleword plus a zero partner, sharing the check bits + // (R10000 UM p. 91, figure 5-7). + let partner = slot ^ WAYS; + s.data[partner] = 0; + s.ecc[partner] = ecc; + } + if l2_op_log(CACHE_LOG_HIT) { + crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", + s.ecc[slot]); + } + } } impl< @@ -347,7 +382,11 @@ impl< _ => "shadow", }; const CACHE_KIND: &'static str = "shadow"; + const R10K_FAMILY: bool = Self::R10K_CACHE_OPS; + const PARTIAL_LWR_KEEPS_UPPER: bool = MODEL == model::R10000; const R10K_CACHE_OPS: bool = MODEL == model::R10000; + const POSTS_UNCACHED_STORES: bool = Self::R10K_CACHE_OPS; + const SECONDARY_TAG_MASK: u64 = if Self::R10K_CACHE_OPS { R10K_L2_TAG_MASK } else { u64::MAX }; } impl< @@ -562,12 +601,16 @@ impl< } match op { - // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim: - // a tag test is a round trip, and round trips do not require - // knowing what the bits mean. + // Index_Store_Tag / Index_Load_Tag. Stored and returned verbatim + // unless the part sets a secondary mask: a tag test is a round trip, + // and round trips do not require knowing what the bits mean. C_IST => { let idx = self.tag_slot(sel, virt_addr); - let mask = if matches!(sel, CACH_SI | CACH_SD) { L2_TAG_MASK } else { u64::MAX }; + let mask = if matches!(sel, CACH_SI | CACH_SD) { + ::SECONDARY_TAG_MASK + } else { + u64::MAX + }; if l2_op_log(CACHE_LOG_OP) { crate::dlog!(LogModule::L2c, "shadow: -> IST va={virt_addr:#012x} idx={idx} stores {:#018x}", phys_addr & mask & !MRU_BIT); @@ -607,7 +650,7 @@ impl< C_R10K_CBARRIER if Self::R10K && sel == CACH_PI => 0, C_R10K_ILD if Self::R10K && matches!(sel, CACH_PI | CACH_PD | CACH_SD) => { let s = self.shadow(sel); - let slot = self.data_slot(s.data.len(), virt_addr); + let slot = self.data_slot(sel, s.data.len(), virt_addr); if s.data.is_empty() { 0 } else { @@ -619,17 +662,8 @@ impl< s.data[slot] } } - C_R10K_ISD if Self::R10K && matches!(sel, CACH_SI | CACH_SD) => { - let s = self.shadow(sel); - let slot = self.data_slot(s.data.len(), virt_addr); - if !s.data.is_empty() { - s.data[slot] = phys_addr; - s.ecc[slot] = unsafe { *self.ecc_in.get() }; - if l2_op_log(CACHE_LOG_HIT) { - crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", - s.ecc[slot]); - } - } + C_R10K_ISD if Self::R10K => { + self.index_store_data(sel, virt_addr, phys_addr); 0 } @@ -767,33 +801,40 @@ mod tests { } /// A tag test is a round trip, so the shadow must return exactly the bits - /// it was given — including any the real layout would not use. The cache + /// it was given, except the reserved ones the manual reads as zero. The cache /// model that preceded this one decoded TagLo into ptag/state/pidx fields /// and re-encoded on the way out, which silently dropped the low seven /// bits and every state code it did not recognise. A walking-1s test then /// failed on its very first bit. #[test] - fn tags_round_trip_every_bit() { + fn tags_round_trip_every_defined_bit() { let c = cache(); for bit in 0..64 { // 32 and 63 are MRU control/state, not tag storage. if bit == 32 || bit == 63 { continue; } let v = 1u64 << bit; c.cache_op(C_IST | CACH_SD, 0, v); + let kept = ::SECONDARY_TAG_MASK & v; assert_eq!( c.cache_op(C_ILT | CACH_SD, 0, 0), - v, + kept, "bit {bit} did not survive a store/load tag round trip" ); } } - /// Index_Store_Data / Index_Load_Data likewise. + /// Index_Store_Data / Index_Load_Data likewise, in the secondary and the + /// primary caches: a PROM's data walk round-trips. #[test] fn secondary_data_round_trips() { let c = cache(); c.cache_op(C_R10K_ISD | CACH_SD, 0x40, 0xdeadbeef); assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 0x40, 0), 0xdeadbeef); + + c.cache_op(C_R10K_ISD | CACH_PI, 4, 0x1234); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, 4, 0), 0x1234); + c.cache_op(C_R10K_ISD | CACH_PD, 8, 0x5678); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PD, 8, 0), 0x5678); } /// The MRU bit is **shared between the ways of a set**: written through @@ -861,23 +902,39 @@ mod tests { assert_eq!(c.cache_op_ecc(), 0x3fe, "way 1's check bits"); } + /// The NEC layout: ten check bits, and a secondary store writes a + /// quadword whose other doubleword is zero and shares the check bits. + #[test] + fn nec_layout_stores_a_quadword_with_ten_check_bits() { + let c = cache(); + c.set_cache_ecc(0); + c.cache_op(C_R10K_ISD | CACH_SD, 8, 0xdead_beef); + c.set_cache_ecc(0xffff_fc01); + c.cache_op(C_R10K_ISD | CACH_SD, 0, 0x1111_2222_3333_4444); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 0, 0), 0x1111_2222_3333_4444); + assert_eq!(c.cache_op_ecc(), 0x001, "ten check bits"); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_SD, 8, 0), 0, "zero partner"); + assert_eq!(c.cache_op_ecc(), 0x001, "shared check bits"); + } + /// An untouched set reports no MRU, so an ordinary tag read is not /// contaminated by it. #[test] fn an_untouched_set_has_no_mru_bit() { let c = cache(); - c.cache_op(C_IST | CACH_SD, 0, 0xdead_beef); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0xdead_beef); + // Only defined tag bits (TagLo bits 9, 12 and 13 read as zero). + c.cache_op(C_IST | CACH_SD, 0, 0xdead_8cef); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0xdead_8cef); } /// Separate lines must not alias onto one another. #[test] fn distinct_lines_hold_distinct_tags() { let c = cache(); - c.cache_op(C_IST | CACH_SD, 0, 0x1111_1111); - c.cache_op(C_IST | CACH_SD, 128, 0x2222_2222); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0x1111_1111); - assert_eq!(c.cache_op(C_ILT | CACH_SD, 128, 0), 0x2222_2222); + c.cache_op(C_IST | CACH_SD, 0, 0x1111_0111); + c.cache_op(C_IST | CACH_SD, 128, 0x2222_0022); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0, 0), 0x1111_0111); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 128, 0), 0x2222_0022); } /// Bit 0 of a CACHE index selects the **way**, and the two ways of a set @@ -905,14 +962,14 @@ mod tests { for set in 0..4u64 { for way in 0..2u64 { let va = set * 128 + way; - c.cache_op(C_IST | CACH_SD, va, 0x1000 + set * 16 + way); + c.cache_op(C_IST | CACH_SD, va, 0x4000 + set * 16 + way); } } for set in 0..4u64 { for way in 0..2u64 { let va = set * 128 + way; assert_eq!(c.cache_op(C_ILT | CACH_SD, va, 0), - 0x1000 + set * 16 + way, + 0x4000 + set * 16 + way, "set {set} way {way} aliased"); } } @@ -960,6 +1017,21 @@ mod tests { assert_eq!(::NAME, "shadow", "existing IP28 snapshots"); } + /// The R10000 returns zero for the reserved bits of a secondary tag and + /// keeps MRU, tag, state, index and ECC (R10000 UM p. 270); the R4400 + /// stores the tag as written. + #[test] + fn the_r10000_masks_secondary_tags() { + fn stored>>(tag: u64) -> u64 { + let c = C::from(Arc::new(Memory::new(1)) as Arc); + c.cache_op(C_IST | CACH_SD, 0, tag); + c.cache_op(C_ILT | CACH_SD, 0, 0) + } + assert_eq!(stored::(u64::MAX), 0x8000_000f_ffff_cdff); + assert_eq!(stored::(u64::MAX), 0x8000_000f_ffff_cdff); + assert_eq!(stored::(0x3000) & 0x3000, 0x3000, "R4400 keeps TagLo bits 12 and 13"); + } + /// The O2 R10000 differs from the IP28's only in its secondary line size. #[test] fn the_o2_r10000_is_the_ip28_part_with_shorter_secondary_lines() { diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 3dffd558c..c1f7db120 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -565,6 +565,27 @@ pub trait CpuModel: MipsCache { /// and `Index_Load_Data` returns into it, neither of which is true of the /// R4000 hit operations that share those encodings. const R10K_CACHE_OPS: bool = false; + /// The part is in the R10000 family, which shares one programming model: + /// FrameMask, and 44 virtual and 40 physical address bits. + /// Established for the R10000. The R12000 is inferred to match: its manual's + /// list of changes from the R10000 has no TLB or CP0 change. The R14000 and + /// R16000 are a model choice that gives them the same TLB. + const R10K_FAMILY: bool = false; + /// A partial LWR keeps bits 63:32 of the destination. Observed on the + /// R10000; whether the later parts do the same is not established, so they + /// keep the sign extension for now. + const PARTIAL_LWR_KEEPS_UPPER: bool = false; + /// An uncached store the system refuses completes instead of raising a + /// Data Bus Error. The R10000 sends it as a write request, which gets no + /// response, and Bus Error comes only from an error response to a read or + /// upgrade request or from uncorrectable read data (R10000 UM pp. 104, 316). + /// The R12000 manual documents the same path and the later parts are taken + /// to match. An uncached SC succeeding is a model choice. Loads and fetches + /// still fault. + const POSTS_UNCACHED_STORES: bool = false; + /// Bits `Index_Store_Tag` keeps for the secondary cache: all of them, except + /// on the R10000 family, whose reserved tag bits read as zero (R10000 UM p. 270). + const SECONDARY_TAG_MASK: u64 = u64::MAX; } pub trait MipsCache: Send + Sync { @@ -3595,7 +3616,10 @@ impl self.cp0_entrylo0 & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits - 3 => self.cp0_entrylo1 & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits + 2 => self.cp0_entrylo0 & self.entrylo_mask, + 3 => self.cp0_entrylo1 & self.entrylo_mask, 4 => self.cp0_context, 5 => self.cp0_pagemask & 0x01FFE000, // PageMask: only bits 24:13 are valid 6 => self.cp0_wired as u64 & 0x3F, // Wired: only bits 5:0 are valid, R4000 manual Table 4-10 @@ -1636,6 +1661,7 @@ impl MipsCore { 18 => self.cp0_watchlo as u64, 19 => self.cp0_watchhi as u64, 20 => self.cp0_xcontext, + 21 => self.cp0_framemask as u64, 25 => { // PerfCnt: R4400 has no performance counters. Log any probe so // a guest expecting them (and routing their overflow onto the @@ -1670,8 +1696,8 @@ impl MipsCore { match reg { 0 => self.cp0_index as u64, 1 => self.cp0_random as u64, - 2 => self.cp0_entrylo0 & 0x3FFFFFFF, - 3 => self.cp0_entrylo1 & 0x3FFFFFFF, + 2 => self.cp0_entrylo0 & self.entrylo_mask, + 3 => self.cp0_entrylo1 & self.entrylo_mask, 4 => self.cp0_context, 5 => self.cp0_pagemask & 0x01FFE000, 6 => self.cp0_wired as u64 & 0x3F, @@ -1688,6 +1714,7 @@ impl MipsCore { 18 => self.cp0_watchlo as u64, 19 => self.cp0_watchhi as u64, 20 => self.cp0_xcontext, + 21 => self.cp0_framemask as u64, 26 => self.cp0_ecc as u64, 27 => self.cp0_cacheerr as u64, 28 => self.cp0_taglo, @@ -1697,6 +1724,14 @@ impl MipsCore { } } + /// An EntryLo value as a TLB write passes it on: FrameMask bits 15:0 clear + /// EntryLo bits 33:18, so the flags IRIX keeps above the PFN's low bits + /// never reach the physical address (R10000 UM v2.0 p. 260). + #[inline] + pub fn framemask_entrylo(&self, lo: u64) -> u64 { + lo & !((self.cp0_framemask as u64) << 18) + } + /// Materialize the virtual CP0 Count: advance `cp0_count` by the whole /// hardware ticks elapsed since the anchor at `count_hz`, move the anchor /// forward by exactly the duration those ticks consumed (the sub-tick @@ -2011,8 +2046,8 @@ impl MipsCore { 0 => self.cp0_index = (value as u32) & (crate::cpu::mips_exec::CP0_INDEX_P | crate::cpu::mips_exec::CP0_INDEX_SLOT_MASK), 1 => { /* Random is read-only */ } - 2 => self.cp0_entrylo0 = value & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits - 3 => self.cp0_entrylo1 = value & 0x3FFFFFFF, // PFN is 24 bits (29:6), flags in lower bits + 2 => self.cp0_entrylo0 = value & self.entrylo_mask, + 3 => self.cp0_entrylo1 = value & self.entrylo_mask, 4 => { //eprintln!("MTC0 Context = {:#018x} (PTEBase={:#018x})", value, value & 0xFFFFFFFF_FF800000); self.cp0_context = value; @@ -2035,16 +2070,16 @@ impl MipsCore { } 10 => { // always use 64bit mask because the entries need to be valid in 64 bit mode even when they were set from 32 bit mode // `log mips mask cp0` shows what the guest wrote against - // what survives the mask. The mask is R4400's 40-bit virtual - // address; the R10000 implements 44. - if cp0_log() && (value & !0xC000_00FF_FFFF_E0FF) != 0 { + // what survives the mask: the model's VPN2 width, 40 bits or the + // R10000 family's 44. + if cp0_log() && (value & !self.entryhi_mask) != 0 { crate::dlog!(crate::devlog::LogModule::Mips, "ip28ehi: wrote {value:#018x} -> kept {:#018x} (lost {:#018x}) pc={:#018x}", - value & 0xC000_00FF_FFFF_E0FF, - value & !0xC000_00FF_FFFF_E0FF, + value & self.entryhi_mask, + value & !self.entryhi_mask, self.pc); } - self.cp0_entryhi = value & 0xC000_00FF_FFFF_E0FF; + self.cp0_entryhi = value & self.entryhi_mask; }, 11 => { // Materialize the count *first* so both new_delta below and @@ -2253,6 +2288,10 @@ impl MipsCore { } self.cp0_xcontext = value; } + // FrameMask: bits 15:0 are implemented, the rest read as zero + // (R10000 UM v2.0 p. 260). Other parts have no register 21 and ignore + // the write. + 21 => self.cp0_framemask = value as u32 & self.framemask_mask, 25 => { #[cfg(feature = "developer_ip7")] eprintln!("[ip7] MTC0 PerfCnt (reg 25) write {:#018x} (ignored)", value); diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 62aaee8de..ecfe79c38 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -577,17 +577,28 @@ pub const TR_GLOBAL: u32 = 1 << 4; /// the privilege epoch that filled it. The only thing rejected here is a VA /// shape that is not a valid address in the first place. #[inline] -fn nutlb_cacheable_segment(va: u64, is_64bit: bool) -> bool { +fn nutlb_cacheable_segment(va: u64, is_64bit: bool) -> bool { if !is_64bit { // Every 32-bit segment (kuseg/kseg0/kseg1/ksseg/kseg3) is a real // address whose finished translation is worth caching. return true; } - // xuseg: bits 63:40 must be zero to be a valid user address. Every other - // 64-bit segment (xsseg, xkphys, xkseg and the 0xFFFFFFFF_* compat - // windows) is fine — translate_64bit_impl raises before returning ok for - // the illegal xkphys sub-ranges, so nothing invalid reaches a fill. - (va >> 62) != 0 || (va >> 40) == 0 + // xuseg: bits 63:`C::VA_BITS` must be zero to be a valid user address. + // Every other 64-bit segment (xsseg, xkphys, xkseg and the 0xFFFFFFFF_* + // compat windows) is fine — translate_64bit_impl raises before returning ok + // for the illegal xkphys sub-ranges, so nothing invalid reaches a fill. + (va >> 62) != 0 || (va >> C::VA_BITS) == 0 +} + +/// The EntryHi bits this model keeps: region, VPN2 up to `C::VA_BITS` and ASID. +#[inline(always)] +const fn model_entryhi_mask() -> u64 { + 0xC000_0000_0000_00FF | crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS) +} + +/// Whether the `n`th refused store is reported: the first four, then each power of two. +pub(crate) fn report_refused_store(n: u64) -> bool { + n <= 4 || n.is_power_of_two() } /// Per-access-class tally of how far a guest load/store would get through the @@ -853,6 +864,9 @@ pub struct MipsExecutor { pub decoded_count: Arc, /// Count of instructions fetched from uncached address space. pub uncached_fetch_count: Arc, + /// Refused uncached stores so far (see `CpuModel::POSTS_UNCACHED_STORES`); + /// sets how often `post_refused_store` reports. + refused_stores: u64, /// Hot-path translation function pointer, updated whenever CP0 Status changes. /// Always the non-debug variant; selects the correct 32/64-bit × privilege specialisation. pub translate_fn: fn(&mut Self, u64, AccessType) -> TranslateResult, @@ -1023,12 +1037,12 @@ pub struct MipsExecutor { pub batch_hook: Option) + Send>>, /// A window of physical addresses that reaches memory at another address, /// `(base, size, target)`, applied to mapped and XKPHYS translations - /// before [`Self::phys_mask`]. The O2 has a no-ECC alias of its memory at + /// after [`Self::phys_mask`]. The O2 has a no-ECC alias of its memory at /// 0x8000_0000 that would otherwise reach the PCI window the bus decodes /// there. None on the other machines. pub phys_alias: Option<(u64, u64, u64)>, /// The physical address bits the board decodes, applied to mapped, - /// XKPHYS and ERL-identity translations after [`Self::phys_alias`]. The + /// XKPHYS and ERL-identity translations before [`Self::phys_alias`]. The /// CPU drives 36 or 40 bits, but the Indy, Indigo2 and O2 decode 32 and /// alias the rest away, and IRIX relies on it: its page-table entries keep /// software bits (the PFN lock among them) just above the PFN, so they @@ -2193,6 +2207,22 @@ impl MipsExecutor { self.sysad.describe_fault(phys_addr, size).map_or_else(String::new, |why| format!(" [{why}]")) } + /// An uncached store the system refused, on a model that posts stores + /// (`CpuModel::POSTS_UNCACHED_STORES`): say so and let the store complete. + /// The caller returns `BUS_OK`. Always built, so a missing device or a + /// guest bug stays visible in every build. A probe loop would flood + /// stderr, so it reports the first four and then every power of two, with + /// the running count. + #[cold] + fn post_refused_store(&mut self, virt_addr: u64, phys_addr: u64, size: usize, val: u64, status: u32) { + self.refused_stores += 1; + let n = self.refused_stores; + if report_refused_store(n) { + eprintln!("Unsupported uncached write{} (posted, completes): PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{} [store {n}]", + size * 8, self.core.pc, virt_addr, phys_addr, val, status, self.fault_note(phys_addr, size)); + } + } + /// Create a new executor from a config and a bus (sysad) and a TLB. /// The cache hierarchy is constructed internally as a unified R4000Cache. pub fn new(sysad: Arc, mut tlb: T, cfg: &MipsCpuConfig) -> Self @@ -2202,7 +2232,16 @@ impl MipsExecutor { let mut core = MipsCore::new(); // Mapped pages decode their C field with this model's table, as KSEG0 // and xkphys do. + tlb.set_va_bits(C::VA_BITS); tlb.set_cache_attrs(Self::cache_attr_table()); + tlb.set_conflict_invalidation(C::R10K_FAMILY); + core.entryhi_mask = model_entryhi_mask::(); + if C::R10K_FAMILY { + core.framemask_mask = 0xFFFF; + // PFN in bits 33:6 and the uncached attribute in 63:62 (R10000 UM v2.0 + // pp. 239-240); 61:34 are reserved. + core.entrylo_mask = 0xC000_0003_FFFF_FFFF; + } // Cache geometry comes from the model type C, so Config describes this CPU. // `mut` is only used by the Triton L2-enable sync below. @@ -2382,6 +2421,7 @@ impl MipsExecutor { ins: DecodedInstr::default(), // scratch slot for uncached fetches decoded_count: Arc::new(AtomicU64::new(0)), uncached_fetch_count: Arc::new(AtomicU64::new(0)), + refused_stores: 0, // Placeholder — overwritten immediately by update_translate_fn below. translate_fn: translate_32_kernel::, // Placeholder — overwritten immediately by update_fpr_mode below. @@ -2618,6 +2658,7 @@ impl MipsExecutor { // set) -- the property a persistent code cache needs. // Several CPUs share compiled code, so no one core's address. core: if self.core.smp || std::env::var_os("IRIS_JIT_PIC").is_some() { None } else { core::num::NonZeroUsize::new(core_addr) }, + lwr_keeps_upper: C::PARTIAL_LWR_KEEPS_UPPER, }; *j.jit_consts.lock() = consts; // The inline-compile path (`jitv2_compile_inline`) takes this @@ -2873,7 +2914,7 @@ impl MipsExecutor { // user address. Everything else is cacheable regardless of which // mode filled it, because privilege is enforced by flushing on // transition rather than by the tag. - if !nutlb_cacheable_segment(va, self.core.is_64bit_mode()) { + if !nutlb_cacheable_segment::(va, self.core.is_64bit_mode()) { return; } @@ -3220,6 +3261,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", pub fn update_translate_fn(&mut self) { use crate::cpu::mips_core::PrivilegeMode; let is_64bit = self.core.is_64bit_mode(); + self.core.lwr_upper_below = if C::PARTIAL_LWR_KEEPS_UPPER && is_64bit { 3 } else { 0 }; let privilege = self.core.get_privilege_mode(); self.translate_fn = match (is_64bit, privilege) { (false, PrivilegeMode::Kernel) => translate_32_kernel::, @@ -4518,11 +4560,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let top_bits = virt_addr >> 62; match top_bits { - // xuseg: 0x0000_0000_0000_0000 - 0x0000_00FF_FFFF_FFFF (user mapped) - // Accessible from all privilege levels + // xuseg is user-mapped below 2^C::VA_BITS (0x0000_00FF_FFFF_FFFF + // for the 40-bit models) and accessible from every privilege level. 0 => { - if (virt_addr >> 40) != 0 { - // Bits 63:40 must be zero for valid user address + if (virt_addr >> C::VA_BITS) != 0 { + // Bits above the model's implemented user VA must be zero if !DEBUG { self.core.cp0_badvaddr = virt_addr; } return TranslateResult::exc(exec_exception(if access_type == AccessType::Write { EXC_ADES } else { EXC_ADEL })); } @@ -4608,15 +4650,17 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } - /// A CPU physical address as the board decodes it: through - /// [`Self::phys_alias`] if it falls in it, then cut to [`Self::phys_mask`]. + /// A CPU physical address as the board decodes it: cut to + /// [`Self::phys_mask`], then through [`Self::phys_alias`] if it falls in it. #[inline(always)] fn fold_phys(&self, pa: u64) -> u64 { - let pa = match self.phys_alias { - Some((base, size, target)) if pa.wrapping_sub(base) < size => pa - base + target, + // Cut first: the PTE software bits above the PFN are not part of the + // address the alias window sees. Window and target lie within the mask. + let pa = pa & self.phys_mask; + match self.phys_alias { + Some((base, size, target)) if pa.wrapping_sub(base) < size => (pa - base + target) & self.phys_mask, _ => pa, - }; - pa & self.phys_mask + } } /// TLB translation. When `DEBUG` is true, CP0 side-effect registers are @@ -4649,7 +4693,8 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", TlbResult::Miss { .. } => { if !DEBUG { self.update_tlb_exception_registers::(virt_addr); } // Miss: XTLB vector (0x080) for 64-bit extended addresses, UTLB vector (0x000) otherwise - TranslateResult::exc(if XTLB != 0 { exec_xtlb_miss(tlb_miss_code) } else { exec_tlb_miss(tlb_miss_code) }) + let xrefill = if C::R10K_FAMILY { self.r10k_xtlb_refill::(virt_addr) } else { XTLB != 0 }; + TranslateResult::exc(if xrefill { exec_xtlb_miss(tlb_miss_code) } else { exec_tlb_miss(tlb_miss_code) }) } TlbResult::Invalid { .. } => { if !DEBUG { self.update_tlb_exception_registers::(virt_addr); } @@ -4663,6 +4708,26 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } } + /// Whether a refill miss on `virt_addr` takes the XTLB vector on the + /// R10000 family. The R10000 chooses by the space the address lies in, + /// not the mode it was referenced from: a user address follows UX, xsseg + /// SX, and kseg3, sseg/ksseg and xkseg follow KX (R10000 UM v2.0 pp. + /// 333-334, 342). The R4400 chooses by the current mode, which is what + /// `XTLB` encodes. So a 64-bit kernel (KX=1, UX=0) touching kuseg takes + /// the 32-bit refill vector here. In 32-bit mode only the low word of the + /// address counts, so it is sign-extended before the region is read. + #[cold] + fn r10k_xtlb_refill(&self, virt_addr: u64) -> bool { + use crate::cpu::mips_core::{STATUS_KX, STATUS_SX, STATUS_UX}; + let va = if XTLB != 0 { virt_addr } else { virt_addr as u32 as i32 as i64 as u64 }; + let x = match va >> 62 { + 0 => STATUS_UX, + 1 => STATUS_SX, + _ => STATUS_KX, + }; + self.core.cp0_status & x != 0 + } + /// Debug helper to translate address without side effects pub fn debug_translate(&mut self, virt_addr: u64) -> TranslateResult { self.translate_impl::(virt_addr, AccessType::Debug) @@ -4673,8 +4738,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[inline] fn update_tlb_exception_registers(&mut self, virt_addr: u64) { const EH_VPN2_32: u64 = 0x0000_0000_FFFF_E000; - const EH_VPN2_64: u64 = 0x0000_00FF_FFFF_E000; const EH_REGION: u64 = 0xC000_0000_0000_0000; + // VPN2 is bits 43:13 on the R10000 family (R10000 UM pp. 219, 288) and + // 39:13 on the R4x00 and R5000. The R12000 is inferred to match (its + // manual lists no TLB change); the R14000 and R16000 are a model choice + // that follows the same width. + let eh_vpn2_64 = crate::cpu::mips_tlb::vpn2_mask(C::VA_BITS); self.core.cp0_badvaddr = virt_addr; @@ -4697,9 +4766,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // // This is MAME r4000's `EH_WM` (0xC000_00FF_FFFF_E0FF) less the ASID // bits, which are carried separately below. - const EH_VPN_MASK: u64 = EH_REGION | EH_VPN2_64; // 0xC000_00FF_FFFF_E000 + let eh_vpn_mask = EH_REGION | eh_vpn2_64; let asid = self.core.cp0_entryhi & 0xFF; - self.core.cp0_entryhi = (virt_addr & EH_VPN_MASK) | asid; + self.core.cp0_entryhi = (virt_addr & eh_vpn_mask) | asid; // Context: PTEBase[63:23] preserved, BadVPN2 = virt_addr[31:13] in bits [22:4]. // Always 32-bit VPN — Context is used by the 32-bit UTLB handler. @@ -4720,10 +4789,6 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // the address it computes lands outside the page table it wired, the // nested miss is unrecoverable, and IRIX panics with "Entered tlbmiss // with invalid vaddr". - // - // EntryHi keeps the 40-bit VPN2 field on every model: a 64-bit kernel - // zero-fills those upper bits itself when it builds an EntryHi value, - // and widening the mask here breaks it. self.core.cp0_xcontext = xcontext_on_miss(C::VA_BITS, self.core.cp0_xcontext, virt_addr); } @@ -5078,9 +5143,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.sysad.write64(phys_addr, val) }; if ws != BUS_OK && ws != BUS_BUSY && !DEBUG { - eprintln!("Bus error on uncached write{}: PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", SIZE*8, self.core.pc, virt_addr, phys_addr, val, ws, self.fault_note(phys_addr, SIZE)); + if C::POSTS_UNCACHED_STORES { + self.post_refused_store(virt_addr, phys_addr, SIZE, val, ws); + BUS_OK + } else { + eprintln!("Bus error on uncached write{}: PC={:016x} VA={:016x} PA={:016x} val={:016x} status={:08x}{}", SIZE*8, self.core.pc, virt_addr, phys_addr, val, ws, self.fault_note(phys_addr, SIZE)); + ws + } + } else { + ws } - ws }; // jitv2_lockstep's load/store verification (see MipsCore's // lockstep_mem field doc comment, and read_data_impl's matching @@ -5134,9 +5206,16 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } let ws = self.sysad.write64_masked(phys_addr, val, mask); if ws != BUS_OK && ws != BUS_BUSY && !DEBUG { - eprintln!("Bus error on uncached write64_masked: PC={:016x} VA={:016x} PA={:016x} val={:016x} mask={:016x} status={:08x}{}", self.core.pc, virt_addr, phys_addr, val, mask, ws, self.fault_note(phys_addr, 8)); + if C::POSTS_UNCACHED_STORES { + self.post_refused_store(virt_addr, phys_addr, mask.count_ones() as usize / 8, val, ws); + BUS_OK + } else { + eprintln!("Bus error on uncached write64_masked: PC={:016x} VA={:016x} PA={:016x} val={:016x} mask={:016x} status={:08x}{}", self.core.pc, virt_addr, phys_addr, val, mask, ws, self.fault_note(phys_addr, 8)); + ws + } + } else { + ws } - ws } } @@ -6354,8 +6433,20 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let mask = 0xFFFFFFFFu32 >> shift; let result = (mem_word >> shift) | (rt_val & !mask); - // Sign-extend to 64 bits - self.core.write_gpr(rt_reg, result as i32 as i64 as u64); + // The manual leaves the upper word of a partial LWR implementation + // specific (NEC U12754E pp. 125-126). The R10000 keeps + // bits 63:32, as observed reference results show; the later + // parts are not established. A full word + // still sign-extends, and so does any word while 64-bit + // operations are off for the current mode (UX/SX/KX clear): + // there a register must hold a sign-extended 32-bit value. + let value = if C::PARTIAL_LWR_KEEPS_UPPER && byte_offset != 3 && self.core.is_64bit_mode() { + (self.core.read_gpr(rt_reg) & 0xffff_ffff_0000_0000) | result as u64 + } else { + // Sign-extend to 64 bits + result as i32 as i64 as u64 + }; + self.core.write_gpr(rt_reg, value); self.handle_exec_complete() } Err(status) => self.finish_status(status) @@ -6541,6 +6632,12 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // R4000 hit operations — translating them would fault on an index that // is not a valid virtual address. See C_R10K_ISD in mips_cache_v2. let sel = cache_op & 3; + // Bit `cache_op` of this mask: the primary-cache index data operations. + const R10K_PRIMARY_DATA_OPS: u32 = (1 << (C_R10K_ILD | CACH_PI)) + | (1 << (C_R10K_ILD | CACH_PD)) + | (1 << (C_R10K_ISD | CACH_PI)) + | (1 << (C_R10K_ISD | CACH_PD)); + let primary_data_op = C::R10K_CACHE_OPS && (1u32 << cache_op) & R10K_PRIMARY_DATA_OPS != 0; let r10k_index_op = C::R10K_CACHE_OPS && match op { C_R10K_CBARRIER => sel == CACH_PI, @@ -6548,8 +6645,11 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", C_R10K_ISD => matches!(sel, CACH_SI | CACH_SD), _ => false, }; - let needs_translation = - !r10k_index_op && matches!(op, C_CDX | C_HINV | C_HWBINV | C_HWB | C_HSV); + // A primary Index Load/Store Data selects by VA, but a mapped one still + // takes TLB Refill and Invalid (VR10000 Instruction Set UM p. 15). The other index + // operations keep not translating their operand. + let needs_translation = primary_data_op + || (!r10k_index_op && matches!(op, C_CDX | C_HINV | C_HWBINV | C_HWB | C_HSV)); let phys_addr = if needs_translation { // Hit operations need address translation @@ -6569,7 +6669,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // that reads only TagLo sees a different tag from the one written. // Harmless for R4400/R5000, whose tags fit in 32 bits and who leave // TagHi zero: the shift-in contributes nothing there. - let stores_taglo = op == C_IST || (r10k_index_op && op == C_R10K_ISD); + let stores_taglo = op == C_IST || ((r10k_index_op || primary_data_op) && op == C_R10K_ISD); let phys_addr_or_taglo = if stores_taglo { ((self.core.cp0_taghi as u64) << 32) | (self.core.cp0_taglo as u32 as u64) } else { @@ -6642,6 +6742,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // Split back the way it arrived. Zeroing TagHi unconditionally // threw away the top of every tag wider than 32 bits. self.core.cp0_taglo = result & 0xFFFF_FFFF; + // A primary data cache access leaves TagHi zero (a model choice). self.core.cp0_taghi = (result >> 32) as u32; } @@ -7129,13 +7230,23 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let g1 = (self.core.cp0_entrylo1 & 1) != 0; let g_combined = if g0 && g1 { 1u64 << 12 } else { 0 }; - // EH_WM = 0xC000_00FF_FFFF_E0FF — same mask MAME applies on TLBWI (clears reserved bits). - // Then clear bit 12 (G bit from EntryHi) and set combined G from Lo0&Lo1. - const EH_WM: u64 = 0xC000_00FF_FFFF_E0FF; + // EntryHi's writable mask (0xC000_00FF_FFFF_E0FF on a 40-bit model, MAME's + // `EH_WM`) clears reserved bits, as on TLBWI. Then clear bit 12 (G bit + // from EntryHi) and set combined G from Lo0&Lo1. + // + // The R10000 family stores zeros in the PFN bits the page size leaves + // out of the address, so TLBR does not return them (R10000 UM v2.0 + // pp. 299-300, TLBWI and TLBWR). PageMask bit n covers physical bit + // n-1, EntryLo bit n-7: 0x6000 (16 KB) clears EntryLo 7:6. The R4400 + // keeps the whole PFN. The manual documents this for the R10000 only; + // for the R12000, R14000 and R16000 it is a model choice, as no manual + // describes their TLB write. + let unused_pfn = if C::R10K_FAMILY { (self.core.cp0_pagemask & 0x01FF_E000) >> 7 } else { 0 }; + let lo = |v: u64| self.core.framemask_entrylo(v) & !unused_pfn; TlbEntry { page_mask: self.core.cp0_pagemask, - entry_hi: (self.core.cp0_entryhi & EH_WM & !0x1000) | g_combined, - entry_lo: [self.core.cp0_entrylo0, self.core.cp0_entrylo1], + entry_hi: (self.core.cp0_entryhi & self.core.entryhi_mask & !0x1000) | g_combined, + entry_lo: [lo(self.core.cp0_entrylo0), lo(self.core.cp0_entrylo1)], selector_bit_shift: 0, // all derived fields overwritten by MipsTlb::write() vcmp32: 0, vpn_hi32: 0, vcmp64: 0, vpn_hi64: 0, offset_mask: 0, pfn_base: [0; 2], @@ -7196,6 +7307,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let entry = self.create_tlb_entry_from_cp0(); self.ip28_trace_tlb_write("tlbwi", index, &entry); self.tlb.write(index, entry); + if self.tlb.take_write_conflict() { + self.core.cp0_status |= crate::cpu::mips_core::STATUS_TS; + } // Flushes the nutlb too — the TLB it caches just changed. self.nanotlb_invalidate(); @@ -7220,6 +7334,9 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let entry = self.create_tlb_entry_from_cp0(); self.ip28_trace_tlb_write("tlbwr", index, &entry); self.tlb.write(index, entry); + if self.tlb.take_write_conflict() { + self.core.cp0_status |= crate::cpu::mips_core::STATUS_TS; + } self.nanotlb_invalidate(); if mips_log(MIPS_LOG_TLB) { dlog_dev!(LogModule::Mips, "TLBWR: Write Random Index {}\n{}", index, self.tlb.format_entry(index)); } @@ -11020,7 +11137,7 @@ impl Saveable for MipsCpu cp0u64!(cp0_taglo); cp0u32!(cp0_taghi); cp0u64!(cp0_badvaddr); cp0u64!(cp0_epc); cp0u64!(cp0_errorepc); cp0u64!(cp0_entrylo0); cp0u64!(cp0_entrylo1); cp0u64!(cp0_context); - cp0u64!(cp0_pagemask); cp0u64!(cp0_entryhi); cp0u64!(cp0_xcontext); + cp0u64!(cp0_pagemask); cp0u64!(cp0_entryhi); cp0u64!(cp0_xcontext); cp0u32!(cp0_framemask); tbl.insert("cp0".into(), toml::Value::Table(cp0)); // FPU @@ -11085,7 +11202,7 @@ impl Saveable for MipsCpu ld32!(cp0_ecc); ld32!(cp0_cacheerr); ld64!(cp0_taglo); ld32!(cp0_taghi); ld64!(cp0_entrylo0); ld64!(cp0_entrylo1); ld64!(cp0_context); ld64!(cp0_pagemask); ld64!(cp0_badvaddr); ld64!(cp0_entryhi); - ld64!(cp0_xcontext); ld64!(cp0_epc); ld64!(cp0_errorepc); + ld64!(cp0_xcontext); ld32!(cp0_framemask); ld64!(cp0_epc); ld64!(cp0_errorepc); // Restart the virtual count from the restored value and re-arm // the compare timer — Instants from the previous run are // meaningless here — with a live ticket for the restored deadline. diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 4098813ec..9cf022b64 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -1227,6 +1227,64 @@ mod tests { 0xAAAAAAAA means a stale nanotlb slot survived the switch"); } + /// Two writes that would match together: on the R10000 family the second + /// removes the first and sets Status.TS, for TLBWI and TLBWR alike; on the + /// R4400 and R5000 both entries stay and TS stays clear. The compiled-code + /// path has nothing to add: jitv2 never admits TLBWI or TLBWR into a + /// region (see the `tlb_ops_are_never_admitted_but_eret_is` test), so + /// they always run this interpreter code. + fn overlapping_tlb_writes(r10k: bool) + where C: CpuModel + From> { + use crate::cpu::mips_tlb::{MipsTlb, Tlb}; + use crate::cpu::mips_core::STATUS_TS; + let mem: Arc = Arc::new(MockMemory::new()); + let mut exec = MipsExecutor::::new(mem, MipsTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + let tlbwi = (OP_COP0 << 26) | (0x10 << 21) | 0x02; + let tlbwr = (OP_COP0 << 26) | (0x10 << 21) | 0x06; + let write = |exec: &mut MipsExecutor, instr: u32, index: u32, pfn: u64| { + exec.core.cp0_index = index; + exec.core.cp0_pagemask = 0; + exec.core.cp0_entryhi = (0x100 << 13) | 10; + exec.core.cp0_entrylo0 = (pfn << 6) | (3 << 3) | (1 << 2) | (1 << 1); + exec.core.cp0_entrylo1 = ((pfn + 1) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + assert_eq!(exec.exec(instr), EXEC_COMPLETE); + }; + let first_survives = |exec: &MipsExecutor| exec.tlb.read(5).entry_hi == (0x100 << 13) | 10; + + write(&mut exec, tlbwi, 5, 0x50); + assert_eq!(exec.core.cp0_status & STATUS_TS, 0, "a write that overlaps nothing leaves TS clear"); + + write(&mut exec, tlbwi, 6, 0x60); + assert_eq!(exec.core.cp0_status & STATUS_TS != 0, r10k, "TLBWI TS"); + assert_eq!(!first_survives(&exec), r10k, "TLBWI removes the overlapped entry"); + if r10k { + assert_eq!(exec.tlb.read(5).page_mask, 0, "PageMask kept"); + } + + // Software clears TS by writing Status; the next overlap sets it again. + exec.core.cp0_status &= !STATUS_TS; + write(&mut exec, tlbwi, 5, 0x50); + exec.core.cp0_status &= !STATUS_TS; + exec.core.cp0_wired = 7; // Random stays out of 0..7, so it cannot pick entries 5 or 6 + write(&mut exec, tlbwr, 0, 0x70); + assert_eq!(exec.core.cp0_status & STATUS_TS != 0, r10k, "TLBWR TS"); + assert_eq!(!first_survives(&exec), r10k, "TLBWR removes the overlapped entry"); + } + + #[test] + fn overlapping_tlb_writes_follow_the_cpu_family() { + std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(|| { + overlapping_tlb_writes::(true); + overlapping_tlb_writes::(true); + overlapping_tlb_writes::(false); + overlapping_tlb_writes::(false); + }) + .unwrap().join().unwrap(); + } + #[cfg(feature = "tlbcheck")] #[test] fn test_tlbwi_breaks_on_duplicate_entry() { @@ -3744,6 +3802,150 @@ mod tests { check::(0xF_FFFF_FFFF); } + /// FrameMask bits 15:0 clear EntryLo bits 33:18 as a TLB write takes them, + /// on the R10000 only (R10000 UM v2.0 p. 260). Physical bit 24 is EntryLo bit 18, + /// mask bit 0; the R4400 has no such register. + #[test] + fn framemask_clears_entrylo_bits_on_a_tlb_write() { + fn phys>>(framemask: u64) -> (u64, u64) { + const VA: u64 = 0x4000_0000; + let mut exec = map_page_masked::(Arc::new(MockMemory::new()), VA, 0x0300_5000, framemask); + (exec.core.read_cp0(21), exec.debug_translate(VA).phys) + } + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + on_big_stack(|| { + // Mask bit 0 is written with bits above 15, which are ignored. + let all = 0xFFFF_0001; + assert_eq!(phys::(0), (0, 0x0300_5000)); + assert_eq!(phys::(all), (1, 0x0200_5000)); + assert_eq!(phys::(all), (0, 0x0300_5000)); + }); + } + + /// TLBWR masks as TLBWI does, and TLBR returns what the TLB holds: the + /// masked EntryLo, not the value that was written. + #[test] + fn framemask_applies_to_tlbwr_and_tlbr_reads_back_the_masked_entry() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + // Write with `funct`, find the entry with TLBP, clear EntryLo0, TLBR, + // and return (physical address of the page, EntryLo0 after TLBR). + fn entry>>(funct: u32) -> (u64, u64) { + const VA: u64 = 0x4000_0000; + let mut exec = map_page_with::(Arc::new(MockMemory::new()), VA, 0x0300_5000, 0xFFFF_0001, funct); + let phys = exec.debug_translate(VA).phys; + for (rd, value) in [(10, VA), (2, 0)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBP), EXEC_COMPLETE); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBR), EXEC_COMPLETE); + (phys, exec.core.read_cp0(2)) + } + on_big_stack(|| { + // FrameMask bit 0 clears EntryLo bit 18, physical bit 24. + let unmasked = ((0x0300_5000u64 >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + let masked = unmasked & !(1 << 18); + for funct in [FUNCT_TLBWI, FUNCT_TLBWR] { + assert_eq!(entry::(funct), (0x0200_5000, masked), "R10000 {funct:#x}"); + assert_eq!(entry::(funct), (0x0300_5000, unmasked), "R4400 {funct:#x}"); + } + }); + } + + /// The R10000 writes zeros into the PFN bits a page size leaves out of the + /// address, so TLBR returns them as zero (R10000 UM v2.0 pp. 299-300, TLBWI + /// and TLBWR); the later parts do the same as a model choice, since no + /// manual describes their TLB write. The R4400 keeps the whole PFN. The values are the PROM + /// diagnostic's patterns on 16 KB pages, where EntryLo bits 7:6 go unused. + /// Translation is the same on both: those bits never reach the address. + #[test] + fn tlb_write_clears_pfn_bits_below_the_page_size_on_the_r10000() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + // Write (lo0, lo1) with PageMask 0x6000 through TLBWI or TLBWR, then + // find the entry with TLBP and TLBR it back. Returns (EntryLo0, EntryLo1, physical address of + // VA + 0x3000). + fn entry>>(funct: u32, lo0: u64, lo1: u64) -> (u64, u64, u64) { + const VA: u64 = 0x4000_0000; + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()) as Arc, crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; + let dmtc0 = |rd: u32| (OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11); + for (rd, value) in [(12, status as u64), (10, VA), (0, 0), (5, 0x6000), (2, lo0), (3, lo1)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec(dmtc0(rd)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | funct), EXEC_COMPLETE); + let phys = exec.debug_translate(VA + 0x3000).phys; + // TLBWR picked the slot: find it with TLBP. + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBP), EXEC_COMPLETE); + for rd in [2, 3] { + exec.core.write_gpr(1, 0); + assert_eq!(exec.exec(dmtc0(rd)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBR), EXEC_COMPLETE); + (exec.core.read_cp0(2), exec.core.read_cp0(3), phys) + } + on_big_stack(|| { + for funct in [FUNCT_TLBWI, FUNCT_TLBWR] { + let (lo0, lo1) = (0xFFFF_FFFF_A5A5_A5A4, 0x5A5A_5A58); + // EntryLo 7:6 cleared; 63:62 and 33:6 otherwise kept. + assert_eq!(entry::(funct, lo0, lo1).0, 0xC000_0003_A5A5_A524, "R10000 {funct:#x}"); + assert_eq!(entry::(funct, lo0, lo1).1, 0x5A5A_5A18, "R10000 {funct:#x}"); + assert_eq!(entry::(funct, lo1, lo0).0, 0x5A5A_5A18, "R10000 {funct:#x}"); + // The R4400 keeps bits 29:0 as written. + assert_eq!(entry::(funct, lo0, lo1).0, 0x25A5_A5A4, "R4400 {funct:#x}"); + assert_eq!(entry::(funct, lo0, lo1).1, 0x1A5A_5A58, "R4400 {funct:#x}"); + + // A 16 KB page at 0x0300_4000 written with PFN bits for + // 0x0300_7000: both parts translate from the 16 KB frame. + let lo = ((0x0300_7000u64 >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + let (r10k, _, r10k_pa) = entry::(funct, lo, 0); + let (r4k, _, r4k_pa) = entry::(funct, lo, 0); + assert_eq!((r10k, r10k_pa), (lo & !0xC0, 0x0300_7000), "R10000 {funct:#x}"); + assert_eq!((r4k, r4k_pa), (lo, 0x0300_7000), "R4400 {funct:#x}"); + } + }); + } + + /// On the R10000 family EntryLo holds a 28-bit PFN (33:6) and the uncached + /// attribute in 63:62 (R10000 UM v2.0 pp. 239-240); the R4400 and R5000 keep a + /// 24-bit PFN and nothing above it. The TLB carries the whole PFN. + #[test] + fn entrylo_covers_the_pfn_and_attribute_bits_of_the_r10000_family() { + fn readback>>() -> u64 { + let bus: Arc = Arc::new(MockMemory::new()); + let mut exec = MipsExecutor::::new(bus, PassthroughTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + exec.core.write_gpr(1, u64::MAX); + // DMTC0 $1, EntryLo0 and EntryLo1; both read back the same field. + for reg in [2u32, 3] { + assert_eq!(exec.exec((0x10 << 26) | (5 << 21) | (1 << 16) | (reg << 11)), EXEC_COMPLETE); + } + let (lo0, lo1) = (exec.core.read_cp0(2), exec.core.read_cp0(3)); + assert_eq!(lo0, lo1); + lo0 + } + assert_eq!(readback::(), 0x3FFF_FFFF); + // Bits 61:34 are not PFN: they read back as zero, so only 33:32 reach physical 39:38. + assert_eq!(readback::(), 0xC000_0003_FFFF_FFFF); + + // The TLB carries the whole PFN: bits 33:32 are physical 39:38, and FrameMask + // is what keeps software flags there out of the address. + on_big_stack(|| { + const VA: u64 = 0x4000_0000; + let mut exec = map_page::( + Arc::new(MockMemory::new()), VA, 0xC0_2054_C000); + assert_eq!(exec.debug_translate(VA).phys, 0xC0_2054_C000); + // FrameMask 0xC000 is EntryLo 33:32: the page lands where its PFN says. + let mut exec = map_page_masked::( + Arc::new(MockMemory::new()), VA, 0xC0_2054_C000, 0xC000); + assert_eq!(exec.debug_translate(VA).phys, 0x2054_C000); + }); + } + /// The Indy, Indigo2 and O2 decode 32 physical bits, and IRIX relies on /// it: on the IP28 the kernel stack's wired entry carries PTE software /// bits as PA[34:33]. `phys_mask` must fold them away before the bus. @@ -5688,6 +5890,57 @@ mod tests { assert_ne!(st & EXEC_IS_XTLB_REFILL, 0, "true xkseg is unchanged: XTLB"); } + /// The R10000 picks the refill vector by the address space a missing + /// address lies in and that space's Status bit: user addresses follow + /// UX, xsseg SX, kernel and compat supervisor addresses KX (R10000 UM + /// v2.0 pp. 333-334). The R4400 picks by the current mode. A 64-bit + /// kernel (KX=1, UX=0) missing on kuseg therefore takes the 32-bit refill + /// vector on the R10000 and XTLB on the R4400; the PROM's utlb + /// diagnostic waits at the 32-bit vector for exactly that miss. + #[test] + fn r10000_refill_vector_follows_the_address_space() { + use crate::cpu::mips_cache_shadow::R10000ShadowCache; + use crate::cpu::mips_core::{STATUS_ERL, STATUS_EXL, STATUS_SX, STATUS_UX}; + use crate::cpu::mips_exec::EXEC_IS_XTLB_REFILL; + // Kernel mode with `xbits` set in Status, empty TLB; is a read of + // `va` a refill miss through XTLB? + fn xrefill>>(xbits: u32, va: u64) -> bool { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()), crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + let status = (exec.core.cp0_status & !(STATUS_ERL | STATUS_EXL | STATUS_KX | STATUS_SX | STATUS_UX | 0x18)) | xbits; + exec.core.write_gpr(1, status as u64); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (12 << 11)), EXEC_COMPLETE); + let st = exec.read_data::<4>(va).expect_err("empty TLB must miss"); + assert_ne!(st & EXEC_IS_TLB_REFILL, 0, "{va:#x} should be a refill miss"); + st & EXEC_IS_XTLB_REFILL != 0 + } + const KUSEG: u64 = 0; + const XSSEG: u64 = 0x4000_0000_0000_0000; + const CKSSEG: u64 = 0xFFFF_FFFF_C000_0000; + const XKSEG: u64 = 0xC000_0000_0000_0000; + on_big_stack(|| { + type R10k = R10000ShadowCache; + // 64-bit kernel, 32-bit user space: kuseg refills through 0x000. + assert!(!xrefill::(STATUS_KX, KUSEG)); + assert!(xrefill::(STATUS_KX, KUSEG), "R4400 follows the mode"); + assert!(xrefill::(STATUS_KX | STATUS_UX, KUSEG)); + // xsseg follows SX, compat ksseg and xkseg follow KX. + assert!(!xrefill::(STATUS_KX, XSSEG)); + assert!(xrefill::(STATUS_KX | STATUS_SX, XSSEG)); + assert!(xrefill::(STATUS_KX, CKSSEG)); + assert!(xrefill::(STATUS_KX, XKSEG)); + // 32-bit kernel: kseg3-side addresses use 0x000, kuseg follows UX. + assert!(!xrefill::(0, CKSSEG)); + assert!(!xrefill::(STATUS_UX, CKSSEG)); + assert!(!xrefill::(0, KUSEG)); + assert!(xrefill::(STATUS_UX, KUSEG)); + assert!(!xrefill::(STATUS_UX, KUSEG), "R4400 follows the mode"); + }); + } + #[test] fn test_xtlb_compare_width_disambiguates_region_field() { std::thread::Builder::new() @@ -6418,4 +6671,236 @@ mod tests { } assert_eq!(outcome[0], outcome[1], "an explanation must not change what the guest sees"); } + + /// Run `f` on a thread with room for a `MipsTlb`, which is large. + fn on_big_stack(f: impl FnOnce() + Send + 'static) { + std::thread::Builder::new().stack_size(16 * 1024 * 1024).spawn(f).unwrap().join().unwrap(); + } + + /// A 64-bit kernel executor with one TLB entry, written through CP0 and + /// TLBWI, that maps the page at `va` to `phys` (cached, valid, dirty). + fn map_page>>( + bus: Arc, va: u64, phys: u64, + ) -> Box> { + map_page_masked::(bus, va, phys, 0) + } + + /// `map_page` with CP0 FrameMask (register 21) written before the TLB write. + fn map_page_masked>>( + bus: Arc, va: u64, phys: u64, framemask: u64, + ) -> Box> { + map_page_with::(bus, va, phys, framemask, FUNCT_TLBWI) + } + + /// `map_page_masked`, writing the entry with `funct` (TLBWI or TLBWR). + fn map_page_with>>( + bus: Arc, va: u64, phys: u64, framemask: u64, funct: u32, + ) -> Box> { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::new(bus, crate::cpu::mips_tlb::MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + // MTC0 must refresh the boxed executor's translation/accessor state. + exec.install_status_cb(); + let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; + let entrylo = ((phys >> 12) << 6) | (3 << 3) | (1 << 2) | (1 << 1); + for (rd, value) in [(12, status as u64), (10, va), (0, 0), (5, 0), (21, framemask), (2, entrylo), (3, 0)] { + exec.core.write_gpr(1, value); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11)), EXEC_COMPLETE); + } + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | funct), EXEC_COMPLETE); + exec + } + + /// EntryHi and the TLB keep every implemented VPN2 bit: 44 virtual-address + /// bits on the R10000, 40 on the R4400. An IRIX n64 stack can sit + /// above the 40-bit limit. + #[test] + fn cp0_entryhi_and_tlb_match_full_xuseg_va() { + on_big_stack(|| { + const VA: u64 = 0x0000_0123_4000_0000; + fn check>>(full_va: bool) { + let mem = Arc::new(MockMemory::new()); + mem.set_word(0x5_0000, 0xdead_beef); + let mut exec = map_page::(mem, VA, 0x5_0000); + let result = exec.read_data::<4>(VA); + if full_va { + assert_eq!(exec.core.cp0_entryhi, VA); + assert_eq!(result, Ok(0xdead_beef)); + } else { + assert_eq!(exec.core.cp0_entryhi, VA & 0xC000_00FF_FFFF_E0FF); + assert_eq!((result.unwrap_err() >> 2) & 0x1f, EXC_ADEL); + } + } + check::(false); + check::(true); + }); + } + + /// A bus where nothing answers: every read and write is refused (the + /// trait's defaults). + struct RefusingBus; + impl BusDevice for RefusingBus {} + + /// R10000: a store the system refuses leaves the processor as a write + /// request, which has no response, so it completes (R10000 UM pp. 104, + /// 316). An uncached SC succeeding is a model choice. A load keeps its + /// Data Bus Error, and so does every access on the R5000. SWL goes through + /// the masked-store path, a separate site. + #[test] + fn refused_uncached_store_completes_only_on_the_r10000() { + fn run>>(op: u32) -> (u32, u64) { + let bus: Arc = Arc::new(RefusingBus); + let mut exec = MipsExecutor::::new(bus, PassthroughTlb::default(), &MipsCpuConfig::indy()); + disable_jitv2(&mut exec); + exec.cache.set_llbit(true); + exec.core.write_gpr(4, 0xFFFF_FFFF_A100_0000); // kseg1: uncached + exec.core.write_gpr(6, 0x1234_5678); + let st = exec.exec(make_i(op, 4, 6, 4)); + if st == EXEC_COMPLETE { + assert_eq!(exec.core.cp0_status & crate::cpu::mips_core::STATUS_EXL, 0, "no exception taken"); + assert_eq!(exec.exec(make_i(OP_ADDIU, 0, 5, 7)), EXEC_COMPLETE); + assert_eq!(exec.core.read_gpr(5), 7, "the next instruction runs"); + } + (st, exec.core.read_gpr(6)) + } + use crate::traits::BUS_ERR; + type R10K = crate::cpu::mips_cache_v2::R10000Cache; + type R10KShadow = crate::cpu::mips_cache_shadow::R10000ShadowCache; + type R5K = crate::cpu::mips_cache_shadow::R5000ShadowCache; + for (op, name) in [(OP_SW, "SW"), (OP_SWL, "SWL")] { + assert_eq!(run::(op).0, EXEC_COMPLETE, "R10000 {name}"); + assert_eq!(run::(op).0, EXEC_COMPLETE, "R10000 shadow {name}"); + assert_eq!(run::(op).0, BUS_ERR, "R5000 {name}"); + } + assert_eq!(run::(OP_SC), (EXEC_COMPLETE, 1), "SC succeeds"); + assert_eq!(run::(OP_SC).0, BUS_ERR, "R5000 SC"); + assert_eq!(run::(OP_LW).0, BUS_ERR, "a refused load still faults"); + } + + #[test] + fn refused_store_reports_are_sparse() { + let shown: Vec = (1..=40).filter(|&n| crate::cpu::mips_exec::report_refused_store(n)).collect(); + assert_eq!(shown, [1, 2, 3, 4, 8, 16, 32]); + } + + /// The O2's no-ECC alias of RAM at 0x8000_0000 is matched on the address the + /// board decodes (32 bits), so a PTE software bit above the PFN cannot make + /// a mapped page miss the alias window. + #[test] + fn phys_alias_is_matched_after_the_physical_mask() { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + use crate::cpu::mips_tlb::MipsTlb; + let (mut exec, _) = create_executor_with_tlb(MipsTlb::default()); + exec.core.cp0_status &= !crate::cpu::mips_core::STATUS_ERL; + exec.phys_mask = u32::MAX as u64; + exec.phys_alias = Some((0x8000_0000, 0x1000_0000, 0)); + // PA 0x1_8000_0000: bit 32 is above what the board decodes. + exec.core.cp0_index = 0; + exec.core.cp0_pagemask = 0; + exec.core.cp0_entryhi = 0x4000_0000; + exec.core.cp0_entrylo0 = (0x18_0000 << 6) | (3 << 3) | (1 << 2) | (1 << 1); + exec.core.cp0_entrylo1 = 0; + assert_eq!(exec.exec((OP_COP0 << 26) | (0x10 << 21) | 0x02), EXEC_COMPLETE); + assert_eq!(exec.debug_translate(0x4000_0000).phys, 0); + }) + .unwrap().join().unwrap(); + } + + /// A partial LWR keeps bits 63:32 of the destination on the R10000 while + /// 64-bit operations are enabled for the current mode, and sign-extends on + /// the R4400 and R5000 and on the R10000 when they are off (a 32-bit user + /// process, UX=0); a complete word sign-extends on all. + /// Compiled LWR reads `core.lwr_upper_below` rather than decoding Status, + /// so it must follow `is_64bit_mode()` through a Status write and through + /// ERET, which changes the mode without writing Status, and stay 0 on a + /// model that does not keep the upper word. + #[test] + fn lwr_upper_below_follows_the_mode_through_status_writes_and_eret() { + use crate::cpu::mips_core::{STATUS_EXL, STATUS_UX}; + fn check>>(keeps_upper: bool) { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + disable_jitv2(&mut exec); + let expect = |exec: &MipsExecutor| { + if keeps_upper && exec.core.is_64bit_mode() { 3 } else { 0 } + }; + for status in 0..=0xffu32 { + exec.set_cp0_status(status); + assert_eq!(exec.core.lwr_upper_below, expect(&exec), "{} Status {status:#x}", C::NAME); + } + exec.set_cp0_status((2 << 3) | STATUS_UX | STATUS_EXL); + assert_eq!(exec.core.lwr_upper_below, 0, "EXL forces kernel mode, where KX is clear"); + exec.core.cp0_epc = 0x1000; + assert_eq!(exec.exec(0x4200_0018), EXEC_COMPLETE, "ERET"); + assert_eq!(exec.core.lwr_upper_below, if keeps_upper { 3 } else { 0 }, "ERET returns to user mode with UX set"); + } + check::(true); + check::(false); + } + + #[test] + fn partial_lwr_keeps_the_upper_word_only_on_the_r10000() { + use crate::cpu::mips_core::{STATUS_KX, STATUS_SX, STATUS_UX}; + const USER: u32 = 2 << 3; + const SUPER: u32 = 1 << 3; + fn check>>(model_keeps_upper: bool) { + let mem = Arc::new(MockMemory::new()); + mem.set_word(0x2000, 0x89ab_cdef); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new(mem, PassthroughTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.core.write_gpr(1, 0x2000); + let old = 0x8123_4567_a1b2_c3d4u64; + // (Status, 64-bit operations on): each mode's own bit decides, + // whatever the other two hold. + for (status, ops64, mode) in [ + (STATUS_KX, true, "kernel KX=1"), + (0, false, "kernel KX=0"), + (USER | STATUS_UX, true, "user UX=1"), + (USER | STATUS_KX | STATUS_SX, false, "user UX=0"), + (SUPER | STATUS_SX, true, "supervisor SX=1"), + (SUPER | STATUS_KX | STATUS_UX, false, "supervisor SX=0"), + ] { + exec.core.cp0_status = status; + for (offset, word) in [0xa1b2_c389u32, 0xa1b2_89ab, 0xa189_abcd, 0x89ab_cdef].into_iter().enumerate() { + exec.core.write_gpr(2, old); + assert_eq!(exec.exec(make_i(OP_LWR, 1, 2, offset as u16)), EXEC_COMPLETE); + let expected = if model_keeps_upper && ops64 && offset != 3 { + (old & 0xffff_ffff_0000_0000) | word as u64 + } else { + word as i32 as i64 as u64 + }; + assert_eq!(exec.core.read_gpr(2), expected, "{} LWR offset {offset}, {mode}", C::NAME); + } + } + } + check::(false); + check::(false); + check::(true); + } + + /// FrameMask and the wide EntryLo belong to the whole R10000 family, not to + /// the parts that share the R10000's partial LWR. No R12000 type exists on + /// this branch, so this cannot run a family part whose LWR differs from + /// the R10000's; it checks the executor's masks against the family constant + /// the gate reads, and that the two behaviours are separate constants. + #[test] + fn framemask_and_entrylo_widths_follow_the_family_constant() { + fn check>>() { + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let exec = Box::new(MipsExecutor::::new(Arc::new(MockMemory::new()), PassthroughTlb::default(), &cfg)); + let (framemask, entrylo) = if C::R10K_FAMILY { (0xFFFF, 0xC000_0003_FFFF_FFFF) } else { (0, 0x3FFF_FFFF) }; + assert_eq!(exec.core.framemask_mask, framemask, "{} FrameMask", C::NAME); + assert_eq!(exec.core.entrylo_mask, entrylo, "{} EntryLo", C::NAME); + } + use crate::cpu::mips_cache_shadow::{R10000ShadowCache, R4400ShadowCache, R5000ShadowCache}; + assert!(R10000ShadowCache::R10K_FAMILY); + assert!(!R4400ShadowCache::R10K_FAMILY && !R5000ShadowCache::R10K_FAMILY); + check::(); + check::(); + check::(); + } } diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 05680e0c6..7315c28ba 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -295,6 +295,11 @@ pub enum TlbResult { }, } +/// VPN2 bits of a virtual address `bits` wide: bit 13 up to `bits - 1`. +pub const fn vpn2_mask(bits: u32) -> u64 { + ((1u64 << bits) - 1) & !0x1fff +} + /// TLB interface for MIPS address translation /// /// The TLB translates virtual page numbers to physical page numbers. @@ -351,6 +356,19 @@ pub trait Tlb { fn power_on(&mut self) {} /// Install the CPU model's meaning for each EntryLo C value. fn set_cache_attrs(&mut self, _table: [CacheAttr; 8]) {} + /// Set the implemented VPN width before installing entries. + fn set_va_bits(&mut self, _bits: u32) {} + /// Make a TLB write invalidate every other entry it would match together + /// with, as the R10000 and R12000 do (NEC U10278EJ4V0UM, Status register + /// TS bit, p.223, and "Avoiding TLB Conflict", p.300): they do not shut + /// down on a multiple match. The R14000 and R16000 are given the same + /// rule as a model choice, matching QEMU's r10k_invalidate_conflicts(); + /// no manual for them was read. Off for the R4400 and R5000, whose + /// behaviour on a multiple match is not modelled. + fn set_conflict_invalidation(&mut self, _on: bool) {} + /// Whether a write since the last call invalidated another entry. The + /// executor reports it in Status.TS. + fn take_write_conflict(&mut self) -> bool { false } fn save_state(&self) -> toml::Value { toml::Value::Table(Default::default()) } fn load_state(&mut self, _v: &toml::Value) -> Result<(), String> { Ok(()) } @@ -534,6 +552,7 @@ pub struct MipsTlb { /// Live JTLB entries for the CPU model in use (<= TLB_MAX_ENTRIES). num_entries: usize, + vpn_mask64: u64, /// What each EntryLo C value means on the CPU model in use; the /// executor installs its table with `set_cache_attrs`. c_attr: [CacheAttr; 8], @@ -552,9 +571,74 @@ pub struct MipsTlb { vmap_touched: std::collections::HashSet, #[cfg(feature = "tlbstats")] pub stats: TlbStats, + /// R10000 family: a write invalidates the entries it overlaps. See + /// [`Tlb::set_conflict_invalidation`]. + invalidate_on_write: bool, + /// Entries written since power-on; a never-written slot cannot conflict. + written: u64, + /// A write invalidated another entry since `take_write_conflict`. + write_conflict: bool, + /// Some entry written since power-on has a PageMask that is not a run of + /// ones from bit 13 (software can write any of bits 24:13). The map + /// ranges of such an entry are not exact, so `invalidate_overlapped` then + /// scans instead of asking the map. + irregular_mask: bool, } impl MipsTlb { + /// The R10000 family never lets a write create a multiple match: every + /// other entry the incoming one overlaps, wired or not, is taken out of + /// matching before the write lands, and Status.TS records that it + /// happened (see [`Tlb::set_conflict_invalidation`]). Overlap is + /// a lookup that could match both: the region and page number agree + /// under the wider of the two page masks, and the ASIDs agree or either + /// entry is global. Entries that are invalid count: they match and + /// fault. The entry reads back as a never-written slot does, zeros + /// except for its PageMask. Runs on the write path only; a lookup is unchanged. + fn invalidate_overlapped(&mut self, index: usize, new: &TlbEntry) { + // An entry that overlaps the new one covers a lookup-map slot the new + // one covers (the map is keyed on VA[31:13], and agreeing on more bits + // agrees on these), and a slot covered by two entries is tagged + // VMAP_MULTI or names the other one. So when every slot of the new + // entry's range is empty or names only `index` (the entry being + // replaced), nothing overlaps and the scan can be skipped. + if !self.irregular_mask { + let (start, count) = Self::vmap_range(new); + let tag = index as u8; + if self.vmap[start..start + count].iter().all(|&v| v == VMAP_MISS || v == tag) { + return; + } + } + let mut live = self.written & !(1u64 << index); + while live != 0 { + let i = live.trailing_zeros() as usize; + live &= live - 1; + if i >= self.num_entries { + continue; + } + let other = &self.entries[i]; + if !(new.is_global() || other.is_global() || new.asid() == other.asid()) { + continue; + } + let both = new.page_mask | other.page_mask | 0x1FFF; + if (new.entry_hi ^ other.entry_hi) & !both & self.vpn_mask64 != 0 { + continue; + } + let old_range = Self::vmap_range(other); + // TLBR of the entry reads zeros for EntryHi and EntryLo but still + // loads PageMask: software that sets PageMask once and relies on a + // read-back restoring it would otherwise write its next entry + // with a 4 KiB mask. + let mut gone = TlbEntry::new(); + gone.page_mask = other.page_mask; + self.entries[i] = gone; + self.shadow[i] = ShadowEntry::invalid(); + self.written &= !(1u64 << i); + self.vmap_vacate(i, old_range); + self.write_conflict = true; + } + } + pub fn new(num_entries: usize) -> Self { assert!(num_entries > 1 && num_entries <= TLB_MAX_ENTRIES, "MipsTlb supports 2..={} entries, got {}", TLB_MAX_ENTRIES, num_entries); @@ -564,12 +648,17 @@ impl MipsTlb { mru_head: [0u8; MRU_LISTS], mru_next: [[MRU_NONE; TLB_MAX_ENTRIES]; MRU_LISTS], num_entries, + vpn_mask64: 0xC000_00FF_FFFF_E000, c_attr: DEFAULT_C_ATTR, vmap: [VMAP_MISS; VMAP_SIZE], #[cfg(feature = "tlbcheck")] vmap_touched: std::collections::HashSet::new(), #[cfg(feature = "tlbstats")] stats: TlbStats::default(), + invalidate_on_write: false, + written: 0, + write_conflict: false, + irregular_mask: false, }; for list in 0..MRU_LISTS { tlb.mru_head[list] = 0; @@ -669,6 +758,15 @@ impl MipsTlb { /// step 3 that it now also aliases. #[inline] fn vmap_replace(&mut self, index: usize, old_range: (usize, usize)) { + self.vmap_vacate(index, old_range); + let (new_start, new_count) = Self::vmap_range(&self.entries[index]); + self.vmap_fill_range(index, new_start, new_count); + } + + /// Steps 1 to 3 of [`Self::vmap_replace`]: take `index` out of the map + /// and let the other entries reclaim what it covered. + #[inline] + fn vmap_vacate(&mut self, index: usize, old_range: (usize, usize)) { let (old_start, old_count) = old_range; self.vmap_clear(old_start, old_count); let old_end = old_start + old_count; @@ -685,8 +783,6 @@ impl MipsTlb { self.vmap_fill_range(i, lo, hi - lo); } } - let (new_start, new_count) = Self::vmap_range(&self.entries[index]); - self.vmap_fill_range(index, new_start, new_count); } /// Full internal consistency check (feature = "tlbcheck"). @@ -1065,7 +1161,7 @@ impl Tlb for MipsTlb { entry.selector_bit_shift = (mask.trailing_ones() - 1) as u8; entry.vcmp32 = !mask & 0x0000_0000_FFFF_E000; entry.vpn_hi32 = entry.entry_hi & entry.vcmp32; - entry.vcmp64 = !mask & 0xC000_00FF_FFFF_E000; + entry.vcmp64 = !mask & self.vpn_mask64; entry.vpn_hi64 = entry.entry_hi & entry.vcmp64; entry.offset_mask = (1u64 << entry.selector_bit_shift) - 1; let pfn_mask = 0xFF_FFFF_FFFF_000 & !entry.offset_mask; @@ -1073,6 +1169,11 @@ impl Tlb for MipsTlb { (entry.entry_lo[0] << 6) & pfn_mask, (entry.entry_lo[1] << 6) & pfn_mask, ]; + self.irregular_mask |= mask & (mask + 1) != 0; + if self.invalidate_on_write { + self.invalidate_overlapped(index, &entry); + } + self.written |= 1 << index; let old_range = Self::vmap_range(&self.entries[index]); self.entries[index] = entry; self.shadow[index] = ShadowEntry::from_entry(&entry, &self.c_attr); @@ -1194,6 +1295,18 @@ impl Tlb for MipsTlb { output } + fn set_conflict_invalidation(&mut self, on: bool) { + self.invalidate_on_write = on; + } + + fn take_write_conflict(&mut self) -> bool { + std::mem::take(&mut self.write_conflict) + } + + fn set_va_bits(&mut self, bits: u32) { + self.vpn_mask64 = 0xC000_0000_0000_0000 | vpn2_mask(bits); + } + fn set_cache_attrs(&mut self, table: [CacheAttr; 8]) { self.c_attr = table; for i in 0..self.num_entries { @@ -1212,6 +1325,9 @@ impl Tlb for MipsTlb { self.mru_next[list][self.num_entries - 1] = MRU_NONE; } self.vmap.fill(VMAP_MISS); + self.written = 0; + self.write_conflict = false; + self.irregular_mask = false; #[cfg(feature = "tlbcheck")] { self.vmap_touched.clear(); } } @@ -1237,7 +1353,7 @@ impl Tlb for MipsTlb { let entry_hi = words[1]; let mask = page_mask | 0x1FFF; let vcmp32 = !mask & 0x0000_0000_FFFF_E000; - let vcmp64 = !mask & 0xC000_00FF_FFFF_E000; + let vcmp64 = !mask & self.vpn_mask64; let shift = (mask.trailing_ones() - 1) as u8; let offset_mask = (1u64 << shift) - 1; let lo = [words[2], words[3]]; @@ -1268,6 +1384,11 @@ impl Tlb for MipsTlb { self.mru_next[list][self.num_entries - 1] = MRU_NONE; } self.vmap.fill(VMAP_MISS); + self.written = !0u64 >> (64 - self.num_entries); + self.irregular_mask = self.entries[..self.num_entries].iter().any(|e| { + let mask = e.page_mask | 0x1FFF; + mask & (mask + 1) != 0 + }); for i in 0..self.num_entries { let (start, count) = Self::vmap_range(&self.entries[i]); self.vmap_fill_range(i, start, count); diff --git a/src/cpu/mips_tlb_test.rs b/src/cpu/mips_tlb_test.rs index 4ebbaeda1..d594d6e74 100644 --- a/src/cpu/mips_tlb_test.rs +++ b/src/cpu/mips_tlb_test.rs @@ -4,6 +4,195 @@ mod tests { use crate::cpu::mips_tlb::*; use crate::cpu::mips_exec::CacheAttr; + /// Region 3 base for the overlap tests below; the frame is arbitrary. + const BASE: u64 = 0xc000_0000_0000_0000; + + fn entry(mask: u64, hi: u64, lo: [u64; 2]) -> TlbEntry { + let mut e = TlbEntry::new(); + e.page_mask = mask; + e.entry_hi = hi; + e.entry_lo = lo; + e + } + + /// CPU 1 of a two-processor Tezro as its TLB looks after the firmware's + /// flush (every entry written invalid: region 3, page number 0, ASID 0, + /// not global, 2 MiB pages) and the kernel's write of the valid global + /// 16 MiB entry at index 1 over the same page number. The firmware + /// entries that overlap it are true duplicates: every compared bit agrees. + fn flushed_then_kernel_mapped(invalidate: bool) -> MipsTlb { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(invalidate); + for index in (0..64).rev() { + tlb.write(index, entry(0x1fe000, BASE, [0; 2])); + } + // PFN 0x40, cacheable, dirty, valid, global, both halves. + tlb.write(1, entry(0x1ffe000, BASE | 0x1000, [(0x40 << 6) | (5 << 3) | 0x7; 2])); + tlb + } + + /// An entry elsewhere, as the kernel writes at index 0 next (wired). + fn write_wired_elsewhere(tlb: &mut MipsTlb) { + tlb.write(0, entry(0x6000, 0xc000_0fff_ffff_9000, [1, 0x2e42f])); + } + + /// Writing the valid entry takes the overlapped firmware entries out of + /// matching, as the R10000 family does, so the fetch the kernel makes + /// next is answered by the valid entry however the lookup map was + /// rebuilt in between. Without the rule the highest-numbered firmware + /// entry answers, as invalid. + #[test] + fn a_write_invalidates_the_entries_it_overlaps() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let va = BASE + 0x1234; + let mut off = flushed_then_kernel_mapped(false); + write_wired_elsewhere(&mut off); + assert!(matches!(off.translate::<1>(va, 0, AccessType::Fetch), TlbResult::Invalid { .. }), + "without the rule, the duplicate answers"); + assert!(!off.take_write_conflict()); + assert_eq!(off.read(2).entry_hi, BASE, "without the rule, the duplicate stays"); + + let mut tlb = flushed_then_kernel_mapped(true); + assert!(tlb.take_write_conflict(), "the writes overlapped entries"); + write_wired_elsewhere(&mut tlb); + match tlb.translate::<1>(va, 0, AccessType::Fetch) { + TlbResult::Hit { phys_addr, .. } => assert_eq!(phys_addr, 0x1234), + other => panic!("expected the valid entry to answer, got {other:?}"), + } + assert_eq!(tlb.read(2).entry_hi, 0); + assert_eq!(tlb.read(1).page_mask, 0x1ffe000); + // An entry in another region overlaps nothing and removes nothing. + tlb.write(5, entry(0x1fe000, 0x8000_0000_0000_0000, [0; 2])); + assert!(!tlb.take_write_conflict()); + assert!(matches!(tlb.translate::<1>(va, 0, AccessType::Fetch), TlbResult::Hit { .. })); + }).unwrap().join().unwrap(); + } + + /// The rule reaches the wired entries (the low indexes the kernel + /// reserves): a new entry over a wired one removes the wired one. + #[test] + fn a_write_invalidates_wired_entries_too() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let va = BASE + 0x2345; + let lo = |pfn: u64| [(pfn << 6) | (5 << 3) | 0x7; 2]; + let run = |on: bool| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(on); + tlb.write(0, entry(0, BASE | 0x2000, lo(0x10))); + tlb.write(40, entry(0, BASE | 0x2000, lo(0x20))); + tlb + }; + let on = run(true); + assert!(on.read(0).entry_hi == 0 && on.read(0).entry_lo == [0; 2], "wired entry 0 is gone"); + assert_eq!(on.read(40).entry_hi, BASE | 0x2000); + let mut on = on; + assert!(on.take_write_conflict()); + match on.translate::<1>(va, 0, AccessType::Read) { + TlbResult::Hit { phys_addr, .. } => assert_eq!(phys_addr, (0x20 << 12) | 0x345), + other => panic!("expected the new entry to answer, got {other:?}"), + } + let off = run(false); + assert_eq!(off.read(0).entry_hi, BASE | 0x2000, "without the rule the wired entry stays"); + }).unwrap().join().unwrap(); + } + + /// TLBR of an invalidated entry still loads its PageMask. The kernel + /// reads a wired sentinel back and reuses PageMask for its next write; a + /// zeroed mask makes that write 4 KiB. + #[test] + fn an_invalidated_entry_keeps_its_page_mask() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(true); + tlb.write(3, entry(0x1fe000, BASE, [0x7; 2])); + tlb.write(9, entry(0x1ffe000, BASE, [0x7; 2])); + let gone = tlb.read(3); + assert_eq!(gone.page_mask, 0x1fe000); + assert_eq!(gone.entry_hi, 0); + assert_eq!(gone.entry_lo, [0; 2]); + assert_eq!(tlb.read(9).page_mask, 0x1ffe000); + }).unwrap().join().unwrap(); + } + + /// Entries that cannot match together are left alone: a different ASID + /// when neither is global, and a page number that differs above the + /// wider mask. + #[test] + fn entries_that_cannot_match_together_are_kept() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(true); + tlb.write(3, entry(0, BASE | 0x4000 | 1, [0x6; 2])); // ASID 1, not global + tlb.write(4, entry(0, BASE | 0x4000 | 2, [0x6; 2])); // ASID 2 + tlb.write(5, entry(0, BASE | 0x8000 | 1, [0x6; 2])); // other page + assert!(!tlb.take_write_conflict()); + for i in [3, 4, 5] { + assert_ne!(tlb.read(i).entry_hi, 0, "entry {i} stays"); + } + // A global entry over that page does match both ASIDs. + tlb.write(6, entry(0, BASE | 0x4000 | 0x1000, [0x7; 2])); // global + assert!(tlb.take_write_conflict()); + assert_eq!(tlb.read(3).entry_hi, 0); + assert_eq!(tlb.read(4).entry_hi, 0); + assert_ne!(tlb.read(5).entry_hi, 0); + }).unwrap().join().unwrap(); + } + + /// The overlap rule written out directly: what a write must remove. + fn overlaps(new: &TlbEntry, other: &TlbEntry) -> bool { + if !(new.is_global() || other.is_global() || new.asid() == other.asid()) { + return false; + } + let both = new.page_mask | other.page_mask | 0x1FFF; + (new.entry_hi ^ other.entry_hi) & !both & 0xC000_00FF_FFFF_E000 == 0 + } + + /// Random writes against a plain list that applies the rule by comparing + /// every pair. The lookup map is how a write finds out it overlaps + /// nothing, so the TLB must agree with the list after every write, with + /// regular page masks and with masks that are not a run of ones. + #[test] + fn writes_remove_exactly_the_overlapped_entries() { + std::thread::Builder::new().stack_size(64 << 20).spawn(|| { + for masks in [&[0u64, 0x6000, 0x1e000, 0x7e000, 0x1fe000, 0x7fe000, 0x1ffe000][..], + &[0u64, 0x6000, 0x2000, 0xa000, 0x1fe000, 0x1ffe000][..]] { + let mut rng = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; rng }; + let mut tlb = MipsTlb::new(64); + tlb.set_conflict_invalidation(true); + let mut model: Vec> = vec![None; 64]; + for _ in 0..4000 { + let index = (next() % 64) as usize; + let mask = masks[(next() % masks.len() as u64) as usize]; + let region = if next() % 4 == 0 { 0x8000_0000_0000_0000 } else { BASE }; + // Few page numbers, so that writes collide often. + let page = (next() % 24) << 13; + let asid = next() % 3; + let global = if next() % 3 == 0 { 0x1000 } else { 0 }; + let new = entry(mask, region | page | global | asid, [(next() % 8) << 1; 2]); + let mut expect_conflict = false; + for (i, slot) in model.iter_mut().enumerate() { + if i != index && slot.as_ref().is_some_and(|o| overlaps(&new, o)) { + let mut gone = TlbEntry::new(); + gone.page_mask = slot.as_ref().unwrap().page_mask; + *slot = Some(gone); + expect_conflict = true; + } + } + model[index] = Some(new); + tlb.write(index, new); + assert_eq!(tlb.take_write_conflict(), expect_conflict); + for (i, slot) in model.iter().enumerate() { + let got = tlb.read(i); + let want = slot.unwrap_or_else(TlbEntry::new); + assert_eq!((got.entry_hi, got.entry_lo, got.page_mask), + (want.entry_hi, want.entry_lo, want.page_mask), "entry {i}"); + } + } + } + }).unwrap().join().unwrap(); + } + #[test] fn test_tlb_entry_flags() { let mut entry = TlbEntry::new(); diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index cdaa19b16..37051b4d8 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -599,8 +599,9 @@ impl OctaneMachine { // IRIX zeroes Count and arms Compare a few instructions later; see // `MipsCore::count_write_samples`. exec.core.count_write_samples = true; - // The R10000 drives 40 physical address bits; HEART decodes them - // all, and the attribute bits above fold away. + // The R10000 drives 40 physical address bits; HEART decodes them all. + // EntryLo bits 33:32 are physical 39:38; a guest that keeps software + // flags there clears them with FrameMask, so no narrowing is needed. exec.phys_mask = (1 << 40) - 1; exec.core.cpu_id = id; exec.core.smp = cpu_count > 1;